fun_ci 1.2.1 → 2.0.0

data/lib/fun_ci/persistence/null_recorder.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Persistence
+    # Records nothing, for a pipeline run with no database.
+    class NullRecorder
+      def create_run(**); end
+      def start_stage(_stage, **); end
+      def end_stage(_job_id, _status); end
+      def keep_exit(_job_id, _exit_status, _signal); end
+      def keep_evidence(_job_id, _document); end
+      def keep_raw(_job_id, _text); end
+      def alongside(_job_id) = []
+      def stage_process(_job_id, _pid); end
+      def slot_taken(_lock_file); end
+      def foreground_done; end
+      def db; end
+      def db_path; end
+      def pipeline_run_id; end
+      def close; end
+      def tolerating = yield
+      def report_trouble_to(_out); end
+    end
+  end
+end

data/lib/fun_ci/persistence/output_tail.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Persistence
+    # The end of what a stage printed, as fun-ci keeps it for a stage that
+    # failed (acceptance-tests.md, AT-9.5): the last 200 lines, at most 64 KB,
+    # colour codes stripped and bytes that aren't UTF-8 replaced. `mask`
+    # sees the last lines before they are cut to size, so no cut splits a
+    # secret it would have masked.
+    module OutputTail
+      LINES = 200
+      BYTES = 65_536
+      COLOUR = /\e\[[0-9;?]*[A-Za-z]/
+
+      def self.of(output, mask: :itself.to_proc)
+        text = output.dup.force_encoding(Encoding::UTF_8).scrub("?").gsub(COLOUR, "")
+        last_bytes(mask.call(text.lines.last(LINES).join))
+      end
+
+      def self.last_bytes(text) = text.byteslice([text.bytesize - BYTES, 0].max..).scrub("")
+      private_class_method :last_bytes
+    end
+  end
+end

data/lib/fun_ci/persistence/pipeline_recorder.rb

@@ -2,64 +2,109 @@
 
 require_relative "pipeline_run"
 require_relative "stage_job"
+require_relative "run_status"
+require_relative "write_trouble"
+require "json"
+require_relative "null_recorder"
+require_relative "project_runs"
+require_relative "retention"
 
 module FunCi
   module Persistence
-    class NullRecorder
-      def create_run(commit_hash:, branch:, project_path: nil) = nil
-      def start_stage(stage) = nil
-      def end_stage(job_id, status) = nil
-      def complete_run = nil
-      def fail_run = nil
-      def db = nil
-      def db_path = nil
-      def pipeline_run_id = nil
-      def close = nil
-    end
-
     class DbRecorder
-      attr_reader :db, :db_path, :pipeline_run_id
+      KEPT_RUNS = Retention::KEPT_RUNS
+
+      attr_reader :db, :db_path, :pipeline_run_id, :trouble
 
-      def self.for_background(db_path, pipeline_run_id)
+      def self.for_background(db_path, pipeline_run_id, trouble: WriteTrouble.silent(db_path))
         db = Database.connection(db_path)
-        new(db, pipeline_run_id: pipeline_run_id)
+        new(db, pipeline_run_id: pipeline_run_id, trouble: trouble)
       end
 
-      def initialize(db, pipeline_run_id: nil)
+      # `trouble` decides what a refused write does; by default, nothing is said.
+      def initialize(db, pipeline_run_id: nil, trouble: nil)
         @db = db
         @db_path = db.filename("main")
         @pipeline_run_id = pipeline_run_id
+        @trouble = trouble || WriteTrouble.silent(@db_path)
+      end
+
+      # Says what goes wrong writing to the database on `out` from now on.
+      def report_trouble_to(out)
+        @trouble = WriteTrouble.new(out, @db_path)
       end
 
+      # The block's value, or nil when the database refused a write in it.
+      def tolerating(&) = @trouble.guard(&)
+
       def close
-        @db.close rescue nil
+        @db.close
+      rescue StandardError
+        nil
       end
 
+      # Notes this process as the one running the pipeline, so cancelling can stop it.
       def create_run(commit_hash:, branch:, project_path: nil)
-        @pipeline_run_id = PipelineRun.create(@db, commit_hash: commit_hash, branch: branch, project_path: project_path)
+        tolerating do
+          @pipeline_run_id = PipelineRun.create(@db, commit_hash: commit_hash, branch: branch,
+                                                     project_path: project_path)
+          PipelineRun.store_trigger_pid(@db, @pipeline_run_id, Process.pid)
+          Retention.apply(@db, project_path) if project_path
+          @pipeline_run_id
+        end
+      end
+
+      def stage_process(job_id, pid)
+        tolerating { StageJob.store_pid(@db, job_id, pid) }
+      end
+
+      # Once it has exited its pid may be reused, so a cancel must not signal it.
+      def foreground_done
+        tolerating { PipelineRun.store_trigger_pid(@db, @pipeline_run_id, nil) }
       end
 
-      def start_stage(stage)
+      def slot_taken(lock_file)
+        tolerating { PipelineRun.store_slot_lock(@db, @pipeline_run_id, lock_file) }
+      end
+
+      def start_stage(stage, budget: nil)
         return nil unless @pipeline_run_id
-        ensure_running
-        job_id = StageJob.create(@db, pipeline_run_id: @pipeline_run_id, stage: stage)
-        StageJob.update_status(@db, job_id, "running")
-        job_id
+
+        tolerating do
+          ensure_running
+          job_id = StageJob.create(@db, pipeline_run_id: @pipeline_run_id, stage: stage, budget: budget)
+          StageJob.update_status(@db, job_id, "running")
+          job_id
+        end
       end
 
-      def end_stage(job_id, status)
-        return unless job_id
-        StageJob.update_status(@db, job_id, status)
+      # Keeps a failed stage's evidence, and the output's tail where an older
+      # fun-ci sharing the database reads it.
+      def keep_evidence(job_id, document)
+        tolerating do
+          StageJob.keep_evidence(@db, job_id, JSON.generate(document.to_h))
+          StageJob.keep_output(@db, job_id, document.tail)
+        end
       end
 
-      def complete_run
-        return unless @pipeline_run_id
-        PipelineRun.update_status(@db, @pipeline_run_id, "completed")
+      # Keeps what a failed stage printed, masked and cut to its window.
+      def keep_raw(job_id, text)
+        tolerating { RawOutputs.beside(@db_path).write(job_id, text) }
       end
 
-      def fail_run
-        return unless @pipeline_run_id
-        PipelineRun.update_status(@db, @pipeline_run_id, "failed")
+      # The stages that shared the slot with this one (architecture.md, "Evidence of a failed stage").
+      def alongside(job_id) = tolerating { StageJob.alongside(@db, job_id) } || []
+
+      def keep_exit(job_id, exit_status, signal)
+        tolerating { StageJob.keep_exit(@db, job_id, exit_status, signal) }
+      end
+
+      # Records the stage's outcome, then settles the run's status from its stages.
+      def end_stage(job_id, status)
+        tolerating do
+          StageJob.update_status(@db, job_id, status)
+          RunStatus.settle(@db, @pipeline_run_id)
+        end
       end
 
       private
@@ -67,6 +112,7 @@ module FunCi
       def ensure_running
         run = PipelineRun.find(@db, @pipeline_run_id)
         return unless run && run[:status] == "scheduled"
+
         PipelineRun.update_status(@db, @pipeline_run_id, "running")
       end
     end

data/lib/fun_ci/persistence/pipeline_run.rb

@@ -5,39 +5,49 @@ require "time"
 module FunCi
   module Persistence
     module PipelineRun
+      COLUMNS = %i[id commit_hash branch status pid project_path created_at updated_at].freeze
+      SELECT = "SELECT #{COLUMNS.join(", ")} FROM pipeline_runs".freeze
+
       def self.create(db, commit_hash:, branch:, project_path: nil)
         now = Time.now.utc.iso8601
         db.execute(
-          "INSERT INTO pipeline_runs (commit_hash, branch, project_path, status, created_at, updated_at) VALUES (?, ?, ?, 'scheduled', ?, ?)",
+          "INSERT INTO pipeline_runs (commit_hash, branch, project_path, status, created_at, updated_at) " \
+          "VALUES (?, ?, ?, 'scheduled', ?, ?)",
           [commit_hash, branch, project_path, now, now]
         )
         db.last_insert_row_id
       end
 
       def self.find(db, id)
-        row = db.execute("SELECT id, commit_hash, branch, status, pid, project_path, created_at, updated_at FROM pipeline_runs WHERE id = ?", [id]).first
-        return nil unless row
-        row_to_hash(row)
+        query(db, "WHERE id = ?", id).first
       end
 
       def self.find_by_branch(db, branch)
-        rows = db.execute("SELECT id, commit_hash, branch, status, pid, project_path, created_at, updated_at FROM pipeline_runs WHERE branch = ? ORDER BY id DESC", [branch])
-        rows.map { |row| row_to_hash(row) }
+        query(db, "WHERE branch = ? ORDER BY id DESC", branch)
       end
 
       def self.find_by_commit(db, commit_hash)
-        rows = db.execute("SELECT id, commit_hash, branch, status, pid, project_path, created_at, updated_at FROM pipeline_runs WHERE commit_hash = ? ORDER BY id DESC", [commit_hash])
-        rows.map { |row| row_to_hash(row) }
+        query(db, "WHERE commit_hash = ? ORDER BY id DESC", commit_hash)
       end
 
+      # The forked slow suite's process.
       def self.store_pid(db, id, pid)
         db.execute("UPDATE pipeline_runs SET pid = ? WHERE id = ?", [pid, id])
       end
 
-      def self.find_running_with_pid(db, branch)
-        row = db.execute("SELECT id, commit_hash, branch, status, pid, project_path, created_at, updated_at FROM pipeline_runs WHERE branch = ? AND status = 'running' AND pid IS NOT NULL ORDER BY id DESC LIMIT 1", [branch]).first
-        return nil unless row
-        row_to_hash(row)
+      # The lock file of the worktree slot the run holds.
+      def self.store_slot_lock(db, id, path)
+        db.execute("UPDATE pipeline_runs SET slot_lock = ? WHERE id = ?", [path, id])
+      end
+
+      # The process running the pipeline itself.
+      def self.store_trigger_pid(db, id, pid)
+        db.execute("UPDATE pipeline_runs SET trigger_pid = ? WHERE id = ?", [pid, id])
+      end
+
+      # When an agent last polled the run, waiting on it (acceptance-tests.md, AT-9.12).
+      def self.mark_waited(db, id, at)
+        db.execute("UPDATE pipeline_runs SET waited_at = ? WHERE id = ?", [at, id])
       end
 
       def self.update_status(db, id, new_status)
@@ -46,14 +56,13 @@ module FunCi
       end
 
       def self.recent(db, limit: 20)
-        rows = db.execute("SELECT id, commit_hash, branch, status, pid, project_path, created_at, updated_at FROM pipeline_runs ORDER BY id DESC LIMIT ?", [limit])
-        rows.map { |row| row_to_hash(row) }
+        query(db, "ORDER BY id DESC LIMIT ?", limit)
       end
 
-      def self.row_to_hash(row)
-        { id: row[0], commit_hash: row[1], branch: row[2], status: row[3], pid: row[4], project_path: row[5], created_at: row[6], updated_at: row[7] }
+      def self.query(db, clause, param)
+        db.execute("#{SELECT} #{clause}", [param]).map { |row| COLUMNS.zip(row).to_h }
       end
-      private_class_method :row_to_hash
+      private_class_method :query
     end
   end
 end

data/lib/fun_ci/persistence/project_runs.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require_relative "pipeline_run"
+
+module FunCi
+  module Persistence
+    # The runs of one project. Every project records its runs in the same
+    # database, so each query is scoped to the project's path.
+    class ProjectRuns
+      def initialize(db, project)
+        @db = db
+        @project = project
+      end
+
+      def latest_of_commit(sha) = query("AND commit_hash = ? ORDER BY id DESC LIMIT 1", sha).first
+
+      # The commit of the first run on the same branch after this one, if any.
+      def superseded_by(run)
+        query("AND branch = ? AND id > ? ORDER BY id LIMIT 1", run[:branch], run[:id]).first&.fetch(:commit_hash)
+      end
+
+      # The newest runs first, on one branch when one is named.
+      def recent(limit:, branch: nil)
+        return query("ORDER BY id DESC LIMIT ?", limit) unless branch
+
+        query("AND branch = ? ORDER BY id DESC LIMIT ?", branch, limit)
+      end
+
+      # Drops the kept output and an older fun-ci's failures of every run but the
+      # project's newest `keep`, and marks their stages pruned.
+      def forget_output(keep:)
+        @db.execute("UPDATE stage_jobs SET output_tail = NULL, failures = NULL, pruned = 1 WHERE pipeline_run_id IN " \
+                    "(SELECT id FROM pipeline_runs WHERE project_path = ? ORDER BY id DESC LIMIT -1 OFFSET ?)",
+                    [@project, keep])
+      end
+
+      # The stages of every run but the project's newest `keep`.
+      def stage_ids_beyond(keep:)
+        @db.execute("SELECT id FROM stage_jobs WHERE pipeline_run_id IN " \
+                    "(SELECT id FROM pipeline_runs WHERE project_path = ? ORDER BY id DESC LIMIT -1 OFFSET ?)",
+                    [@project, keep]).flatten
+      end
+
+      private
+
+      def query(clause, *params)
+        @db.execute("#{PipelineRun::SELECT} WHERE project_path = ? #{clause}", [@project, *params])
+           .map { |row| PipelineRun::COLUMNS.zip(row).to_h }
+      end
+    end
+  end
+end

data/lib/fun_ci/persistence/raw_outputs.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "zlib"
+
+module FunCi
+  module Persistence
+    # What each failed stage printed, masked and cut to its window, kept as a
+    # gzip file per stage in a directory beside the database: SQLite would
+    # not give a deleted blob's space back without a VACUUM (architecture.md, "Evidence of a failed stage").
+    class RawOutputs
+      def self.beside(db_path) = new(File.join(File.dirname(db_path), "raw"))
+
+      def initialize(dir)
+        @dir = dir
+      end
+
+      def write(job_id, text)
+        FileUtils.mkdir_p(@dir)
+        Zlib::GzipWriter.open(path(job_id)) { |gzip| gzip.write(text) }
+      end
+
+      def read(job_id)
+        File.exist?(path(job_id)) ? Zlib::GzipReader.open(path(job_id), &:read) : nil
+      end
+
+      # How many bytes it kept, from the size gzip records at the file's end.
+      def bytes(job_id)
+        File.exist?(path(job_id)) ? File.binread(path(job_id), 4, File.size(path(job_id)) - 4).unpack1("V") : nil
+      end
+
+      def delete(job_ids) = job_ids.each { |job_id| FileUtils.rm_f(path(job_id)) }
+
+      # Deletes what it kept of every stage but those named.
+      def keep_only(job_ids)
+        kept = job_ids.to_set(&:to_s)
+        Dir.glob("*.gz", base: @dir).reject { |name| kept.include?(File.basename(name, ".gz")) }
+           .each { |name| FileUtils.rm_f(File.join(@dir, name)) }
+      end
+
+      private
+
+      def path(job_id) = File.join(@dir, "#{job_id}.gz")
+    end
+  end
+end

data/lib/fun_ci/persistence/retention.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require_relative "project_runs"
+require_relative "raw_outputs"
+require_relative "stage_job"
+
+module FunCi
+  module Persistence
+    # How long fun-ci keeps what failed stages left, applied when a run
+    # starts: the evidence for a project's 50 newest runs, the raw output for
+    # its 10 newest, and no raw output whose stage row is gone.
+    module Retention
+      KEPT_RUNS = 50
+      KEPT_RAW = 10
+
+      def self.apply(db, project)
+        runs = ProjectRuns.new(db, project)
+        runs.forget_output(keep: KEPT_RUNS)
+        raw = RawOutputs.beside(db.filename("main"))
+        raw.delete(runs.stage_ids_beyond(keep: KEPT_RAW))
+        raw.keep_only(StageJob.ids(db))
+      end
+    end
+  end
+end

data/lib/fun_ci/persistence/run_status.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require "time"
+
+module FunCi
+  module Persistence
+    # A run's status, worked out from its stages alone (acceptance-tests.md,
+    # AT-7.1): failed once any stage failed or timed out, completed once all
+    # four passed, and unchanged until then. One statement reads the stage rows
+    # and writes the run, so the foreground pipeline and the forked slow suite
+    # can't overwrite each other's verdict; failed, completed and cancelled
+    # runs are never touched again.
+    module RunStatus
+      OF_RUN = "pipeline_runs.id = stage_jobs.pipeline_run_id"
+      DERIVED = <<~SQL.freeze
+        CASE
+          WHEN EXISTS (SELECT 1 FROM stage_jobs WHERE #{OF_RUN} AND status IN ('failed', 'timed_out')) THEN 'failed'
+          WHEN (SELECT COUNT(DISTINCT stage) FROM stage_jobs WHERE #{OF_RUN} AND status = 'completed'
+                AND stage IN ('lint', 'build', 'fast', 'slow')) = 4 THEN 'completed'
+          ELSE status
+        END
+      SQL
+      SETTLE = "UPDATE pipeline_runs SET status = #{DERIVED}, updated_at = ? " \
+               "WHERE id = ? AND status IN ('scheduled', 'running') AND status != #{DERIVED}".freeze
+
+      def self.settle(db, id)
+        db.execute(SETTLE, [Time.now.utc.iso8601, id])
+      end
+    end
+  end
+end

data/lib/fun_ci/persistence/stage_job.rb

@@ -6,39 +6,84 @@ module FunCi
   module Persistence
     module StageJob
       TERMINAL_STATUSES = %w[completed failed timed_out cancelled].freeze
+      FIELDS = %i[id pipeline_run_id stage status started_at completed_at finished_order output_tail failures
+                  exit_status signal budget pruned evidence].freeze
+      COLUMNS = FIELDS.join(", ")
+      NEXT_IN_RUN = "(SELECT COALESCE(MAX(others.finished_order), 0) + 1 FROM stage_jobs AS others " \
+                    "WHERE others.pipeline_run_id = stage_jobs.pipeline_run_id)"
+      TIMESTAMP_COLUMNS = TERMINAL_STATUSES.to_h { |status| [status, "completed_at"] }
+                                           .merge("running" => "started_at").freeze
 
-      def self.create(db, pipeline_run_id:, stage:)
+      def self.create(db, pipeline_run_id:, stage:, budget: nil)
         db.execute(
-          "INSERT INTO stage_jobs (pipeline_run_id, stage, status) VALUES (?, ?, 'scheduled')",
-          [pipeline_run_id, stage]
+          "INSERT INTO stage_jobs (pipeline_run_id, stage, status, budget) VALUES (?, ?, 'scheduled', ?)",
+          [pipeline_run_id, stage, budget]
         )
         db.last_insert_row_id
       end
 
+      # The stage script's process, which leads a process group of its own.
+      def self.store_pid(db, id, pid)
+        db.execute("UPDATE stage_jobs SET pid = ? WHERE id = ?", [pid, id])
+      end
+
+      # The end of what the stage printed, kept when it failed (acceptance-tests.md, AT-9.5).
+      def self.keep_output(db, id, tail)
+        db.execute("UPDATE stage_jobs SET output_tail = ? WHERE id = ?", [tail, id])
+      end
+
+      # What was kept about why the stage failed, as JSON (architecture.md, "Evidence of a failed stage").
+      def self.keep_evidence(db, id, evidence_json)
+        db.execute("UPDATE stage_jobs SET evidence = ? WHERE id = ?", [evidence_json, id])
+      end
+
+      # How the stage's process exited: its status, or the signal that ended it.
+      def self.keep_exit(db, id, exit_status, signal)
+        db.execute("UPDATE stage_jobs SET exit_status = ?, signal = ? WHERE id = ?", [exit_status, signal, id])
+      end
+
+      # The other stages of its run that were running at any moment since it started.
+      def self.alongside(db, id)
+        db.execute("SELECT others.stage FROM stage_jobs AS others JOIN stage_jobs AS me ON me.id = ? " \
+                   "WHERE others.pipeline_run_id = me.pipeline_run_id AND others.id != me.id " \
+                   "AND others.started_at IS NOT NULL " \
+                   "AND (others.completed_at IS NULL OR others.completed_at > me.started_at) ORDER BY others.id",
+                   [id]).flatten
+      end
+
+      def self.ids(db) = db.execute("SELECT id FROM stage_jobs").flatten
+
       def self.find(db, id)
-        row = db.execute("SELECT id, pipeline_run_id, stage, status, started_at, completed_at FROM stage_jobs WHERE id = ?", [id]).first
+        row = db.execute("SELECT #{COLUMNS} FROM stage_jobs WHERE id = ?", [id]).first
         return nil unless row
+
         row_to_hash(row)
       end
 
+      # The run's jobs, in the order they were created.
+      def self.for_run(db, pipeline_run_id)
+        db.execute("SELECT #{COLUMNS} FROM stage_jobs WHERE pipeline_run_id = ? ORDER BY id", [pipeline_run_id])
+          .map { |row| row_to_hash(row) }
+      end
+
+      # A stage that finishes is numbered after every other of its run that
+      # finished before it, in the same statement, so the order holds however
+      # close together they finished.
       def self.update_status(db, id, new_status)
-        now = Time.now.utc.iso8601
-        if new_status == "running"
-          db.execute("UPDATE stage_jobs SET status = ?, started_at = ? WHERE id = ?", [new_status, now, id])
-        elsif TERMINAL_STATUSES.include?(new_status)
-          db.execute("UPDATE stage_jobs SET status = ?, completed_at = ? WHERE id = ?", [new_status, now, id])
-        else
-          db.execute("UPDATE stage_jobs SET status = ? WHERE id = ?", [new_status, id])
-        end
+        column = TIMESTAMP_COLUMNS.fetch(new_status)
+        order = TERMINAL_STATUSES.include?(new_status) ? NEXT_IN_RUN : "finished_order"
+        db.execute("UPDATE stage_jobs SET status = ?, #{column} = ?, finished_order = #{order} WHERE id = ?",
+                   [new_status, Time.now.utc.iso8601(3), id])
       end
 
       def self.elapsed_duration(job)
         return nil unless job[:started_at] && job[:completed_at]
+
         Time.parse(job[:completed_at]) - Time.parse(job[:started_at])
       end
 
       def self.row_to_hash(row)
-        { id: row[0], pipeline_run_id: row[1], stage: row[2], status: row[3], started_at: row[4], completed_at: row[5] }
+        FIELDS.zip(row).to_h
       end
       private_class_method :row_to_hash
     end

data/lib/fun_ci/persistence/state_dir.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module FunCi
+  module Persistence
+    # Where fun-ci keeps its database: the user's state directory, which every
+    # process of that user agrees on (acceptance-tests.md, AT-9.1). Only that
+    # user can read it, since what fun-ci keeps can hold a secret masking missed.
+    module StateDir
+      MODE = 0o700
+
+      # Creates the directory if need be, and makes it the user's alone.
+      def self.prepare(path)
+        FileUtils.mkdir_p(path)
+        File.chmod(MODE, path)
+      end
+
+      def self.path(env)
+        base = env["XDG_STATE_HOME"].to_s
+        base = File.join(env.fetch("HOME"), ".local", "state") if base.empty?
+        File.join(base, "fun-ci")
+      end
+    end
+  end
+end

data/lib/fun_ci/persistence/write_trouble.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "sqlite3"
+require "stringio"
+
+module FunCi
+  module Persistence
+    # What happens when the database won't take a write (acceptance-tests.md,
+    # AT-8.2, AT-8.4): the pipeline carries on without recording it, and the
+    # developer is told once, with what to do about it.
+    class WriteTrouble
+      # SystemCallError: the files kept beside the database, such as raw output.
+      UNWRITABLE = [SQLite3::FullException, SQLite3::ReadOnlyException, SQLite3::IOException,
+                    SQLite3::CantOpenException, SystemCallError].freeze
+      BUSY = "fun-ci: couldn't record this run: the database stayed busy. Your stages still ran. " \
+             "Trigger it again to record it."
+      STILL_RAN = "Check the disk has space. Your stages still ran."
+
+      # Says nothing, for a process whose output nobody reads.
+      def self.silent(db_path) = new(StringIO.new, db_path)
+
+      def initialize(out, db_path)
+        @out = out
+        @db_path = db_path
+        @told = false
+      end
+
+      # The block's value, or nil when the database refused the write.
+      def guard
+        yield
+      rescue SQLite3::BusyException
+        tell(BUSY)
+      rescue *UNWRITABLE
+        tell("fun-ci: can't write to the database at #{@db_path}. #{STILL_RAN}")
+      end
+
+      private
+
+      def tell(message)
+        @out.puts(message) unless @told
+        @told = true
+        nil
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/background_fork.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative "../persistence/pipeline_recorder"
+require_relative "../persistence/pipeline_run"
+
+module FunCi
+  module Pipeline
+    # Runs the slow suite in a forked child. A child must not inherit an open
+    # SQLite connection, so the parent's recorder is closed before the fork and
+    # #launch answers a fresh one for the parent to carry on with. The slot was
+    # shared for the slow suite before the fork; after it, each process drops
+    # the other's share and keeps its own.
+    class BackgroundFork
+      def initialize(recorder, slot)
+        @recorder = recorder
+        @slot = slot
+      end
+
+      def launch(db_path:, pipeline_run_id:, job_id:, executor:)
+        @recorder.close
+        pid = fork { run_in_child(db_path, pipeline_run_id, job_id, executor) }
+        @slot.release
+        Process.detach(pid)
+        reopened = Persistence::DbRecorder.for_background(db_path, pipeline_run_id, trouble: @recorder.trouble)
+        reopened.tolerating { Persistence::PipelineRun.store_pid(reopened.db, pipeline_run_id, pid) }
+        reopened
+      end
+
+      private
+
+      # Records its own pid before the slow suite starts, so a cancel can't
+      # miss it; the parent records it too, for its callers.
+      def run_in_child(db_path, pipeline_run_id, job_id, executor)
+        @slot.release
+        recorder = Persistence::DbRecorder.for_background(db_path, pipeline_run_id)
+        recorder.tolerating { Persistence::PipelineRun.store_pid(recorder.db, pipeline_run_id, Process.pid) }
+        executor.call(recorder, job_id)
+        recorder.close
+      end
+    end
+  end
+end