fun_ci 2.0.2 → 2.1.0

data/lib/fun_ci/evidence/caps.rb

@@ -15,8 +15,13 @@ module FunCi
       # What a line costs in JSON besides its own bytes: quotes and a comma.
       LINE_COST = 3
 
-      def initialize(limits = REAL)
+      # How many bytes of JSON a document makes.
+      JSON_SIZE = ->(document) { JSON.generate(document.to_h).bytesize }
+
+      # size: measures a document, JSON_SIZE but where a test counts the measuring.
+      def initialize(limits = REAL, size: JSON_SIZE)
         @limits = limits
+        @size = size
       end
 
       def apply(document)
@@ -36,7 +41,7 @@ module FunCi
         document
       end
 
-      def over(document) = JSON.generate(document.to_h).bytesize - @limits.bytes
+      def over(document) = @size.call(document) - @limits.bytes
 
       def few_failures(document)
         left_out = document.failures.size - @limits.failures

data/lib/fun_ci/persistence/active_runs.rb

@@ -52,16 +52,17 @@ module FunCi
       end
 
       def self.where(db, clause, *values)
-        db.execute("SELECT id, commit_hash, trigger_pid, pid, slot_lock FROM pipeline_runs " \
+        db.execute("SELECT id, commit_hash, trigger_pid, pid, slot_lock, fetch_pgid FROM pipeline_runs " \
                    "WHERE #{clause} AND #{ACTIVE} ORDER BY id", values).map { |row| active_run(db, row) }
       end
       private_class_method :where
 
-      def self.active_run(db, (id, commit_hash, trigger_pid, pid, slot_lock))
+      # The groups are each running stage's, and the trunk fetch's (architecture.md, Checking against the trunk).
+      def self.active_run(db, (id, commit_hash, trigger_pid, pid, slot_lock, fetch_pgid))
         groups = db.execute("SELECT pid FROM stage_jobs WHERE pipeline_run_id = ? AND #{ACTIVE} AND pid IS NOT NULL",
                             [id]).flatten
-        ActiveRun.new(id: id, commit_hash: commit_hash, processes: [trigger_pid, pid].compact, stage_groups: groups,
-                      slot_lock: slot_lock)
+        ActiveRun.new(id: id, commit_hash: commit_hash, processes: [trigger_pid, pid].compact,
+                      stage_groups: [*groups, *fetch_pgid], slot_lock: slot_lock)
       end
       private_class_method :active_run
     end

data/lib/fun_ci/persistence/database.rb

@@ -28,6 +28,24 @@ module FunCi
         )
       SQL
 
+      # One row per commit and trunk SHA, written once (architecture.md, Checking against the trunk).
+      TRUNK_CHECKS_TABLE = <<~SQL
+        CREATE TABLE IF NOT EXISTS trunk_checks (
+          id INTEGER PRIMARY KEY,
+          project_path TEXT, commit_hash TEXT, trunk_remote TEXT, trunk_branch TEXT, trunk_sha TEXT,
+          trunk_seen_at TEXT, checked_at TEXT,
+          outcome TEXT, ahead INTEGER, behind INTEGER, files TEXT, reason TEXT,
+          UNIQUE (project_path, commit_hash, trunk_sha)
+        )
+      SQL
+
+      # When fun-ci last fetched each project's trunk (architecture.md, Checking against the trunk).
+      TRUNK_FETCHES_TABLE = <<~SQL
+        CREATE TABLE IF NOT EXISTS trunk_fetches (
+          project_path TEXT PRIMARY KEY, claimed_at INTEGER, fetched_at TEXT, failures INTEGER, error TEXT
+        )
+      SQL
+
       # Opening and migrating hold an exclusive lock on a file beside the
       # database, so fun-ci processes starting at once set it up one at a time.
       # SQLite's busy timeout doesn't cover the switch to WAL, and a column can
@@ -44,11 +62,14 @@ module FunCi
                        %w[stage_jobs output_tail TEXT], %w[stage_jobs failures TEXT],
                        %w[pipeline_runs waited_at TEXT], %w[stage_jobs exit_status INTEGER],
                        %w[stage_jobs signal TEXT], %w[stage_jobs budget INTEGER], %w[stage_jobs pruned INTEGER],
-                       %w[stage_jobs evidence TEXT]].freeze
+                       %w[stage_jobs evidence TEXT], %w[pipeline_runs fetch_pgid INTEGER],
+                       %w[pipeline_runs trunk_started_at TEXT]].freeze
 
       def self.migrate!(db)
         with_setup_lock(db.filename("main")) do
-          [PIPELINE_RUNS_TABLE, STAGE_JOBS_TABLE].each { |table| db.execute(table) }
+          [PIPELINE_RUNS_TABLE, STAGE_JOBS_TABLE, TRUNK_CHECKS_TABLE, TRUNK_FETCHES_TABLE].each do |table|
+            db.execute(table)
+          end
           ADDED_COLUMNS.each { |table, column, type| add_column_if_missing(db, table, column, type) }
           db.execute(NAME_DETACHED)
         end

data/lib/fun_ci/persistence/null_recorder.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "trunk_recording"
+
 module FunCi
   module Persistence
     # Records nothing, for a pipeline run with no database.
@@ -10,6 +12,12 @@ module FunCi
       def keep_exit(_job_id, _exit_status, _signal); end
       def keep_evidence(_job_id, _document); end
       def keep_raw(_job_id, _text); end
+      def trunk_checked(_check); end
+      def trunk_fetched(_fetched, _tip); end
+      def trunk_fetch_process(_pgid); end
+      def trunk_check_started; end
+      def trunk_heads(**) = {}
+      def trunk_fetches = TrunkRecording::NoFetches.new
       def alongside(_job_id) = []
       def stage_process(_job_id, _pid); end
       def slot_taken(_lock_file); end

data/lib/fun_ci/persistence/pipeline_recorder.rb

@@ -8,10 +8,13 @@ require "json"
 require_relative "null_recorder"
 require_relative "project_runs"
 require_relative "retention"
+require_relative "trunk_recording"
 
 module FunCi
   module Persistence
     class DbRecorder
+      include TrunkRecording
+
       KEPT_RUNS = Retention::KEPT_RUNS
 
       attr_reader :db, :db_path, :pipeline_run_id, :trouble

data/lib/fun_ci/persistence/pipeline_run.rb

@@ -5,7 +5,7 @@ require "time"
 module FunCi
   module Persistence
     module PipelineRun
-      COLUMNS = %i[id commit_hash branch status pid project_path created_at updated_at].freeze
+      COLUMNS = %i[id commit_hash branch status pid project_path created_at updated_at trunk_started_at].freeze
       SELECT = "SELECT #{COLUMNS.join(", ")} FROM pipeline_runs".freeze
 
       def self.create(db, commit_hash:, branch:, project_path: nil)
@@ -50,6 +50,11 @@ module FunCi
         db.execute("UPDATE pipeline_runs SET waited_at = ? WHERE id = ?", [at, id])
       end
 
+      # When the run began checking its commit against the trunk (architecture.md, Checking against the trunk).
+      def self.trunk_started(db, id, at)
+        db.execute("UPDATE pipeline_runs SET trunk_started_at = ? WHERE id = ?", [at.utc.iso8601, id])
+      end
+
       def self.update_status(db, id, new_status)
         now = Time.now.utc.iso8601
         db.execute("UPDATE pipeline_runs SET status = ?, updated_at = ? WHERE id = ?", [new_status, now, id])
@@ -59,8 +64,24 @@ module FunCi
         query(db, "ORDER BY id DESC LIMIT ?", limit)
       end
 
-      def self.query(db, clause, param)
-        db.execute("#{SELECT} #{clause}", [param]).map { |row| COLUMNS.zip(row).to_h }
+      # How much a branch whose newest run has a status needs you: a failure most.
+      NEEDING = "CASE status WHEN 'failed' THEN 0 WHEN 'timed_out' THEN 1 WHEN 'running' THEN 2 " \
+                "WHEN 'scheduled' THEN 3 WHEN 'completed' THEN 4 ELSE 5 END"
+
+      # The newest run of each of `limit` branches: those that most need you,
+      # then those run most recently, in that order.
+      def self.branch_heads(db, limit:)
+        query(db, "WHERE id IN (SELECT MAX(id) FROM pipeline_runs GROUP BY project_path, branch) " \
+                  "ORDER BY #{NEEDING}, id DESC LIMIT ?", limit)
+      end
+
+      # A project's branch's runs, newest first.
+      def self.of_branch(db, project_path, branch, limit:)
+        query(db, "WHERE project_path IS ? AND branch = ? ORDER BY id DESC LIMIT ?", project_path, branch, limit)
+      end
+
+      def self.query(db, clause, *params)
+        db.execute("#{SELECT} #{clause}", params).map { |row| COLUMNS.zip(row).to_h }
       end
       private_class_method :query
     end

data/lib/fun_ci/persistence/trunk_checks.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require "json"
+require "time"
+require_relative "../trunk/check"
+
+module FunCi
+  module Persistence
+    # A project's checks against its trunk. A check is a fact about a commit
+    # and a trunk SHA, so each pair is written once and never changed.
+    class TrunkChecks
+      TIP = %i[trunk_remote trunk_branch trunk_sha trunk_seen_at].freeze
+      MERGE = %i[outcome ahead behind files reason].freeze
+      COLUMNS = [:commit_hash, *TIP, *MERGE].freeze
+      SELECT = "SELECT #{COLUMNS.join(", ")} FROM trunk_checks".freeze
+
+      def initialize(db, project)
+        @db = db
+        @project = project
+      end
+
+      def record(check, checked_at:)
+        @db.execute("INSERT OR IGNORE INTO trunk_checks (project_path, checked_at, #{COLUMNS.join(", ")}) " \
+                    "VALUES (?, ?, #{(["?"] * COLUMNS.size).join(", ")})",
+                    [@project, checked_at.utc.iso8601, check.commit, *tip_row(check.tip), *merge_row(check.merge)])
+      end
+
+      # A fetch found the trunk still at `sha`: the checks against it are as fresh as that fetch.
+      def seen(sha, at:)
+        @db.execute("UPDATE trunk_checks SET trunk_seen_at = ? WHERE project_path = ? AND trunk_sha = ?",
+                    [at.utc.iso8601, @project, sha])
+      end
+
+      # The commit's check against the trunk SHA fun-ci saw last, or nil.
+      def latest(sha)
+        found = @db.execute("#{SELECT} WHERE project_path = ? AND commit_hash = ? " \
+                            "ORDER BY COALESCE(trunk_seen_at, checked_at) DESC, id DESC LIMIT 1", [@project, sha]).first
+        found && Trunk::Check.new(commit: found.first, tip: tip(found[1, TIP.size]),
+                                  merge: merge(found.last(MERGE.size)))
+      end
+
+      # { commit => trunk SHA } for each branch's newest run, other than `except`,
+      # whose commit has been checked against a trunk tip.
+      def branch_heads(except:)
+        heads = @db.execute("SELECT commit_hash FROM pipeline_runs WHERE id IN (SELECT MAX(id) FROM pipeline_runs " \
+                            "WHERE project_path = ? GROUP BY branch) AND commit_hash != ?", [@project, except]).flatten
+        heads.to_h { |sha| [sha, latest(sha)&.tip&.sha] }.compact
+      end
+
+      private
+
+      def tip_row(tip) = tip ? [tip.remote, tip.branch, tip.sha, tip.seen_at.utc.iso8601] : [nil] * TIP.size
+      def merge_row(merge) = [merge.outcome, merge.ahead, merge.behind, JSON.generate(merge.files), merge.reason]
+
+      def tip(values)
+        remote, branch, sha, seen_at = values
+        return nil unless sha
+
+        Trunk::Tip.new(remote: remote, branch: branch, sha: sha, seen_at: Time.parse(seen_at))
+      end
+
+      def merge(values)
+        outcome, ahead, behind, files, reason = values
+        Trunk::Merge.new(outcome: outcome, ahead: ahead, behind: behind, files: JSON.parse(files), reason: reason)
+      end
+    end
+  end
+end

data/lib/fun_ci/persistence/trunk_fetches.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "time"
+require_relative "../trunk/fetch"
+require_relative "../trunk/check"
+
+module FunCi
+  module Persistence
+    # When fun-ci last fetched each project's trunk, and how that went
+    # (architecture.md, Checking against the trunk). A fetch is claimed
+    # in one statement, so of two processes claiming at once only one
+    # fetches; after failures the interval doubles, up to an hour.
+    class TrunkFetches
+      LONGEST = 3600
+      Last = Trunk::LastFetch
+
+      def initialize(db, project)
+        @db = db
+        @project = project
+      end
+
+      # `now` when this process may fetch now, and the interval starts again; nil otherwise.
+      def claim(now:, interval:)
+        ensure_row
+        @db.execute("UPDATE trunk_fetches SET claimed_at = ? WHERE project_path = ? AND (claimed_at IS NULL " \
+                    "OR claimed_at + MIN(? * (1 << failures), ?) <= ?)",
+                    [now.to_i, @project, interval, LONGEST, now.to_i])
+        now if @db.changes == 1
+      end
+
+      def finished(fetched, at:)
+        ensure_row
+        return failed(fetched.error) if fetched.error
+
+        @db.execute("UPDATE trunk_fetches SET fetched_at = ?, failures = 0, error = NULL WHERE project_path = ?",
+                    [at.utc.iso8601, @project])
+      end
+
+      # The last fetch tried, or nil when there has been none.
+      def last
+        row = @db.execute("SELECT fetched_at, error FROM trunk_fetches WHERE project_path = ?", [@project]).first
+        row && Last.new(fetched_at: row[0] && Time.parse(row[0]), error: row[1])
+      end
+
+      private
+
+      def ensure_row
+        @db.execute("INSERT OR IGNORE INTO trunk_fetches (project_path, failures) VALUES (?, 0)", [@project])
+      end
+
+      def failed(error)
+        @db.execute("UPDATE trunk_fetches SET failures = failures + 1, error = ? WHERE project_path = ?",
+                    [error, @project])
+      end
+    end
+  end
+end

data/lib/fun_ci/persistence/trunk_recording.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require_relative "pipeline_run"
+require_relative "trunk_checks"
+require_relative "trunk_fetches"
+
+module FunCi
+  module Persistence
+    # What a run records about the trunk (architecture.md, Checking against the trunk), under the
+    # run's project. Expects @db and @pipeline_run_id, and #tolerating.
+    module TrunkRecording
+      # A fetch that never happens, for a run whose project can't be read.
+      class NoFetches
+        def claim(now:, **) = now
+        def last = nil
+      end
+
+      def trunk_check_started
+        tolerating { PipelineRun.trunk_started(@db, @pipeline_run_id, Time.now) }
+      end
+
+      def trunk_fetches = tolerating { TrunkFetches.new(@db, project) } || NoFetches.new
+
+      # The fetch's process group, which cancelling the run kills.
+      def trunk_fetch_process(pgid)
+        tolerating { @db.execute("UPDATE pipeline_runs SET fetch_pgid = ? WHERE id = ?", [pgid, @pipeline_run_id]) }
+      end
+
+      # How the fetch went; a tip it fetched counts as seen now in every check against it.
+      def trunk_fetched(fetched, tip)
+        tolerating do
+          TrunkFetches.new(@db, project).finished(fetched, at: Time.now)
+          TrunkChecks.new(@db, project).seen(tip.sha, at: Time.now) if tip && !fetched.error
+        end
+      end
+
+      # { commit => trunk SHA } for the other branches' newest checked runs.
+      def trunk_heads(except:) = tolerating { TrunkChecks.new(@db, project).branch_heads(except: except) } || {}
+
+      def trunk_checked(check)
+        tolerating { TrunkChecks.new(@db, project).record(check, checked_at: Time.now) }
+      end
+
+      private
+
+      def project = PipelineRun.find(@db, @pipeline_run_id)[:project_path]
+    end
+  end
+end

data/lib/fun_ci/pipeline/pipeline_forker.rb

@@ -4,18 +4,33 @@ require_relative "../persistence/database"
 require_relative "../persistence/pipeline_recorder"
 require_relative "trigger_params"
 require_relative "../setup/project_config"
+require_relative "../trunk/first_fetch"
 
 module FunCi
   module Pipeline
     class PipelineForker
-      # Answers whether it started a run: none in a project not set up for fun-ci.
+      # A run started, and what to say before the project's first fetch of its trunk, if this is it.
+      Forked = Data.define(:notice)
+
+      # Answers a Forked, or false in a project not set up for fun-ci. What the
+      # run prints goes nowhere, so the notice is worked out here, before the fork.
       def self.fork_pipeline(commit_hash:, branch:, db_path:)
         return false unless Setup::ProjectConfig.new(Dir.pwd).validate.empty?
 
+        notice = first_fetch(db_path)
         pid = fork do
           run_in_child(commit_hash: commit_hash, branch: branch, db_path: db_path)
         end
         Process.detach(pid)
+        Forked.new(notice: notice)
+      end
+
+      # The connection closes before the fork, which must not inherit it.
+      def self.first_fetch(db_path)
+        db = Persistence::Database.connection(db_path)
+        Trunk::FirstFetch.notice(Dir.pwd, db)
+      ensure
+        db&.close
       end
 
       # Trigger requires this file, through TriggerCommand, so it is loaded
@@ -34,7 +49,7 @@ module FunCi
         Trigger.new(project: Dir.pwd, commit: Commit.new(sha: commit_hash, branch: branch),
                     io: Io.new(stdout: File.open(File::NULL, "w")), seams: Seams.new(recorder: recorder))
       end
-      private_class_method :trigger
+      private_class_method :trigger, :first_fetch
     end
   end
 end

data/lib/fun_ci/pipeline/process_runner.rb

@@ -33,14 +33,28 @@ module FunCi
         end
       end
 
+      # A command started and let go, and the pipe that carries what it prints.
+      Started = Data.define(:pid, :reader)
+
       def run_process_with_timeout(cmd, budget, launch: Launch.new, timer: BUDGET, &)
+        await_process(start_process(cmd, launch, &), budget, launch: launch, timer: timer)
+      end
+
+      # Starts the command and yields its pid before letting it run.
+      def start_process(cmd, launch = Launch.new, &)
         reader, writer = IO.pipe
-        pid = start(cmd, writer, launch, &)
+        Started.new(pid: start(cmd, writer, launch, &), reader: reader)
+      end
+
+      # [what it printed, its status (nil when killed), whether it ran over budget].
+      def await_process(started, budget, launch: Launch.new, timer: BUDGET)
         printed = launch.output
-        reading = Thread.new { read_until_closed(reader, printed) }
-        timer.call(reading, budget) ? process_finished(pid, text(reading.value)) : over_budget(pid, reading, launch)
+        reading = Thread.new { read_until_closed(started.reader, printed) }
+        return process_finished(started.pid, text(reading.value)) if timer.call(reading, budget)
+
+        over_budget(started.pid, reading, launch)
       ensure
-        ignoring_errors { reader&.close }
+        ignoring_errors { started.reader.close }
       end
 
       private

data/lib/fun_ci/pipeline/trigger.rb

@@ -7,6 +7,8 @@ require_relative "trigger_command"
 require_relative "slot_run"
 require_relative "stale_pipeline_canceller"
 require_relative "workspaces"
+require_relative "trunk_run"
+require_relative "../trunk/checker"
 
 module FunCi
   module Pipeline
@@ -30,7 +32,7 @@ module FunCi
         return unknown_commit unless known_commit?
 
         start_run
-        run_in(workspace.acquire(@commit.sha))
+        checking_the_trunk { run_in(workspace.acquire(@commit.sha)) }
       end
 
       # The background launcher swaps in a fresh recorder after forking, so
@@ -54,6 +56,13 @@ module FunCi
         committed.folder_exists? ? committed : Setup::ProjectConfig.new(@project)
       end
 
+      # The check is recorded by the recorder the run holds once its stages are done.
+      def checking_the_trunk
+        trunk = TrunkRun.start(@seams.trunk || Trunk::Checker.for(@project), @commit.sha, recorder)
+        @io.stdout.puts(trunk.notice) if trunk.notice
+        yield.tap { trunk.finish(recorder) }
+      end
+
       def handle_config_errors(config)
         config.validate.each { |e| @io.stdout.puts "fun-ci: #{e}" }
         unless config.folder_exists?

data/lib/fun_ci/pipeline/trigger_command.rb

@@ -6,10 +6,10 @@ require_relative "pipeline_forker"
 module FunCi
   module Pipeline
     # `fun-ci trigger [--background] <commit-hash> <branch>`. --no-validate is
-    # the 1.x name for --background, kept for one release.
+    # the 1.x name for --background, kept until 3.0.
     class TriggerCommand
       BACKGROUND = %w[--background --no-validate].freeze
-      DEPRECATED = "fun-ci: --no-validate is now --background; the old name goes in 2.1."
+      DEPRECATED = "fun-ci: --no-validate is now --background; the old name goes in 3.0."
 
       def initialize(io:, recorder:, pipeline_forker:, project: Dir.pwd)
         @io = io
@@ -47,12 +47,16 @@ module FunCi
       end
 
       # The forker answers whether it started a run; one that did says how an
-      # agent gets its verdict (acceptance-tests.md, AT-9.14).
+      # agent gets its verdict (acceptance-tests.md, AT-9.14), and what the
+      # forker says of a first fetch of the trunk.
       def fork_pipeline(commit)
         db_path = @recorder.db_path
         @recorder.close
-        started = @pipeline_forker.call(commit_hash: commit.sha, branch: commit.branch, db_path: db_path)
-        say_how_to_wait(commit.sha[0, 7]) if started
+        forked = @pipeline_forker.call(commit_hash: commit.sha, branch: commit.branch, db_path: db_path)
+        return 0 unless forked
+
+        say_how_to_wait(commit.sha[0, 7])
+        @io.stdout.puts(forked.notice) if forked.notice
         0
       end
 

data/lib/fun_ci/pipeline/trigger_params.rb

@@ -31,9 +31,11 @@ module FunCi
     # which masking and injected command runners see; `clock` answers seconds
     # for the evidence budget; `extractor_runner` makes what runs a project's
     # own extractors (`run:` entries), given dir:, env: and scratch:;
-    # `process_table` answers what ps lists, for a stage over budget.
+    # `process_table` answers what ps lists, for a stage over budget; `trunk`
+    # checks a commit against the trunk (#check(sha) answers a Trunk::Check,
+    # or nil for no check), nil for the project's own.
     Seams = Data.define(:command_runner, :time_budgets, :commit_validator, :recorder, :background_launcher, :workspace,
-                        :stage_dir, :environment, :clock, :extractor_runner, :process_table)
+                        :stage_dir, :environment, :clock, :extractor_runner, :process_table, :trunk)
 
     # Reopened rather than given as a block to Data.define, so tools that read
     # the source (mutineer) see these as Seams' methods.
@@ -43,7 +45,7 @@ module FunCi
           workspace: nil, commit_validator: method(:commit_exists?), stage_dir: StageDir.method(:create),
           environment: ENV.to_h, clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) },
           extractor_runner: Evidence::CommandRunner.method(:new),
-          process_table: Evidence::ProcessTable.method(:now) }
+          process_table: Evidence::ProcessTable.method(:now), trunk: nil }
       end
 
       def self.commit_exists?(sha) = Open3.capture2e(GitEnvironment::CLEAN, "git", "cat-file", "-t", sha).last.success?

data/lib/fun_ci/pipeline/trunk_run.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require_relative "../trunk/check"
+require_relative "../trunk/checker"
+
+module FunCi
+  module Pipeline
+    # A run's check against the trunk (architecture.md, Checking against the
+    # trunk). It begins before the stages, where the database may be used
+    # (claiming a fetch, recording its process, reading the other branches'
+    # heads); finishes in a thread beside them, which touches no database,
+    # since the slow suite's fork closes the recorder, checking the other
+    # heads too when the trunk has moved; and is recorded once the stages are
+    # done, through whichever recorder the run holds then. Whatever goes wrong
+    # is recorded as an unknown check rather than raised into the run.
+    class TrunkRun
+      Result = Trunk::Checker::Result
+      # What the thread found: the run's own result, and the other heads' checks.
+      Found = Data.define(:result, :rechecks)
+
+      # For a project that checks no trunk.
+      class Skipped
+        def finish(_recorder) = nil
+        def notice = nil
+      end
+
+      # trunk: begins a check (#start(sha, fetches), nil for none), finishes it
+      # (#finish(pending)) and checks other heads against a tip (#recheck(heads, tip)).
+      def self.start(trunk, sha, recorder)
+        pending = trunk.start(sha, recorder.trunk_fetches) { |pid| recorder.trunk_fetch_process(pid) }
+        return Skipped.new unless pending
+
+        recorder.trunk_check_started
+        heads = recorder.trunk_heads(except: sha)
+        new(Thread.new { found(trunk, pending, sha, heads) }, trunk.notice(pending))
+      rescue StandardError => e
+        new(Thread.new { Found.new(result: failed(sha, e), rechecks: []) })
+      end
+
+      def self.found(trunk, pending, sha, heads)
+        rechecked(trunk, heads, Found.new(result: finished(trunk, pending, sha), rechecks: []))
+      end
+
+      def self.finished(trunk, pending, sha)
+        trunk.finish(pending)
+      rescue StandardError => e
+        failed(sha, e)
+      end
+
+      # A recheck that goes wrong costs the other heads a fresher check, never the run its own.
+      def self.rechecked(trunk, heads, found)
+        tip = found.result.check&.tip
+        tip ? found.with(rechecks: trunk.recheck(heads, tip)) : found
+      rescue StandardError
+        found
+      end
+
+      def self.failed(sha, error)
+        check = Trunk::Check.new(commit: sha, tip: nil,
+                                 merge: Trunk::Merge.unknown("the check failed: #{error.message}"))
+        Result.new(check: check, fetched: nil)
+      end
+      private_class_method :found, :finished, :rechecked, :failed
+
+      attr_reader :notice
+
+      # notice: what to tell the developer of a first fetch, or nil.
+      def initialize(thread, notice = nil)
+        @thread = thread
+        @notice = notice
+      end
+
+      def finish(recorder)
+        found = @thread.value
+        result = found.result
+        recorder.trunk_fetched(result.fetched, result.check&.tip) if result.fetched
+        [result.check, *found.rechecks].compact.each { |check| recorder.trunk_checked(check) }
+      end
+    end
+  end
+end

data/lib/fun_ci/setup/agent_instructions.rb

@@ -15,6 +15,9 @@ module FunCi
         `fun-ci wait` command it prints, in the background, and act on its exit
         code: 0 passed, 1 failed, 2 over budget, 3 undecided, 4 superseded.
         `fun-ci wait --need all` must exit 0 before the work is called done.
+        Before calling work done, run `fun-ci wait --need all --trunk`; 6 means
+        the commit conflicts with the trunk, which has to be integrated first,
+        as it prints.
         `fun-ci runs` lists recent runs, and `fun-ci status` says where one stands.
       MD
 

data/lib/fun_ci/setup/project_config.rb

@@ -38,6 +38,8 @@ module FunCi
       def any_project_presets = preset_candidates.reject(&:because).map { |found| found.preset.name }
 
       def worktree_slots = settings.worktree_slots
+      def trunk = settings.trunk
+      def trunk_fetch = settings.trunk_fetch
 
       def script_path(stage)
         File.join(@fun_ci_dir, "#{stage}.sh")
@@ -49,7 +51,7 @@ module FunCi
 
       private
 
-      def settings = Settings.new(File.join(@fun_ci_dir, "config"))
+      def settings = Settings.at(File.join(@fun_ci_dir, "config"))
 
       def preset_candidates
         evidence = Evidence::Settings.load(File.join(@fun_ci_dir, "config"))