fun_ci 2.2.0 → 2.4.0

data/lib/fun_ci/jobs/standings.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "time"
 require_relative "folders"
 require_relative "due"
 require_relative "state"
@@ -19,6 +20,9 @@ module FunCi
       def needs_you? = State.needs_you?(state)
       def due_now? = due.now?
       def due_at = due.at
+
+      # When a run waiting its turn (Schedule) is to start; nil for any other.
+      def starts_at = state == "scheduled" ? Time.parse(run[:started_at]) : nil
     end
 
     # Where each of a project's jobs stands at `now`, read once for the

data/lib/fun_ci/jobs/state.rb

@@ -4,12 +4,13 @@ module FunCi
   module Jobs
     # A job's state, read once from its latest run as job_runs keeps it
     # (design.md, Daily and weekly jobs): `due` while it never ran, then
-    # `running`, `passed`, `failed`, `over_budget`, `cancelled`, or `lost`
+    # `scheduled` while it waits its turn to start (Schedule), `running`,
+    # `passed`, `failed`, `over_budget`, `cancelled`, or `lost`
     # when its process died before it said how the run ended. Each surface
     # (the console, the agent commands) says these in its own words.
     module State
       OF_STATUS = { "completed" => "passed", "failed" => "failed", "timed_out" => "over_budget",
-                    "running" => "running", "cancelled" => "cancelled" }.freeze
+                    "running" => "running", "scheduled" => "scheduled", "cancelled" => "cancelled" }.freeze
       NEEDS_YOU = %w[failed over_budget lost].freeze
 
       # run: { status:, completed_at: }, or nil when the job never ran.

data/lib/fun_ci/jobs/wall_clock_wait.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Jobs
+    # Waits seconds by the wall clock, a minute at a time, for a job's turn
+    # (Schedule), which can be hours away: sleep's own count stops while the
+    # machine sleeps, and a laptop asleep for part of the wait would start the
+    # job late. It sleeps no more minutes than the wait holds and one, so a
+    # clock that never moves can't keep it waiting for ever.
+    class WallClockWait
+      MINUTE = 60
+
+      def initialize(clock: -> { Time.now }, sleep: Kernel.method(:sleep))
+        @clock = clock
+        @sleep = sleep
+      end
+
+      def call(seconds)
+        until_at = @clock.call + seconds
+        ((seconds / MINUTE).ceil + 1).times do
+          left = until_at - @clock.call
+          break unless left.positive?
+
+          @sleep.call([left, MINUTE].min)
+        end
+      end
+    end
+  end
+end

data/lib/fun_ci/persistence/active_jobs.rb

@@ -2,50 +2,47 @@
 
 require "time"
 require_relative "active_runs"
-require "json"
-require_relative "../evidence/document"
+require_relative "died"
 
 module FunCi
   module Persistence
     # Daily and weekly jobs' runs still running, of every project, as the
     # console sees them: to cancel one, and to notice one whose process died.
     module ActiveJobs
+      # A run is still going while it runs, or waits its turn to (Jobs::Schedule).
+      ACTIVE = "status IN ('running', 'scheduled')"
+
       # The job run with this id as an ActiveRun, when it still runs; none otherwise.
       def self.with_id(db, id)
         db.execute("SELECT id, commit_hash, pid, group_pid, lock_file FROM job_runs " \
-                   "WHERE id = ? AND status = 'running'", [id]).map do |(run_id, sha, pid, group_pid, lock_file)|
+                   "WHERE id = ? AND #{ACTIVE}", [id]).map do |(run_id, sha, pid, group_pid, lock_file)|
           ActiveRun.new(id: run_id, commit_hash: sha, processes: [pid].compact, stage_groups: [group_pid].compact,
                         slot_lock: lock_file)
         end
       end
 
       # [id, lock file] of each job run still running.
-      def self.running(db) = db.execute("SELECT id, lock_file FROM job_runs WHERE status = 'running'")
-
-      # What a run whose process died keeps: that it has no result, and why
-      # that may be. It keeps no end, since when it was found dead says
-      # nothing of how long it ran.
-      DIED_FACT = { name: "ended", extractor: "fun-ci",
-                    value: "its process stopped before it said how the run ended (the machine slept or " \
-                           "restarted, or something killed it), so this run has no result" }.freeze
-      DIED = Evidence::Document.new(chosen: [], failures: [], excerpts: [], problems: [], facts: [DIED_FACT])
+      def self.running(db) = db.execute("SELECT id, lock_file FROM job_runs WHERE #{ACTIVE}")
 
-      def self.cancelled(db, id) = ended(db, id, "cancelled")
+      def self.cancelled(db, id) = ended(db, id, "cancelled", ACTIVE)
 
-      # A run whose process is gone without recording how it ended: failed, with no end.
+      # A run whose process is gone without recording how it ended: failed,
+      # with no end; one that was still waiting its turn never ran, so it is
+      # cancelled, and the job is due at the next commit.
       def self.died(db, id)
+        ended(db, id, "cancelled", "status = 'scheduled'")
         db.execute("UPDATE job_runs SET status = 'failed', evidence = ? WHERE id = ? AND status = 'running'",
-                   [JSON.generate(DIED.to_h), id])
+                   [Died.evidence("run"), id])
       end
 
       # The ids of a project's job's runs still running.
       def self.running_of(db, project, name)
-        db.execute("SELECT id FROM job_runs WHERE project_path = ? AND job = ? AND status = 'running'",
+        db.execute("SELECT id FROM job_runs WHERE project_path = ? AND job = ? AND #{ACTIVE}",
                    [project, name]).flatten
       end
 
-      def self.ended(db, id, status)
-        db.execute("UPDATE job_runs SET status = ?, completed_at = ? WHERE id = ? AND status = 'running'",
+      def self.ended(db, id, status, was)
+        db.execute("UPDATE job_runs SET status = ?, completed_at = ? WHERE id = ? AND #{was}",
                    [status, Time.now.utc.iso8601(3), id])
       end
       private_class_method :ended

data/lib/fun_ci/persistence/active_runs.rb

@@ -3,6 +3,7 @@
 require "time"
 require_relative "pipeline_run"
 require_relative "stage_job"
+require_relative "died"
 
 module FunCi
   module Persistence
@@ -39,10 +40,11 @@ module FunCi
                    "AND pipeline_runs.pid IS NOT NULL")
       end
 
-      # Records the slow stage failed, unless its result was recorded meanwhile.
+      # Records the slow stage failed, and that its process died, unless its
+      # result was recorded meanwhile.
       def self.slow_suite_died(db, job_id)
-        db.execute("UPDATE stage_jobs SET status = 'failed', completed_at = ? WHERE id = ? AND status = 'running'",
-                   [Time.now.utc.iso8601(3), job_id])
+        db.execute("UPDATE stage_jobs SET status = 'failed', completed_at = ?, evidence = ? " \
+                   "WHERE id = ? AND status = 'running'", [Time.now.utc.iso8601(3), Died.evidence("stage"), job_id])
       end
 
       def self.cancelled(db, run)

data/lib/fun_ci/persistence/died.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "../evidence/document"
+
+module FunCi
+  module Persistence
+    # The evidence kept of a stage or job run whose process died: that it has
+    # no result, and why that may be, so it reads apart from a failure of the
+    # code. It keeps no end, since when it was found dead says nothing of how
+    # long it ran.
+    module Died
+      def self.evidence(what)
+        fact = { name: "ended", extractor: "fun-ci",
+                 value: "its process stopped before it said how the #{what} ended (the machine slept or " \
+                        "restarted, or something killed it), so this #{what} has no result" }
+        JSON.generate(Evidence::Document.new(chosen: [], failures: [], excerpts: [], problems: [], facts: [fact]).to_h)
+      end
+    end
+  end
+end

data/lib/fun_ci/persistence/job_recorder.rb

@@ -35,6 +35,7 @@ module FunCi
 
       # No stage shares a job's worktree.
       def alongside(_id) = []
+      def alongside_elsewhere(_id) = []
 
       private
 

data/lib/fun_ci/persistence/job_runs.rb

@@ -33,6 +33,13 @@ module FunCi
         end
       end
 
+      # Records the claimed run `id` waiting its turn to start at `at` (Jobs::Schedule).
+      def wait_until(id, at) = update(id, "scheduled", at, "status = 'running'")
+
+      # Records the run `id` running from `now`, its turn come, and answers how
+      # many runs that was: none when it was cancelled while it waited.
+      def start_turn(id, now) = update(id, "running", now, "status = 'scheduled'")
+
       # The project's newest job runs, newest first.
       def recent(limit:) = query("WHERE project_path = ? ORDER BY id DESC LIMIT ?", @project, limit)
 
@@ -51,6 +58,11 @@ module FunCi
         @db.last_insert_row_id
       end
 
+      def update(id, status, at, was)
+        @db.execute("UPDATE job_runs SET status = ?, started_at = ? WHERE id = ? AND #{was}", [status, stamp(at), id])
+        @db.changes
+      end
+
       def forget_beyond(name)
         ids = @db.execute("SELECT id FROM job_runs WHERE project_path = ? AND job = ? ORDER BY id DESC " \
                           "LIMIT -1 OFFSET ?", [@project, name, KEPT]).flatten

data/lib/fun_ci/persistence/null_recorder.rb

@@ -19,6 +19,7 @@ module FunCi
       def trunk_heads(**) = {}
       def trunk_fetches = TrunkRecording::NoFetches.new
       def alongside(_job_id) = []
+      def alongside_elsewhere(_job_id) = []
       def stage_process(_job_id, _pid); end
       def slot_taken(_lock_file); end
       def foreground_done; end

data/lib/fun_ci/persistence/pipeline_recorder.rb

@@ -97,6 +97,7 @@ module FunCi
 
       # The stages that shared the slot with this one (architecture.md, "Evidence of a failed stage").
       def alongside(job_id) = tolerating { StageJob.alongside(@db, job_id) } || []
+      def alongside_elsewhere(job_id) = tolerating { StageJob.alongside_elsewhere(@db, job_id) } || []
 
       def keep_exit(job_id, exit_status, signal)
         tolerating { StageJob.keep_exit(@db, job_id, exit_status, signal) }

data/lib/fun_ci/persistence/pipeline_run.rb

@@ -5,7 +5,8 @@ require "time"
 module FunCi
   module Persistence
     module PipelineRun
-      COLUMNS = %i[id commit_hash branch status pid project_path created_at updated_at trunk_started_at].freeze
+      COLUMNS = %i[id commit_hash branch status pid project_path created_at updated_at trunk_started_at
+                   trigger_pid].freeze
       SELECT = "SELECT #{COLUMNS.join(", ")} FROM pipeline_runs".freeze
 
       def self.create(db, commit_hash:, branch:, project_path: nil)
@@ -55,9 +56,8 @@ module FunCi
         db.execute("UPDATE pipeline_runs SET trunk_started_at = ? WHERE id = ?", [at.utc.iso8601, id])
       end
 
-      def self.update_status(db, id, new_status)
-        now = Time.now.utc.iso8601
-        db.execute("UPDATE pipeline_runs SET status = ?, updated_at = ? WHERE id = ?", [new_status, now, id])
+      def self.update_status(db, id, new_status, at: Time.now)
+        db.execute("UPDATE pipeline_runs SET status = ?, updated_at = ? WHERE id = ?", [new_status, at.utc.iso8601, id])
       end
 
       def self.recent(db, limit: 20)
@@ -68,11 +68,15 @@ module FunCi
       NEEDING = "CASE status WHEN 'failed' THEN 0 WHEN 'timed_out' THEN 1 WHEN 'running' THEN 2 " \
                 "WHEN 'scheduled' THEN 3 WHEN 'completed' THEN 4 ELSE 5 END"
 
-      # The newest run of each of `limit` branches: those that most need you,
-      # then those run most recently, in that order.
-      def self.branch_heads(db, limit:)
-        query(db, "WHERE id IN (SELECT MAX(id) FROM pipeline_runs GROUP BY project_path, branch) " \
-                  "ORDER BY #{NEEDING}, id DESC LIMIT ?", limit)
+      # Each branch's newest run.
+      HEADS = "SELECT MAX(id) FROM pipeline_runs GROUP BY project_path, branch"
+
+      # The newest run of each of `limit` branches, of the `among` whose newest
+      # run changed most recently: those that most need you, then those run
+      # most recently, in that order.
+      def self.branch_heads(db, limit:, among:)
+        query(db, "WHERE id IN (SELECT id FROM pipeline_runs WHERE id IN (#{HEADS}) " \
+                  "ORDER BY updated_at DESC, id DESC LIMIT ?) ORDER BY #{NEEDING}, id DESC LIMIT ?", among, limit)
       end
 
       # A project's branch's runs, newest first.

data/lib/fun_ci/persistence/stage_job.rb

@@ -51,6 +51,18 @@ module FunCi
                    [id]).flatten
       end
 
+      # Stages of the project's other runs running at any moment between this
+      # stage's start and now, as "<short commit>'s <stage>".
+      def self.alongside_elsewhere(db, id)
+        db.execute("SELECT runs.commit_hash, others.stage FROM stage_jobs AS others " \
+                   "JOIN pipeline_runs AS runs ON runs.id = others.pipeline_run_id " \
+                   "JOIN stage_jobs AS me ON me.id = ? JOIN pipeline_runs AS mine ON mine.id = me.pipeline_run_id " \
+                   "WHERE runs.project_path IS mine.project_path AND runs.id != mine.id " \
+                   "AND others.started_at IS NOT NULL " \
+                   "AND (others.completed_at IS NULL OR others.completed_at > me.started_at) ORDER BY others.id",
+                   [id]).map { |sha, stage| "#{sha[0, 7]}'s #{stage}" }
+      end
+
       def self.ids(db) = db.execute("SELECT id FROM stage_jobs").flatten
 
       def self.find(db, id)

data/lib/fun_ci/pipeline/budgets.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Pipeline
+    # Each stage's time budget in seconds (design.md); a worktree just made
+    # gives lint and build the slow suite's, once (SlotRun).
+    DEFAULT_BUDGETS = { "lint" => 30, "build" => 30, "fast" => 10, "slow" => 300 }.freeze
+  end
+end

data/lib/fun_ci/pipeline/group_kill.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Pipeline
+    # Kills a process group and what left it (ProcessTree): TERM, then KILL.
+    # A member forking as the KILL arrives leaves a child in the group that
+    # the signal missed, so the group is killed again while any member lives,
+    # a bounded number of times, since a mutant may stop it ever going.
+    module GroupKill
+      ROUNDS = 10
+
+      # escaped: further targets, -pgid for a group, pid for a process.
+      def self.kill(pid, escaped, signal: Process.method(:kill))
+        %w[TERM KILL].product([-pid, *escaped]).each { |name, target| send_to(signal, name, target) }
+        ROUNDS.times do
+          return unless lives?(pid, signal)
+
+          send_to(signal, "KILL", -pid)
+        end
+      end
+
+      # On macOS a group left with only zombies answers EPERM, not ESRCH.
+      def self.lives?(pid, signal)
+        signal.call(0, -pid)
+        true
+      rescue Errno::ESRCH, Errno::EPERM
+        false
+      end
+
+      def self.send_to(signal, name, target)
+        signal.call(name, target)
+      rescue Errno::ESRCH, Errno::EPERM
+        # Gone already.
+      end
+      private_class_method :lives?, :send_to
+    end
+  end
+end

data/lib/fun_ci/pipeline/pipeline_forker.rb

@@ -25,13 +25,15 @@ module FunCi
       # fork. The project's due jobs start beside the run, each in a process
       # of its own (design.md, Daily and weekly jobs), once the run has
       # started, and trouble starting them never stops it. jobs: what starts
-      # them, given the commit and the database's path.
+      # them, given the commit (by its full SHA, as the run keeps it) and the
+      # database's path.
       def self.fork_pipeline(commit_hash:, branch:, db_path:, jobs: method(:start_due_jobs))
         return false unless Setup::ProjectConfig.new(Dir.pwd).validate.empty?
 
         notice = first_fetch(db_path)
-        Process.detach(fork { run_in_child(commit_hash: commit_hash, branch: branch, db_path: db_path) })
-        Forked.new(notice: notice, jobs: trouble { jobs.call(Commit.new(sha: commit_hash, branch: branch), db_path) })
+        commit = Commit.new(sha: Seams.full_sha(commit_hash) || commit_hash, branch: branch)
+        Process.detach(fork { run_in_child(commit_hash: commit.sha, branch: branch, db_path: db_path) })
+        Forked.new(notice: notice, jobs: trouble { jobs.call(commit, db_path) })
       end
 
       # What went wrong starting the jobs, or nil.
@@ -50,21 +52,24 @@ module FunCi
         db&.close
       end
 
-      # Which jobs are due is read before any fork, which must not inherit the connection.
-      def self.start_due_jobs(commit, db_path)
-        due_jobs(db_path).each { |job| Jobs::JobFork.start(job, commit, project: Dir.pwd, db_path: db_path) }
+      # Which jobs are due, and when each one's turn comes, is read before any
+      # fork, which must not inherit the connection. fork: what starts each.
+      def self.start_due_jobs(commit, db_path, fork: Jobs::JobFork.new(project: Dir.pwd, db_path: db_path))
+        due_jobs(db_path).each { |job, at| fork.start(job, commit, at: at) }
       end
 
       def self.due_jobs(db_path)
         db = Persistence::Database.connection(db_path)
-        Jobs::DueJobs.new(Dir.pwd, db, now: Time.now).list
+        Jobs::DueJobs.new(Dir.pwd, db, now: Time.now).starts
       ensure
         db&.close
       end
 
       # Trigger requires this file, through TriggerCommand, so it is loaded
-      # here, where it is used, rather than above.
+      # here, where it is used, rather than above. The run leads a session of
+      # its own, so that ending the shell that committed doesn't end it.
       def self.run_in_child(commit_hash:, branch:, db_path:)
+        Process.setsid
         require_relative "trigger"
         recorder = Persistence::DbRecorder.new(Persistence::Database.connection(db_path))
         trigger = trigger(commit_hash, branch, recorder)
@@ -78,7 +83,7 @@ module FunCi
         Trigger.new(project: Dir.pwd, commit: Commit.new(sha: commit_hash, branch: branch),
                     io: Io.new(stdout: File.open(File::NULL, "w")), seams: Seams.new(recorder: recorder))
       end
-      private_class_method :trigger, :first_fetch, :start_due_jobs, :due_jobs, :trouble
+      private_class_method :trigger, :first_fetch, :due_jobs, :trouble
     end
   end
 end

data/lib/fun_ci/pipeline/priorities.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Pipeline
+    # What a daily or weekly job and the slow suite are started under, so
+    # they give way to the stages a push waits for: each a prefix of the
+    # command a stage's shell execs, or "" for none. On macOS nice barely
+    # moves the scheduler, and a QoS clamp of utility lets a job have every
+    # idle core yet yield to the stages; the slow suite is left unclamped
+    # there, since clamped beside clamped jobs it crawls. Elsewhere nice does it.
+    Priorities = Data.define(:job, :slow)
+
+    class Priorities
+      MACOS = new(job: "taskpolicy -c utility ", slow: "")
+      OTHER = new(job: "nice -n 19 ", slow: "nice -n 10 ")
+      # fun-ci's own priority, for both.
+      NONE = new(job: "", slow: "")
+
+      # platform: as RUBY_PLATFORM names it.
+      def self.for(platform) = platform.include?("darwin") ? MACOS : OTHER
+    end
+  end
+end

data/lib/fun_ci/pipeline/process_runner.rb

@@ -3,6 +3,9 @@
 require_relative "gate"
 require_relative "git_environment"
 require_relative "output_window"
+require_relative "process_tree"
+require_relative "group_kill"
+require_relative "../evidence/process_table"
 
 module FunCi
   module Pipeline
@@ -24,13 +27,15 @@ module FunCi
 
       # Where the command runs, what it finds in its environment besides
       # fun-ci's own, the window what it prints is written to, what to do
-      # with its pid when it runs over budget, before it is killed, and the
-      # open files it holds as long as it runs, such as a lock.
-      Launch = Data.define(:chdir, :env, :output, :before_kill, :held)
+      # with its pid when it runs over budget, before it is killed, the
+      # open files it holds as long as it runs, such as a lock, and what it
+      # is run under (a Priorities prefix, "" for fun-ci's own priority).
+      Launch = Data.define(:chdir, :env, :output, :before_kill, :held, :priority)
 
       class Launch
         def initialize(**given)
-          super(chdir: Dir.pwd, env: {}, output: OutputWindow.in_memory, before_kill: nil, held: [], **given)
+          super(chdir: Dir.pwd, env: {}, output: OutputWindow.in_memory, before_kill: nil, held: [], priority: "",
+                **given)
         end
       end
 
@@ -77,7 +82,7 @@ module FunCi
       end
 
       def spawn_gated(cmd, writer, gate, launch)
-        Process.spawn(GitEnvironment::CLEAN.merge(launch.env), format(GATED, cmd),
+        Process.spawn(GitEnvironment::CLEAN.merge(launch.env), format(GATED, "#{launch.priority}#{cmd}"),
                       out: writer, err: writer, 9 => gate.child_end, pgroup: true, chdir: launch.chdir, **held(launch))
       end
 
@@ -110,8 +115,10 @@ module FunCi
         [output, status, false]
       end
 
+      # With the group go what it started that left it (ProcessTree), found
+      # before the kill orphans them (GroupKill).
       def kill_process_group(pid)
-        %w[TERM KILL].each { |signal| ignoring_errors { Process.kill(signal, -pid) } }
+        GroupKill.kill(pid, ProcessTree.escaped([pid], Evidence::ProcessTable.now))
         ignoring_errors { Process.waitpid(pid) }
       end
 

data/lib/fun_ci/pipeline/process_tree.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Pipeline
+    # What to kill besides a stage's process groups: each process descended
+    # from one of their members, by parent id, that has left for a group or
+    # session of its own, as Node's detached children do. The rows must be
+    # read before the kill, since a killed parent's children move to init.
+    # A double-forked daemon has moved to init already and is not reached.
+    module ProcessTree
+      # Signal targets: -pid for an escapee leading its own group, which
+      # takes that group with it, and the pid alone of one in any other group.
+      def self.escaped(groups, rows)
+        escapees = outside(groups, descendants(in_groups(groups, rows), rows))
+        leaders = escapees.select { |row| row.pgid == row.pid }.map(&:pid)
+        leaders.map(&:-@) + escapees.reject { |row| leaders.include?(row.pgid) }.map(&:pid)
+      end
+
+      def self.in_groups(groups, rows) = rows.select { |row| groups.include?(row.pgid) }
+      def self.outside(groups, rows) = rows.reject { |row| groups.include?(row.pgid) }
+
+      # Bounded by the table's size, so a table whose parent ids form a cycle still ends.
+      def self.descendants(found, rows)
+        rows.size.times do
+          more = children(found.map(&:pid), rows) - found
+          return found if more.empty?
+
+          found += more
+        end
+        found
+      end
+
+      def self.children(parents, rows) = rows.select { |row| parents.include?(row.ppid) }
+      private_class_method :in_groups, :outside, :descendants, :children
+    end
+  end
+end

data/lib/fun_ci/pipeline/progress_reporter.rb

@@ -17,6 +17,11 @@ module FunCi
         @stdout.puts "fun-ci: fast #{result_marker(passed)}"
       end
 
+      def cold_slot(seconds)
+        @stdout.puts "fun-ci: a cold worktree (new, just cleaned, or unused for a day), so lint and build have " \
+                     "#{seconds}s this once."
+      end
+
       def slow_launched
         @stdout.puts "fun-ci: slow (running in background)"
       end

data/lib/fun_ci/pipeline/run_canceller.rb

@@ -5,18 +5,24 @@ require_relative "../persistence/active_jobs"
 require_relative "../persistence/run_status"
 require_relative "../persistence/write_trouble"
 require_relative "slot_lock"
+require_relative "process_tree"
+require_relative "../evidence/process_table"
 
 module FunCi
   module Pipeline
     # Stops every process of a run. The run's own processes go first, so none
     # of them records the run as failed once its stages die; then each stage's
-    # process group. A run whose slot nobody holds any more has died already,
+    # process group, and last what a stage started that left its group
+    # (ProcessTree). A run whose slot nobody holds any more has died already,
     # and the pids it recorded may belong to other processes now, so it is
     # left alone; so is this process.
     class RunCanceller
-      def initialize(killer: Process.method(:kill), slot_held: SlotLock.method(:held?))
+      # process_table: what ps lists now, as Evidence::ProcessTable rows.
+      def initialize(killer: Process.method(:kill), slot_held: SlotLock.method(:held?),
+                     process_table: Evidence::ProcessTable.method(:now))
         @killer = killer
         @slot_held = slot_held
+        @process_table = process_table
       end
 
       # Stops the run and records it, and its unfinished stages, cancelled.
@@ -60,11 +66,17 @@ module FunCi
       def stop(run)
         return unless alive?(run)
 
-        (run.processes - [Process.pid] + run.stage_groups.map(&:-@)).each { |pid| kill(pid) }
+        (run.processes - [Process.pid] + run.stage_groups.map(&:-@) + escaped(run)).each { |pid| kill(pid) }
       end
 
       private
 
+      def escaped(run)
+        return [] if run.stage_groups.empty?
+
+        ProcessTree.escaped(run.stage_groups, @process_table.call)
+      end
+
       # A stage its process started after `run` was read. A run's process
       # records a stage's group before it lets the stage start, so once those
       # processes are dead, looking again finds every stage they started.

data/lib/fun_ci/pipeline/slot.rb

@@ -25,6 +25,17 @@ module FunCi
         @holders -= 1
         @lock.close if @holders.zero? && !@lock.closed?
       end
+
+      # Whether its worktree was just made, so has no caches yet.
+      def cold? = false
+
+      # The same slot, before anyone shares it, in a worktree just made.
+      def as_cold = ColdSlot.new(path, lock, lock_file)
+    end
+
+    # A slot whose worktree was just made: its build starts from nothing.
+    class ColdSlot < Slot
+      def cold? = true
     end
   end
 end

data/lib/fun_ci/pipeline/slot_run.rb

@@ -34,22 +34,36 @@ module FunCi
 
       def recorder = @seams.recorder
       def progress = ProgressReporter.new(stdout: @io.stdout)
-      def stage_runner = StageRunner.new(commit: @commit, stdout: @io.stdout, seams: @seams, dir: @slot.path)
+
+      def stage_runner(seams = @seams)
+        StageRunner.new(commit: @commit, stdout: @io.stdout, seams: seams, dir: @slot.path)
+      end
 
       # This process's part is over; the slow suite may still be running.
       def released(exit_code)
         @slot.release
-        recorder.foreground_done
         exit_code
       end
 
       def phase_one_passed?(config)
-        results = %w[lint build].to_h { |stage| [stage, Thread.new { stage_runner.passes?(config, stage) }] }
+        runner = stage_runner(phase_one_seams)
+        results = %w[lint build].to_h { |stage| [stage, Thread.new { runner.passes?(config, stage) }] }
                                 .transform_values(&:value)
         progress.phase_one_result(results)
         results.values.all?
       end
 
+      # A worktree just made has no caches, so its lint and build are no
+      # measure of the project's: they get the slow suite's budget, this once.
+      def phase_one_seams
+        return @seams unless @slot.cold?
+
+        cold = @seams.budgets["slow"]
+        progress.cold_slot(cold)
+        longer = %w[lint build].to_h { |stage| [stage, [@seams.budgets[stage], cold].max] }
+        @seams.with(time_budgets: @seams.time_budgets.merge(longer))
+      end
+
       def fast_passed?(config)
         passed = stage_runner.passes?(config, "fast")
         progress.fast_result(passed)
@@ -65,7 +79,8 @@ module FunCi
       # Called with the recorder and job to record the slow suite with, in
       # whichever process runs it; the slot is let go once it is recorded.
       def slow_suite(config, slot)
-        execution = StageExecution.new(seams: @seams, dir: slot.path, commit: @commit)
+        execution = StageExecution.new(seams: @seams, dir: slot.path, commit: @commit,
+                                       launching: { priority: @seams.priorities.slow })
         command = config.stage_command("slow", @commit.sha)
         ->(recorder, job_id) { holding(slot) { execution.run("slow", command, recorder, job_id) } }
       end