fun_ci 2.2.0 → 2.3.0

data/lib/fun_ci/console/job_message.rb

@@ -14,20 +14,26 @@ module FunCi
 
       def self.from(standing)
         { project: standing.project, name: standing.name, cadence: standing.cadence,
-          status: STATUS.fetch(standing.state, standing.state), **run(standing.run),
+          status: STATUS.fetch(standing.state, standing.state), **run(standing.run, standing.starts_at),
           due_at: standing.due_at&.to_i }.compact
       end
 
-      # What the latest run says: which, on what, and when.
-      def self.run(run)
+      # What the latest run says: which, on what, and when; a run waiting its
+      # turn (Jobs::Schedule) has not started, and says when it starts.
+      def self.run(run, starts_at)
         return {} unless run
 
-        { run_id: run[:id], sha: run[:commit_hash], branch: run[:branch], started_at: epoch(run[:started_at]),
-          updated_at: epoch(run[:completed_at] || run[:started_at]) }
+        { run_id: run[:id], sha: run[:commit_hash], branch: run[:branch], **times(run, starts_at) }
+      end
+
+      def self.times(run, starts_at)
+        return { starts_at: starts_at.to_i } if starts_at
+
+        { started_at: epoch(run[:started_at]), updated_at: epoch(run[:completed_at] || run[:started_at]) }
       end
 
       def self.epoch(iso) = Time.parse(iso).to_i
-      private_class_method :run, :epoch
+      private_class_method :run, :times, :epoch
     end
   end
 end

data/lib/fun_ci/console/job_order.rb

@@ -3,14 +3,15 @@
 module FunCi
   module Console
     # The order of the job section's rows (design.md, The console): what
-    # needs you first, then running, due and passed; among equals, the
+    # needs you first, then running, waiting its turn to (Jobs::Schedule),
+    # due and passed; among equals, the
     # board's projects in its order, then each project's jobs by name. Rows
     # answer #state (Jobs::State), #project and #name.
     module JobOrder
-      URGENCY = { "failed" => 0, "lost" => 0, "over_budget" => 1, "running" => 2, "due" => 3, "cancelled" => 3,
-                  "passed" => 4 }.freeze
+      URGENCY = { "failed" => 0, "lost" => 0, "over_budget" => 1, "running" => 2, "scheduled" => 3, "due" => 4,
+                  "cancelled" => 4, "passed" => 5 }.freeze
       # A state this fun-ci doesn't know, which a newer one sharing the database may have written.
-      UNKNOWN = 5
+      UNKNOWN = 6
 
       def self.of(rows, projects:)
         rows.sort_by do |row|

data/lib/fun_ci/console/key_handler.rb

@@ -67,8 +67,10 @@ module FunCi
         @confirm_cancel = [:run, run] if run[:status] == "running"
       end
 
-      # A running job (a Jobs::Standing) once the user confirms.
+      # A job (a Jobs::Standing) waiting its turn is cancelled at once; a
+      # running one once the user confirms.
       def offer_job_cancel(job)
+        cancel(:job, job) if job.state == "scheduled"
         @confirm_cancel = [:job, job] if job.state == "running"
       end
 

data/lib/fun_ci/evidence/settings.rb

@@ -18,7 +18,7 @@ module FunCi
 
       # The settings in the config file at `path`, or the defaults without one.
       def self.load(path)
-        new(File.exist?(path) ? (YAML.safe_load_file(path) || {})["evidence"] : nil)
+        new(File.exist?(path) ? (YAML.safe_load_file(path, aliases: true) || {})["evidence"] : nil)
       rescue Psych::Exception, NoMethodError, TypeError
         new(nil)
       end

data/lib/fun_ci/jobs/due.rb

@@ -6,8 +6,8 @@ module FunCi
   module Jobs
     # Whether a job is due, from its latest run alone ({ status:, started_at: },
     # or nil when it never ran): when it never ran, when that run was
-    # cancelled, or a period after that run started, unless it still runs
-    # (design.md, Daily and weekly jobs).
+    # cancelled, or a period after that run started, unless it still runs or
+    # waits to (design.md, Daily and weekly jobs).
     class Due
       def initialize(latest, period, now:)
         @latest = latest
@@ -26,7 +26,8 @@ module FunCi
 
       private
 
-      def running? = @latest[:status] == "running"
+      # A run waiting its turn (Schedule) counts as running: it is the job's run.
+      def running? = %w[running scheduled].include?(@latest[:status])
       def again = Time.parse(@latest[:started_at]) + @period
     end
   end

data/lib/fun_ci/jobs/due_jobs.rb

@@ -1,7 +1,10 @@
 # frozen_string_literal: true
 
+require "time"
 require_relative "standings"
+require_relative "schedule"
 require_relative "../pipeline/run_canceller"
+require_relative "../setup/project_config"
 
 module FunCi
   module Jobs
@@ -11,15 +14,36 @@ module FunCi
     # recorded failed first, or it would never be due again, and no process
     # would start to find it dead.
     class DueJobs
+      BEGUN = %w[running scheduled].freeze
+      DAY = 86_400
+
       def initialize(project, db, now:)
         @project = project
         @db = db
         @now = now
       end
 
-      def list
+      def list = standings.select(&:due_now?).map(&:job)
+
+      # [job, when its turn to start comes] for each due job (Schedule): a day
+      # shared evenly among the project's jobs apart, or its `job_spacing`,
+      # after its jobs still running or waiting to. Each is due again a
+      # period after its own start, so the spread holds from day to day.
+      def starts
+        all = standings
+        Schedule.new(spacing(all.size), begun(all), now: @now).starts(all.select(&:due_now?).map(&:job))
+      end
+
+      private
+
+      def spacing(jobs) = Setup::ProjectConfig.new(@project).job_spacing || (DAY / [jobs, 1].max)
+
+      # When each of the project's jobs still running or waiting to began, or is to.
+      def begun(all) = all.select { |one| BEGUN.include?(one.state) }.map { |one| Time.parse(one.run[:started_at]) }
+
+      def standings
         Pipeline::RunCanceller.new.record_dead_jobs(@db)
-        Standings.new(@db, @project, now: @now).all.select(&:due_now?).map(&:job)
+        Standings.new(@db, @project, now: @now).all
       end
     end
   end

data/lib/fun_ci/jobs/job_fork.rb

@@ -3,6 +3,7 @@
 require_relative "job_run"
 require_relative "../persistence/database"
 require_relative "../pipeline/worktrees"
+require_relative "../pipeline/priorities"
 
 module FunCi
   module Jobs
@@ -10,17 +11,24 @@ module FunCi
     # jobs). The trigger that forks it exits when its pipeline ends, and the
     # terminal of the commit may close; a job can run for hours, so it leads a
     # session of its own, prints nowhere, and opens its own connection, since
-    # a child must not inherit one.
-    module JobFork
-      def self.start(job, commit, project:, db_path:)
-        Process.detach(fork { run_in_child(job, commit, project, db_path) })
+    # a child must not inherit one. `priorities` says what its script runs under.
+    class JobFork
+      def initialize(project:, db_path:, priorities: Pipeline::Priorities.for(RUBY_PLATFORM))
+        @project = project
+        @db_path = db_path
+        @priorities = priorities
       end
 
-      def self.run_in_child(job, commit, project, db_path)
+      # at: when the job's turn to start comes (Schedule); now when nil.
+      def start(job, commit, at: nil) = Process.detach(fork { run_in_child(job, commit, at) })
+
+      private
+
+      def run_in_child(job, commit, at)
         Process.setsid
         silence
-        db = Persistence::Database.connection(db_path)
-        JobRun.new(job, commit, site(project, db), Pipeline::Seams.new).start
+        db = Persistence::Database.connection(@db_path)
+        JobRun.new(job, commit, site(db), Pipeline::Seams.new(priorities: @priorities)).start(at: at)
       ensure
         db&.close
       end
@@ -28,18 +36,17 @@ module FunCi
       # The process's own descriptors, whatever $stdout and $stderr stand for
       # (a test's StringIO, a guard's file), onto an open /dev/null, since
       # reopening by path can't change a stream's access mode.
-      def self.silence
+      def silence
         File.open(File::NULL, File::RDWR) do |null|
           (0..2).each { |fd| IO.for_fd(fd, autoclose: false).reopen(null) }
         end
       end
 
-      def self.site(project, db)
-        worktrees = Pipeline::Worktrees.new(project)
-        Site.new(project: project, db: db, worktrees: worktrees, locks: Locks.new(worktrees.jobs_root),
+      def site(db)
+        worktrees = Pipeline::Worktrees.new(@project)
+        Site.new(project: @project, db: db, worktrees: worktrees, locks: Locks.new(worktrees.jobs_root),
                  clock: -> { Time.now })
       end
-      private_class_method :run_in_child, :silence, :site
     end
   end
 end

data/lib/fun_ci/jobs/job_run.rb

@@ -14,8 +14,8 @@ module FunCi
     # One run of a job, in the process that runs it (design.md, Daily and
     # weekly jobs): with the job's lock held, a run left running is known to
     # be dead and is recorded failed, then the run is claimed if the job is
-    # due, and the job runs on the commit in its own worktree, recorded as a
-    # stage is.
+    # due, waits its turn if it has one (Schedule), and the job runs on the
+    # commit in its own worktree, recorded as a stage is.
     class JobRun
       def initialize(job, commit, site, seams)
         @job = job
@@ -25,9 +25,10 @@ module FunCi
       end
 
       # The run's id, or nil when another process runs the job or it isn't due.
-      def start
+      # at: when its turn to start comes (Schedule); now when nil.
+      def start(at: nil)
         slot = @site.locks.take(@job.name)
-        slot && holding(slot) { claimed(slot) }
+        slot && holding(slot) { claimed(slot, at || @site.clock.call) }
       end
 
       private
@@ -41,10 +42,27 @@ module FunCi
         slot.release
       end
 
-      def claimed(slot)
+      def claimed(slot, at)
         runs.died(@job.name)
         id = runs.claim(@job, commit: @commit.to_h, lock_file: slot.lock_file, now: @site.clock.call)
-        id && execute(slot, id)
+        id && (turn_came?(id, at) ? execute(slot, id) : id)
+      end
+
+      # Waits, scheduled, for the run's turn, the lock held; false when it was
+      # cancelled meanwhile.
+      def turn_came?(id, at)
+        seconds = at - @site.clock.call
+        return true unless seconds.positive?
+
+        scheduled(id, at)
+        @site.wait.call(seconds)
+        runs.start_turn(id, @site.clock.call).positive?
+      end
+
+      # This process is recorded, so a cancel stops it while it waits.
+      def scheduled(id, at)
+        recorder.started_by(id, Process.pid, budget: budget)
+        runs.wait_until(id, at)
       end
 
       def execute(slot, id)
@@ -64,7 +82,8 @@ module FunCi
 
       def execution(slot)
         Pipeline::StageExecution.new(seams: seams, dir: slot.path, commit: @commit,
-                                     launching: { env: { "FUN_CI_JOB" => @job.name }, held: [slot.lock] })
+                                     launching: { env: { "FUN_CI_JOB" => @job.name }, held: [slot.lock],
+                                                  priority: seams.priorities.job })
       end
 
       # The commit's own copy of the script when it has one, as a pipeline's stages are.

data/lib/fun_ci/jobs/schedule.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Jobs
+    # When each of the jobs a commit starts begins (design.md, Daily and
+    # weekly jobs): one at a time, `spacing` seconds apart, and no sooner
+    # than `spacing` after the latest start of the project's jobs already
+    # running or waiting to (`begun`), so their work is spread over time
+    # rather than taking every core at once. A spacing of 0 starts all now.
+    class Schedule
+      def initialize(spacing, begun, now:)
+        @spacing = spacing
+        @begun = begun
+        @now = now
+      end
+
+      # [job, its start] for each of `jobs`, in their order.
+      def starts(jobs) = jobs.each_with_index.map { |job, index| [job, first + (index * @spacing)] }
+
+      private
+
+      def first = @begun.empty? ? @now : [@now, @begun.max + @spacing].max
+    end
+  end
+end

data/lib/fun_ci/jobs/site.rb

@@ -1,10 +1,17 @@
 # frozen_string_literal: true
 
+require_relative "wall_clock_wait"
+
 module FunCi
   module Jobs
     # Where a project's jobs run and are recorded: the project's path, the
-    # database, its Pipeline::Worktrees (#check_out), its job Locks, and
-    # `clock`, which answers the time now.
-    Site = Data.define(:project, :db, :worktrees, :locks, :clock)
+    # database, its Pipeline::Worktrees (#check_out), its job Locks,
+    # `clock`, which answers the time now, and `wait`, which waits the
+    # seconds it is given, for a run's turn to start (Schedule).
+    Site = Data.define(:project, :db, :worktrees, :locks, :clock, :wait)
+
+    class Site
+      def initialize(wait: WallClockWait.new, **given) = super
+    end
   end
 end

data/lib/fun_ci/jobs/standings.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "time"
 require_relative "folders"
 require_relative "due"
 require_relative "state"
@@ -19,6 +20,9 @@ module FunCi
       def needs_you? = State.needs_you?(state)
       def due_now? = due.now?
       def due_at = due.at
+
+      # When a run waiting its turn (Schedule) is to start; nil for any other.
+      def starts_at = state == "scheduled" ? Time.parse(run[:started_at]) : nil
     end
 
     # Where each of a project's jobs stands at `now`, read once for the

data/lib/fun_ci/jobs/state.rb

@@ -4,12 +4,13 @@ module FunCi
   module Jobs
     # A job's state, read once from its latest run as job_runs keeps it
     # (design.md, Daily and weekly jobs): `due` while it never ran, then
-    # `running`, `passed`, `failed`, `over_budget`, `cancelled`, or `lost`
+    # `scheduled` while it waits its turn to start (Schedule), `running`,
+    # `passed`, `failed`, `over_budget`, `cancelled`, or `lost`
     # when its process died before it said how the run ended. Each surface
     # (the console, the agent commands) says these in its own words.
     module State
       OF_STATUS = { "completed" => "passed", "failed" => "failed", "timed_out" => "over_budget",
-                    "running" => "running", "cancelled" => "cancelled" }.freeze
+                    "running" => "running", "scheduled" => "scheduled", "cancelled" => "cancelled" }.freeze
       NEEDS_YOU = %w[failed over_budget lost].freeze
 
       # run: { status:, completed_at: }, or nil when the job never ran.

data/lib/fun_ci/jobs/wall_clock_wait.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Jobs
+    # Waits seconds by the wall clock, a minute at a time, for a job's turn
+    # (Schedule), which can be hours away: sleep's own count stops while the
+    # machine sleeps, and a laptop asleep for part of the wait would start the
+    # job late. It sleeps no more minutes than the wait holds and one, so a
+    # clock that never moves can't keep it waiting for ever.
+    class WallClockWait
+      MINUTE = 60
+
+      def initialize(clock: -> { Time.now }, sleep: Kernel.method(:sleep))
+        @clock = clock
+        @sleep = sleep
+      end
+
+      def call(seconds)
+        until_at = @clock.call + seconds
+        ((seconds / MINUTE).ceil + 1).times do
+          left = until_at - @clock.call
+          break unless left.positive?
+
+          @sleep.call([left, MINUTE].min)
+        end
+      end
+    end
+  end
+end

data/lib/fun_ci/persistence/active_jobs.rb

@@ -10,17 +10,20 @@ module FunCi
     # Daily and weekly jobs' runs still running, of every project, as the
     # console sees them: to cancel one, and to notice one whose process died.
     module ActiveJobs
+      # A run is still going while it runs, or waits its turn to (Jobs::Schedule).
+      ACTIVE = "status IN ('running', 'scheduled')"
+
       # The job run with this id as an ActiveRun, when it still runs; none otherwise.
       def self.with_id(db, id)
         db.execute("SELECT id, commit_hash, pid, group_pid, lock_file FROM job_runs " \
-                   "WHERE id = ? AND status = 'running'", [id]).map do |(run_id, sha, pid, group_pid, lock_file)|
+                   "WHERE id = ? AND #{ACTIVE}", [id]).map do |(run_id, sha, pid, group_pid, lock_file)|
           ActiveRun.new(id: run_id, commit_hash: sha, processes: [pid].compact, stage_groups: [group_pid].compact,
                         slot_lock: lock_file)
         end
       end
 
       # [id, lock file] of each job run still running.
-      def self.running(db) = db.execute("SELECT id, lock_file FROM job_runs WHERE status = 'running'")
+      def self.running(db) = db.execute("SELECT id, lock_file FROM job_runs WHERE #{ACTIVE}")
 
       # What a run whose process died keeps: that it has no result, and why
       # that may be. It keeps no end, since when it was found dead says
@@ -30,22 +33,25 @@ module FunCi
                            "restarted, or something killed it), so this run has no result" }.freeze
       DIED = Evidence::Document.new(chosen: [], failures: [], excerpts: [], problems: [], facts: [DIED_FACT])
 
-      def self.cancelled(db, id) = ended(db, id, "cancelled")
+      def self.cancelled(db, id) = ended(db, id, "cancelled", ACTIVE)
 
-      # A run whose process is gone without recording how it ended: failed, with no end.
+      # A run whose process is gone without recording how it ended: failed,
+      # with no end; one that was still waiting its turn never ran, so it is
+      # cancelled, and the job is due at the next commit.
       def self.died(db, id)
+        ended(db, id, "cancelled", "status = 'scheduled'")
         db.execute("UPDATE job_runs SET status = 'failed', evidence = ? WHERE id = ? AND status = 'running'",
                    [JSON.generate(DIED.to_h), id])
       end
 
       # The ids of a project's job's runs still running.
       def self.running_of(db, project, name)
-        db.execute("SELECT id FROM job_runs WHERE project_path = ? AND job = ? AND status = 'running'",
+        db.execute("SELECT id FROM job_runs WHERE project_path = ? AND job = ? AND #{ACTIVE}",
                    [project, name]).flatten
       end
 
-      def self.ended(db, id, status)
-        db.execute("UPDATE job_runs SET status = ?, completed_at = ? WHERE id = ? AND status = 'running'",
+      def self.ended(db, id, status, was)
+        db.execute("UPDATE job_runs SET status = ?, completed_at = ? WHERE id = ? AND #{was}",
                    [status, Time.now.utc.iso8601(3), id])
       end
       private_class_method :ended

data/lib/fun_ci/persistence/job_runs.rb

@@ -33,6 +33,13 @@ module FunCi
         end
       end
 
+      # Records the claimed run `id` waiting its turn to start at `at` (Jobs::Schedule).
+      def wait_until(id, at) = update(id, "scheduled", at, "status = 'running'")
+
+      # Records the run `id` running from `now`, its turn come, and answers how
+      # many runs that was: none when it was cancelled while it waited.
+      def start_turn(id, now) = update(id, "running", now, "status = 'scheduled'")
+
       # The project's newest job runs, newest first.
       def recent(limit:) = query("WHERE project_path = ? ORDER BY id DESC LIMIT ?", @project, limit)
 
@@ -51,6 +58,11 @@ module FunCi
         @db.last_insert_row_id
       end
 
+      def update(id, status, at, was)
+        @db.execute("UPDATE job_runs SET status = ?, started_at = ? WHERE id = ? AND #{was}", [status, stamp(at), id])
+        @db.changes
+      end
+
       def forget_beyond(name)
         ids = @db.execute("SELECT id FROM job_runs WHERE project_path = ? AND job = ? ORDER BY id DESC " \
                           "LIMIT -1 OFFSET ?", [@project, name, KEPT]).flatten

data/lib/fun_ci/persistence/pipeline_run.rb

@@ -5,7 +5,8 @@ require "time"
 module FunCi
   module Persistence
     module PipelineRun
-      COLUMNS = %i[id commit_hash branch status pid project_path created_at updated_at trunk_started_at].freeze
+      COLUMNS = %i[id commit_hash branch status pid project_path created_at updated_at trunk_started_at
+                   trigger_pid].freeze
       SELECT = "SELECT #{COLUMNS.join(", ")} FROM pipeline_runs".freeze
 
       def self.create(db, commit_hash:, branch:, project_path: nil)

data/lib/fun_ci/pipeline/budgets.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Pipeline
+    # Each stage's time budget in seconds (design.md); a worktree just made
+    # gives lint and build the slow suite's, once (SlotRun).
+    DEFAULT_BUDGETS = { "lint" => 30, "build" => 30, "fast" => 10, "slow" => 300 }.freeze
+  end
+end

data/lib/fun_ci/pipeline/pipeline_forker.rb

@@ -25,13 +25,15 @@ module FunCi
       # fork. The project's due jobs start beside the run, each in a process
       # of its own (design.md, Daily and weekly jobs), once the run has
       # started, and trouble starting them never stops it. jobs: what starts
-      # them, given the commit and the database's path.
+      # them, given the commit (by its full SHA, as the run keeps it) and the
+      # database's path.
       def self.fork_pipeline(commit_hash:, branch:, db_path:, jobs: method(:start_due_jobs))
         return false unless Setup::ProjectConfig.new(Dir.pwd).validate.empty?
 
         notice = first_fetch(db_path)
-        Process.detach(fork { run_in_child(commit_hash: commit_hash, branch: branch, db_path: db_path) })
-        Forked.new(notice: notice, jobs: trouble { jobs.call(Commit.new(sha: commit_hash, branch: branch), db_path) })
+        commit = Commit.new(sha: Seams.full_sha(commit_hash) || commit_hash, branch: branch)
+        Process.detach(fork { run_in_child(commit_hash: commit.sha, branch: branch, db_path: db_path) })
+        Forked.new(notice: notice, jobs: trouble { jobs.call(commit, db_path) })
       end
 
       # What went wrong starting the jobs, or nil.
@@ -50,14 +52,15 @@ module FunCi
         db&.close
       end
 
-      # Which jobs are due is read before any fork, which must not inherit the connection.
-      def self.start_due_jobs(commit, db_path)
-        due_jobs(db_path).each { |job| Jobs::JobFork.start(job, commit, project: Dir.pwd, db_path: db_path) }
+      # Which jobs are due, and when each one's turn comes, is read before any
+      # fork, which must not inherit the connection. fork: what starts each.
+      def self.start_due_jobs(commit, db_path, fork: Jobs::JobFork.new(project: Dir.pwd, db_path: db_path))
+        due_jobs(db_path).each { |job, at| fork.start(job, commit, at: at) }
       end
 
       def self.due_jobs(db_path)
         db = Persistence::Database.connection(db_path)
-        Jobs::DueJobs.new(Dir.pwd, db, now: Time.now).list
+        Jobs::DueJobs.new(Dir.pwd, db, now: Time.now).starts
       ensure
         db&.close
       end
@@ -78,7 +81,7 @@ module FunCi
         Trigger.new(project: Dir.pwd, commit: Commit.new(sha: commit_hash, branch: branch),
                     io: Io.new(stdout: File.open(File::NULL, "w")), seams: Seams.new(recorder: recorder))
       end
-      private_class_method :trigger, :first_fetch, :start_due_jobs, :due_jobs, :trouble
+      private_class_method :trigger, :first_fetch, :due_jobs, :trouble
     end
   end
 end

data/lib/fun_ci/pipeline/priorities.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Pipeline
+    # What a daily or weekly job and the slow suite are started under, so
+    # they give way to the stages a push waits for: each a prefix of the
+    # command a stage's shell execs, or "" for none. On macOS nice barely
+    # moves the scheduler, and a QoS clamp of utility lets a job have every
+    # idle core yet yield to the stages; the slow suite is left unclamped
+    # there, since clamped beside clamped jobs it crawls. Elsewhere nice does it.
+    Priorities = Data.define(:job, :slow)
+
+    class Priorities
+      MACOS = new(job: "taskpolicy -c utility ", slow: "")
+      OTHER = new(job: "nice -n 19 ", slow: "nice -n 10 ")
+      # fun-ci's own priority, for both.
+      NONE = new(job: "", slow: "")
+
+      # platform: as RUBY_PLATFORM names it.
+      def self.for(platform) = platform.include?("darwin") ? MACOS : OTHER
+    end
+  end
+end

data/lib/fun_ci/pipeline/process_runner.rb

@@ -24,13 +24,15 @@ module FunCi
 
       # Where the command runs, what it finds in its environment besides
       # fun-ci's own, the window what it prints is written to, what to do
-      # with its pid when it runs over budget, before it is killed, and the
-      # open files it holds as long as it runs, such as a lock.
-      Launch = Data.define(:chdir, :env, :output, :before_kill, :held)
+      # with its pid when it runs over budget, before it is killed, the
+      # open files it holds as long as it runs, such as a lock, and what it
+      # is run under (a Priorities prefix, "" for fun-ci's own priority).
+      Launch = Data.define(:chdir, :env, :output, :before_kill, :held, :priority)
 
       class Launch
         def initialize(**given)
-          super(chdir: Dir.pwd, env: {}, output: OutputWindow.in_memory, before_kill: nil, held: [], **given)
+          super(chdir: Dir.pwd, env: {}, output: OutputWindow.in_memory, before_kill: nil, held: [], priority: "",
+                **given)
         end
       end
 
@@ -77,7 +79,7 @@ module FunCi
       end
 
       def spawn_gated(cmd, writer, gate, launch)
-        Process.spawn(GitEnvironment::CLEAN.merge(launch.env), format(GATED, cmd),
+        Process.spawn(GitEnvironment::CLEAN.merge(launch.env), format(GATED, "#{launch.priority}#{cmd}"),
                       out: writer, err: writer, 9 => gate.child_end, pgroup: true, chdir: launch.chdir, **held(launch))
       end
 

data/lib/fun_ci/pipeline/progress_reporter.rb

@@ -17,6 +17,10 @@ module FunCi
         @stdout.puts "fun-ci: fast #{result_marker(passed)}"
       end
 
+      def cold_slot(seconds)
+        @stdout.puts "fun-ci: a new worktree, whose caches are empty, so lint and build have #{seconds}s this once."
+      end
+
       def slow_launched
         @stdout.puts "fun-ci: slow (running in background)"
       end

data/lib/fun_ci/pipeline/slot.rb

@@ -25,6 +25,17 @@ module FunCi
         @holders -= 1
         @lock.close if @holders.zero? && !@lock.closed?
       end
+
+      # Whether its worktree was just made, so has no caches yet.
+      def cold? = false
+
+      # The same slot, before anyone shares it, in a worktree just made.
+      def as_cold = ColdSlot.new(path, lock, lock_file)
+    end
+
+    # A slot whose worktree was just made: its build starts from nothing.
+    class ColdSlot < Slot
+      def cold? = true
     end
   end
 end