fun_ci 2.2.0-x86_64-linux → 2.4.0-x86_64-linux

data/lib/fun_ci/agent/status_text.rb

@@ -5,6 +5,8 @@ require_relative "verdict"
 require_relative "digest"
 require_relative "trunk_text"
 require_relative "job_report"
+require_relative "starts_in"
+require_relative "../pipeline/budgets"
 
 module FunCi
   module Agent
@@ -29,10 +31,13 @@ module FunCi
       # `  soak (weekly job) FAILED  fun-ci why --job soak`; `where` names the
       # commit it tested when that is another.
       def self.job_line(job, where = "")
-        said = "  #{job.name} (#{job.cadence} job) #{WORDS.fetch(job.state, job.state)}#{where}"
+        said = "  #{job.name} (#{job.cadence} job) #{WORDS.fetch(job.state, job.state)}#{starts(job)}#{where}"
         job.needs_you? ? "#{said}  fun-ci why --job #{job.name}" : said
       end
 
+      # `, starts in 8m` for a job waiting its turn.
+      def self.starts(job) = job.starts_in ? ", #{StartsIn.words(job.starts_in)}" : ""
+
       # The trunk lines; nothing while the check is going, unless asked about
       # (right after a commit it would say nothing useful).
       def self.trunk(report, asked)
@@ -45,10 +50,17 @@ module FunCi
 
       def self.stage_line(stage, needed)
         seconds = stage.seconds ? format("%6.1fs", stage.seconds) : " " * 7
-        note = needed.include?(stage.name) ? "" : " (not needed)"
+        note = (needed.include?(stage.name) ? "" : " (not needed)") + longer(stage)
         "  #{stage.name.ljust(6)} #{WORDS.fetch(stage.state, stage.state).ljust(12)}#{seconds}#{note}".rstrip
       end
 
+      # Only a cold worktree (WorktreePool) gives a stage more than its usual budget
+      # (AT-1.14), which a run in the background says nowhere else.
+      def self.longer(stage)
+        usual = Pipeline::DEFAULT_BUDGETS[stage.name]
+        stage.budget && usual && stage.budget > usual ? "  (cold worktree, budget #{stage.budget}s)" : ""
+      end
+
       def self.footer(report)
         return ["fun-ci why #{report.sha[0, 7]} #{report.deciding}"] if report.deciding
         return ["fun-ci why #{report.sha[0, 7]} trunk"] if report.trunk&.state == "conflicts"
@@ -56,7 +68,7 @@ module FunCi
 
         ["Superseded by #{report.superseded_by[0, 7]}."]
       end
-      private_class_method :stage_line, :footer, :job_lines, :job_line
+      private_class_method :stage_line, :longer, :footer, :job_lines, :job_line, :starts
     end
   end
 end

data/lib/fun_ci/agent/trunk_reading.rb

@@ -28,7 +28,8 @@ module FunCi
         return shown(check) if check
 
         started = run[:trunk_started_at]
-        started && Trunk::Shown.unchecked(started: Time.parse(started), now: @clock.now)
+        started && Trunk::Shown.unchecked(started: Time.parse(started), now: @clock.now,
+                                          stages_running: !run[:trigger_pid].nil?)
       end
 
       # Checks the commit against where the trunk is now, without fetching, and keeps the check,

data/lib/fun_ci/agent/wait_command.rb

@@ -52,9 +52,10 @@ module FunCi
         ExitCode::FOR.fetch(report.verdict)
       end
 
+      # A waiter keeps its run from being superseded, unless it follows the branch to the newer commit.
       def poll(sha, options)
         @context.pipeline.watch(@context.db)
-        reports.mark_waited(sha, @context.clock.now)
+        reports.mark_waited(sha, @context.clock.now) unless options.follow_branch
         report_for(sha, options) || start_after_grace(sha)
       end
 

data/lib/fun_ci/cli.rb

@@ -2,6 +2,7 @@
 
 require_relative "../fun_ci"
 require_relative "cli_help"
+require_relative "cli_command_help"
 require_relative "pipeline/trigger_params"
 require_relative "persistence/state_dir"
 require_relative "setup/commands"
@@ -17,7 +18,7 @@ module FunCi
       "prune" => :run_prune,
       "extract" => :run_extract
     }.freeze
-    AGENT_COMMANDS = %w[status runs wait events why jobs].freeze
+    AGENT_COMMANDS = %w[status runs wait events why jobs cancel].freeze
 
     def self.default_db_dir = Persistence::StateDir.path(ENV)
 
@@ -37,6 +38,7 @@ module FunCi
       return CliHelp.version(@io.stdout) if subcommand == "--version"
 
       return unknown_command(subcommand) unless ROUTES.key?(subcommand) || AGENT_COMMANDS.include?(subcommand)
+      return CliCommandHelp.help(@io.stdout, subcommand) if CliCommandHelp.asked?(args.drop(1))
 
       dispatch(subcommand, args.drop(1))
     end

data/lib/fun_ci/cli_command_help.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require_relative "cli_help"
+
+module FunCi
+  # What `fun-ci COMMAND --help` prints in place of running the command: its
+  # usage, what it is for and the options it takes, the last two read from
+  # `fun-ci --help`, whose options name in brackets the commands taking them.
+  module CliCommandHelp
+    FLAGS = %w[-h --help].freeze
+    USAGES = {
+      "trigger" => "fun-ci trigger [--background] <commit-hash> <branch>",
+      "console" => "fun-ci console",
+      "init" => "fun-ci init [--everything] [--stack NAME]",
+      "install-hooks" => "fun-ci install-hooks [post-commit | pre-push]",
+      "check" => "fun-ci check",
+      "prune" => "fun-ci prune",
+      "status" => "fun-ci status [REV] [--need LEVEL] [--trunk] [--json]",
+      "wait" => "fun-ci wait [REV] [--need LEVEL] [--within TIME] [--follow-branch] [--trunk] [--json]",
+      "runs" => "fun-ci runs [-n N] [--branch NAME] [--json]",
+      "events" => "fun-ci events [--follow] [--only failures]",
+      "why" => "fun-ci why [REV] [STAGE | trunk] [--need LEVEL] [--raw] [--json]\n       " \
+               "fun-ci why --job NAME [--raw] [--json]",
+      "jobs" => "fun-ci jobs [--json]",
+      "cancel" => "fun-ci cancel --job NAME",
+      "extract" => "fun-ci extract STAGE --output FILE [--exit N | --timed-out] [--json]"
+    }.freeze
+
+    def self.asked?(args) = args.any? { |arg| FLAGS.include?(arg) }
+
+    def self.help(out, command)
+      out.puts "Usage: #{USAGES.fetch(command)}", "", purpose(command)
+      options = options_of(command)
+      out.puts "", "Options:", *options unless options.empty?
+      0
+    end
+
+    def self.purpose(command) = CliHelp::TEXT[/^  #{Regexp.escape(command)} +(.+)$/, 1]
+
+    def self.options_of(command)
+      lines = CliHelp::TEXT.lines.select { |line| takes?(line, command) }
+      lines.map { |line| line.sub(/\([^)]+\) /, "") }
+    end
+
+    def self.takes?(option_line, command)
+      commands = option_line[/^\s+-\S.*?\(([^)]+)\)/, 1]
+      !commands.nil? && commands.split(", ").include?(command)
+    end
+    private_class_method :purpose, :options_of, :takes?
+  end
+end

data/lib/fun_ci/cli_help.rb

@@ -32,33 +32,34 @@ module FunCi
         events         Print this project's runs' events as JSON lines
         why            Print everything kept about why a commit's stage failed, or its conflict with the trunk
         jobs           List this project's daily and weekly jobs and how each stands
+        cancel         Cancel a daily or weekly job's run: cancel --job NAME
         extract        Try a stage's evidence extractors on a saved output
 
       Options:
-        -h, --help     Show this help message
+        -h, --help     Show this help message; after a command, that command's usage
         --version      Show version
 
-      Trigger options:
-        --background   Run the pipeline in the background and return at once
-
-      Agent options:
+      Command options:
+        --background   (trigger) Run the pipeline in the background and return at once
+        --everything   (init) Run init + install-hooks + check in one step
+        --stack NAME   (init) Write the scripts for this stack, such as go, instead of the one detected
         --need LEVEL   (status, wait, why) What must pass: build, fast (default) or all
         --within TIME  (wait) Give up undecided after 30, 30s or 5m
         --follow-branch  (wait) Move on to the newer commit that superseded the run
         --trunk        (status, wait) Exit 6 when the run passed but conflicts with the trunk
         --raw          (why) Print the stage's whole output as it was kept
-        --job NAME     (why) Explain the daily or weekly job's latest run instead of a commit's stage
+        --job NAME     (why, cancel) The daily or weekly job to explain or cancel, instead of a commit
         -n N           (runs) How many runs to list (default 10)
         --branch NAME  (runs) Only runs on this branch
         --follow       (events) Keep printing events as they happen, until stopped
         --only failures  (events) Print only failed stages, superseded runs and failed jobs
-        --json         Print JSON instead of text (events always print JSON lines)
+        --output FILE  (extract) The saved output of the stage to read
+        --exit N       (extract) The exit status the stage ended with (default 1)
+        --timed-out    (extract) The stage ran out of time instead of exiting
+        --json         (status, wait, runs, why, jobs, extract) Print JSON instead of text; events always print JSON lines
         Exit codes: 0 passed, 1 failed, 2 over budget, 3 undecided,
         4 superseded, 5 no run, 6 conflicts with the trunk (--trunk), 64 usage error
 
-      Init options:
-        --everything   Run init + install-hooks + check in one step
-
       Examples:
         fun-ci init --everything
         fun-ci trigger abc1234 main

data/lib/fun_ci/console/board_data.rb

@@ -19,6 +19,9 @@ module FunCi
       # How many of a branch's runs are read to fold its cancelled ones and
       # find its standing against the trunk.
       BRANCH_WINDOW = 100
+      # How many branches the board holds at most: those whose newest run
+      # changed most recently, so old failures leave it in time.
+      RECENT_BRANCHES = 20
 
       # `clock` answers the time now, which decides which jobs are due.
       def initialize(db, page_size: 15, run_canceller: Pipeline::RunCanceller.new, clock: -> { Time.now })
@@ -34,7 +37,7 @@ module FunCi
       def resize(page_size) = @paging.resize(page_size)
 
       # Whether there are branches beyond those `runs` loads.
-      def more? = Persistence::PipelineRun.branch_heads(@db, limit: @paging.limit + 1).size > @paging.limit
+      def more? = heads(@paging.limit + 1).size > @paging.limit
 
       # Records failed each slow suite whose process died (acceptance-tests.md, AT-8.3).
       def record_dead_slow_suites = @run_canceller.record_dead(@db)
@@ -55,7 +58,7 @@ module FunCi
       # and then those run most recently, in RowOrder: a run of cancelled
       # runs folded into one, with its stages and the branch's standing against the trunk.
       def runs
-        RowOrder.of(Persistence::PipelineRun.branch_heads(@db, limit: @paging.limit).map { |head| branch_row(head) })
+        RowOrder.of(heads(@paging.limit).map { |head| branch_row(head) })
       end
 
       # The projects among `runs` whose trunk is stale.
@@ -74,6 +77,8 @@ module FunCi
 
       private
 
+      def heads(limit) = Persistence::PipelineRun.branch_heads(@db, limit: limit, among: RECENT_BRANCHES)
+
       def branch_row(head)
         branch_runs = Persistence::PipelineRun.of_branch(@db, head[:project_path], head[:branch], limit: BRANCH_WINDOW)
         row = CancelledFolding.fold(branch_runs).first

data/lib/fun_ci/console/job_message.rb

@@ -14,20 +14,26 @@ module FunCi
 
       def self.from(standing)
         { project: standing.project, name: standing.name, cadence: standing.cadence,
-          status: STATUS.fetch(standing.state, standing.state), **run(standing.run),
+          status: STATUS.fetch(standing.state, standing.state), **run(standing.run, standing.starts_at),
           due_at: standing.due_at&.to_i }.compact
       end
 
-      # What the latest run says: which, on what, and when.
-      def self.run(run)
+      # What the latest run says: which, on what, and when; a run waiting its
+      # turn (Jobs::Schedule) has not started, and says when it starts.
+      def self.run(run, starts_at)
         return {} unless run
 
-        { run_id: run[:id], sha: run[:commit_hash], branch: run[:branch], started_at: epoch(run[:started_at]),
-          updated_at: epoch(run[:completed_at] || run[:started_at]) }
+        { run_id: run[:id], sha: run[:commit_hash], branch: run[:branch], **times(run, starts_at) }
+      end
+
+      def self.times(run, starts_at)
+        return { starts_at: starts_at.to_i } if starts_at
+
+        { started_at: epoch(run[:started_at]), updated_at: epoch(run[:completed_at] || run[:started_at]) }
       end
 
       def self.epoch(iso) = Time.parse(iso).to_i
-      private_class_method :run, :epoch
+      private_class_method :run, :times, :epoch
     end
   end
 end

data/lib/fun_ci/console/job_order.rb

@@ -3,14 +3,15 @@
 module FunCi
   module Console
     # The order of the job section's rows (design.md, The console): what
-    # needs you first, then running, due and passed; among equals, the
+    # needs you first, then running, waiting its turn to (Jobs::Schedule),
+    # due and passed; among equals, the
     # board's projects in its order, then each project's jobs by name. Rows
     # answer #state (Jobs::State), #project and #name.
     module JobOrder
-      URGENCY = { "failed" => 0, "lost" => 0, "over_budget" => 1, "running" => 2, "due" => 3, "cancelled" => 3,
-                  "passed" => 4 }.freeze
+      URGENCY = { "failed" => 0, "lost" => 0, "over_budget" => 1, "running" => 2, "scheduled" => 3, "due" => 4,
+                  "cancelled" => 4, "passed" => 5 }.freeze
       # A state this fun-ci doesn't know, which a newer one sharing the database may have written.
-      UNKNOWN = 5
+      UNKNOWN = 6
 
       def self.of(rows, projects:)
         rows.sort_by do |row|

data/lib/fun_ci/console/key_handler.rb

@@ -67,8 +67,10 @@ module FunCi
         @confirm_cancel = [:run, run] if run[:status] == "running"
       end
 
-      # A running job (a Jobs::Standing) once the user confirms.
+      # A job (a Jobs::Standing) waiting its turn is cancelled at once; a
+      # running one once the user confirms.
       def offer_job_cancel(job)
+        cancel(:job, job) if job.state == "scheduled"
         @confirm_cancel = [:job, job] if job.state == "running"
       end
 

data/lib/fun_ci/console/trunk_marks.rb

@@ -44,10 +44,12 @@ module FunCi
 
       def stale_since(project, now)
         last = Persistence::TrunkFetches.new(@db, project).last
-        return nil unless last && (last.error || now - last.fetched_at > Trunk::Shown::FRESH_FOR)
+        return nil unless last && (last.error || old?(last.fetched_at, now))
 
         { project: project, since: last.fetched_at&.to_i }
       end
+
+      def old?(fetched_at, now) = !fetched_at.nil? && now - fetched_at > Trunk::Shown::FRESH_FOR
     end
   end
 end

data/lib/fun_ci/evidence/about_stage.rb

@@ -10,7 +10,7 @@ module FunCi
     # command to read.
     module AboutStage
       def self.of(sources, outcome, output)
-        About.new(**outcome.to_h.except(:overrun), **started(sources), **where(sources, output),
+        About.new(**outcome.to_h.except(:overrun, :elsewhere), **started(sources), **where(sources, output),
                   stage: sources.stage, budget: sources.budget, commit: sources.commit)
       end
 

data/lib/fun_ci/evidence/collector.rb

@@ -37,7 +37,7 @@ module FunCi
       def collect(output, outcome = FAILED)
         result = extracted(output, outcome)
         overrun = outcome.overrun || NOTHING
-        parts = [["fun-ci", own_facts(outcome.alongside)], *overrun.parts, *result.parts, ["output-tail", tail(output)]]
+        parts = [["fun-ci", own_facts(outcome)], *overrun.parts, *result.parts, ["output-tail", tail(output)]]
         Caps.new.apply(masking.document(assemble(parts, [overrun, result])))
       end
 
@@ -76,10 +76,13 @@ module FunCi
 
       def on_of(raw) = raw.is_a?(Hash) ? raw["on"] : nil
 
-      def own_facts(alongside)
-        Findings.new(facts: alongside.empty? ? [] : [{ name: "alongside", value: alongside.join(", ") }])
+      def own_facts(outcome)
+        named = { "alongside" => outcome.alongside, "other runs alongside" => outcome.elsewhere }
+        Findings.new(facts: named.reject { |_, stages| stages.empty? }.map { |name, stages| fact(name, stages) })
       end
 
+      def fact(name, stages) = { name: name, value: stages.join(", ") }
+
       def tail(output)
         text = Persistence::OutputTail.of(output, mask: masking.method(:mask))
         Findings.new(excerpts: [{ title: Document::TAIL_TITLE, location: "output", lines: text.lines(chomp: true) }])

data/lib/fun_ci/evidence/outcome.rb

@@ -4,12 +4,15 @@ module FunCi
   module Evidence
     # How a failed stage ended, known once it has: its state in the words
     # `status --json` uses, its exit status or signal, and the other stages
-    # that shared its slot while it ran, and for an overrun, what was found
+    # that shared its slot while it ran, the stages of the project's other runs
+    # that ran meanwhile (elsewhere), and for an overrun, what was found
     # before the kill (an Extraction::Result).
-    Outcome = Data.define(:state, :exit_status, :signal, :alongside, :overrun)
+    Outcome = Data.define(:state, :exit_status, :signal, :alongside, :elsewhere, :overrun)
 
     class Outcome
-      def initialize(**given) = super(exit_status: nil, signal: nil, alongside: [], overrun: nil, **given)
+      def initialize(**given)
+        super(exit_status: nil, signal: nil, alongside: [], elsewhere: [], overrun: nil, **given)
+      end
     end
   end
 end

data/lib/fun_ci/evidence/settings.rb

@@ -18,7 +18,7 @@ module FunCi
 
       # The settings in the config file at `path`, or the defaults without one.
       def self.load(path)
-        new(File.exist?(path) ? (YAML.safe_load_file(path) || {})["evidence"] : nil)
+        new(File.exist?(path) ? (YAML.safe_load_file(path, aliases: true) || {})["evidence"] : nil)
       rescue Psych::Exception, NoMethodError, TypeError
         new(nil)
       end

data/lib/fun_ci/jobs/due.rb

@@ -6,8 +6,8 @@ module FunCi
   module Jobs
     # Whether a job is due, from its latest run alone ({ status:, started_at: },
     # or nil when it never ran): when it never ran, when that run was
-    # cancelled, or a period after that run started, unless it still runs
-    # (design.md, Daily and weekly jobs).
+    # cancelled, or a period after that run started, unless it still runs or
+    # waits to (design.md, Daily and weekly jobs).
     class Due
       def initialize(latest, period, now:)
         @latest = latest
@@ -26,7 +26,8 @@ module FunCi
 
       private
 
-      def running? = @latest[:status] == "running"
+      # A run waiting its turn (Schedule) counts as running: it is the job's run.
+      def running? = %w[running scheduled].include?(@latest[:status])
       def again = Time.parse(@latest[:started_at]) + @period
     end
   end

data/lib/fun_ci/jobs/due_jobs.rb

@@ -1,7 +1,10 @@
 # frozen_string_literal: true
 
+require "time"
 require_relative "standings"
+require_relative "schedule"
 require_relative "../pipeline/run_canceller"
+require_relative "../setup/project_config"
 
 module FunCi
   module Jobs
@@ -11,15 +14,36 @@ module FunCi
     # recorded failed first, or it would never be due again, and no process
     # would start to find it dead.
     class DueJobs
+      BEGUN = %w[running scheduled].freeze
+      DAY = 86_400
+
       def initialize(project, db, now:)
         @project = project
         @db = db
         @now = now
       end
 
-      def list
+      def list = standings.select(&:due_now?).map(&:job)
+
+      # [job, when its turn to start comes] for each due job (Schedule): a day
+      # shared evenly among the project's jobs apart, or its `job_spacing`,
+      # after its jobs still running or waiting to. Each is due again a
+      # period after its own start, so the spread holds from day to day.
+      def starts
+        all = standings
+        Schedule.new(spacing(all.size), begun(all), now: @now).starts(all.select(&:due_now?).map(&:job))
+      end
+
+      private
+
+      def spacing(jobs) = Setup::ProjectConfig.new(@project).job_spacing || (DAY / [jobs, 1].max)
+
+      # When each of the project's jobs still running or waiting to began, or is to.
+      def begun(all) = all.select { |one| BEGUN.include?(one.state) }.map { |one| Time.parse(one.run[:started_at]) }
+
+      def standings
         Pipeline::RunCanceller.new.record_dead_jobs(@db)
-        Standings.new(@db, @project, now: @now).all.select(&:due_now?).map(&:job)
+        Standings.new(@db, @project, now: @now).all
       end
     end
   end

data/lib/fun_ci/jobs/job_fork.rb

@@ -3,6 +3,7 @@
 require_relative "job_run"
 require_relative "../persistence/database"
 require_relative "../pipeline/worktrees"
+require_relative "../pipeline/priorities"
 
 module FunCi
   module Jobs
@@ -10,17 +11,24 @@ module FunCi
     # jobs). The trigger that forks it exits when its pipeline ends, and the
     # terminal of the commit may close; a job can run for hours, so it leads a
     # session of its own, prints nowhere, and opens its own connection, since
-    # a child must not inherit one.
-    module JobFork
-      def self.start(job, commit, project:, db_path:)
-        Process.detach(fork { run_in_child(job, commit, project, db_path) })
+    # a child must not inherit one. `priorities` says what its script runs under.
+    class JobFork
+      def initialize(project:, db_path:, priorities: Pipeline::Priorities.for(RUBY_PLATFORM))
+        @project = project
+        @db_path = db_path
+        @priorities = priorities
       end
 
-      def self.run_in_child(job, commit, project, db_path)
+      # at: when the job's turn to start comes (Schedule); now when nil.
+      def start(job, commit, at: nil) = Process.detach(fork { run_in_child(job, commit, at) })
+
+      private
+
+      def run_in_child(job, commit, at)
         Process.setsid
         silence
-        db = Persistence::Database.connection(db_path)
-        JobRun.new(job, commit, site(project, db), Pipeline::Seams.new).start
+        db = Persistence::Database.connection(@db_path)
+        JobRun.new(job, commit, site(db), Pipeline::Seams.new(priorities: @priorities)).start(at: at)
       ensure
         db&.close
       end
@@ -28,18 +36,17 @@ module FunCi
       # The process's own descriptors, whatever $stdout and $stderr stand for
       # (a test's StringIO, a guard's file), onto an open /dev/null, since
       # reopening by path can't change a stream's access mode.
-      def self.silence
+      def silence
         File.open(File::NULL, File::RDWR) do |null|
           (0..2).each { |fd| IO.for_fd(fd, autoclose: false).reopen(null) }
         end
       end
 
-      def self.site(project, db)
-        worktrees = Pipeline::Worktrees.new(project)
-        Site.new(project: project, db: db, worktrees: worktrees, locks: Locks.new(worktrees.jobs_root),
+      def site(db)
+        worktrees = Pipeline::Worktrees.new(@project)
+        Site.new(project: @project, db: db, worktrees: worktrees, locks: Locks.new(worktrees.jobs_root),
                  clock: -> { Time.now })
       end
-      private_class_method :run_in_child, :silence, :site
     end
   end
 end

data/lib/fun_ci/jobs/job_run.rb

@@ -14,8 +14,8 @@ module FunCi
     # One run of a job, in the process that runs it (design.md, Daily and
     # weekly jobs): with the job's lock held, a run left running is known to
     # be dead and is recorded failed, then the run is claimed if the job is
-    # due, and the job runs on the commit in its own worktree, recorded as a
-    # stage is.
+    # due, waits its turn if it has one (Schedule), and the job runs on the
+    # commit in its own worktree, recorded as a stage is.
     class JobRun
       def initialize(job, commit, site, seams)
         @job = job
@@ -25,9 +25,10 @@ module FunCi
       end
 
       # The run's id, or nil when another process runs the job or it isn't due.
-      def start
+      # at: when its turn to start comes (Schedule); now when nil.
+      def start(at: nil)
         slot = @site.locks.take(@job.name)
-        slot && holding(slot) { claimed(slot) }
+        slot && holding(slot) { claimed(slot, at || @site.clock.call) }
       end
 
       private
@@ -41,10 +42,27 @@ module FunCi
         slot.release
       end
 
-      def claimed(slot)
+      def claimed(slot, at)
         runs.died(@job.name)
         id = runs.claim(@job, commit: @commit.to_h, lock_file: slot.lock_file, now: @site.clock.call)
-        id && execute(slot, id)
+        id && (turn_came?(id, at) ? execute(slot, id) : id)
+      end
+
+      # Waits, scheduled, for the run's turn, the lock held; false when it was
+      # cancelled meanwhile.
+      def turn_came?(id, at)
+        seconds = at - @site.clock.call
+        return true unless seconds.positive?
+
+        scheduled(id, at)
+        @site.wait.call(seconds)
+        runs.start_turn(id, @site.clock.call).positive?
+      end
+
+      # This process is recorded, so a cancel stops it while it waits.
+      def scheduled(id, at)
+        recorder.started_by(id, Process.pid, budget: budget)
+        runs.wait_until(id, at)
       end
 
       def execute(slot, id)
@@ -64,7 +82,8 @@ module FunCi
 
       def execution(slot)
         Pipeline::StageExecution.new(seams: seams, dir: slot.path, commit: @commit,
-                                     launching: { env: { "FUN_CI_JOB" => @job.name }, held: [slot.lock] })
+                                     launching: { env: { "FUN_CI_JOB" => @job.name }, held: [slot.lock],
+                                                  priority: seams.priorities.job })
       end
 
       # The commit's own copy of the script when it has one, as a pipeline's stages are.

data/lib/fun_ci/jobs/locks.rb

@@ -6,7 +6,7 @@ require_relative "../pipeline/slot"
 module FunCi
   module Jobs
     # Each job's worktree and the lock beside it, under a project's
-    # <git-common-dir>/fun-ci/jobs: `<name>` and `<name>.lock`. Whoever holds
+    # Worktrees#jobs_root: `<name>` and `<name>.lock`. Whoever holds
     # a job's lock runs it, so it runs once at a time, and a lock dies with
     # the process that held it.
     class Locks

data/lib/fun_ci/jobs/schedule.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Jobs
+    # When each of the jobs a commit starts begins (design.md, Daily and
+    # weekly jobs): one at a time, `spacing` seconds apart, and no sooner
+    # than `spacing` after the latest start of the project's jobs already
+    # running or waiting to (`begun`), so their work is spread over time
+    # rather than taking every core at once. A spacing of 0 starts all now.
+    class Schedule
+      def initialize(spacing, begun, now:)
+        @spacing = spacing
+        @begun = begun
+        @now = now
+      end
+
+      # [job, its start] for each of `jobs`, in their order.
+      def starts(jobs) = jobs.each_with_index.map { |job, index| [job, first + (index * @spacing)] }
+
+      private
+
+      def first = @begun.empty? ? @now : [@now, @begun.max + @spacing].max
+    end
+  end
+end

data/lib/fun_ci/jobs/site.rb

@@ -1,10 +1,17 @@
 # frozen_string_literal: true
 
+require_relative "wall_clock_wait"
+
 module FunCi
   module Jobs
     # Where a project's jobs run and are recorded: the project's path, the
-    # database, its Pipeline::Worktrees (#check_out), its job Locks, and
-    # `clock`, which answers the time now.
-    Site = Data.define(:project, :db, :worktrees, :locks, :clock)
+    # database, its Pipeline::Worktrees (#check_out), its job Locks,
+    # `clock`, which answers the time now, and `wait`, which waits the
+    # seconds it is given, for a run's turn to start (Schedule).
+    Site = Data.define(:project, :db, :worktrees, :locks, :clock, :wait)
+
+    class Site
+      def initialize(wait: WallClockWait.new, **given) = super
+    end
   end
 end