fun_ci 2.1.1-aarch64-linux → 2.3.0-aarch64-linux

data/lib/fun_ci/jobs/job_run.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require_relative "folders"
+require_relative "locks"
+require_relative "site"
+require_relative "../persistence/job_runs"
+require_relative "../persistence/job_recorder"
+require_relative "../pipeline/stage_execution"
+require_relative "../pipeline/trigger_params"
+require_relative "../evidence/document"
+
+module FunCi
+  module Jobs
+    # One run of a job, in the process that runs it (design.md, Daily and
+    # weekly jobs): with the job's lock held, a run left running is known to
+    # be dead and is recorded failed, then the run is claimed if the job is
+    # due, waits its turn if it has one (Schedule), and the job runs on the
+    # commit in its own worktree, recorded as a stage is.
+    class JobRun
+      def initialize(job, commit, site, seams)
+        @job = job
+        @commit = commit
+        @site = site
+        @seams = seams
+      end
+
+      # The run's id, or nil when another process runs the job or it isn't due.
+      # at: when its turn to start comes (Schedule); now when nil.
+      def start(at: nil)
+        slot = @site.locks.take(@job.name)
+        slot && holding(slot) { claimed(slot, at || @site.clock.call) }
+      end
+
+      private
+
+      def runs = Persistence::JobRuns.new(@site.db, @site.project)
+      def recorder = Persistence::JobRecorder.new(@site.db)
+
+      def holding(slot)
+        yield
+      ensure
+        slot.release
+      end
+
+      def claimed(slot, at)
+        runs.died(@job.name)
+        id = runs.claim(@job, commit: @commit.to_h, lock_file: slot.lock_file, now: @site.clock.call)
+        id && (turn_came?(id, at) ? execute(slot, id) : id)
+      end
+
+      # Waits, scheduled, for the run's turn, the lock held; false when it was
+      # cancelled meanwhile.
+      def turn_came?(id, at)
+        seconds = at - @site.clock.call
+        return true unless seconds.positive?
+
+        scheduled(id, at)
+        @site.wait.call(seconds)
+        runs.start_turn(id, @site.clock.call).positive?
+      end
+
+      # This process is recorded, so a cancel stops it while it waits.
+      def scheduled(id, at)
+        recorder.started_by(id, Process.pid, budget: budget)
+        runs.wait_until(id, at)
+      end
+
+      def execute(slot, id)
+        recorder.started_by(id, Process.pid, budget: budget)
+        @site.worktrees.check_out(slot.path, @commit.sha)
+        execution(slot).run(@job.stage, command(slot.path), recorder, id)
+        id
+      rescue StandardError => e
+        failed(id, e.message)
+      end
+
+      # The script holds the job's lock too, so the job reads as alive while
+      # any of it runs, its runner gone or not.
+      # A day for every job, unless a test says otherwise.
+      def seams = @seams.with(time_budgets: { @job.stage => Job::BUDGET }.merge(@seams.time_budgets))
+      def budget = seams.budgets[@job.stage]
+
+      def execution(slot)
+        Pipeline::StageExecution.new(seams: seams, dir: slot.path, commit: @commit,
+                                     launching: { env: { "FUN_CI_JOB" => @job.name }, held: [slot.lock],
+                                                  priority: seams.priorities.job })
+      end
+
+      # The commit's own copy of the script when it has one, as a pipeline's stages are.
+      def command(dir)
+        committed = File.join(dir, ".fun-ci", @job.cadence, "#{@job.name}.sh")
+        (File.exist?(committed) ? @job.with(script: committed) : @job).command(@commit.sha)
+      end
+
+      # Whatever stopped the run, so it isn't left running: a run left running
+      # is only found dead once its lock is free, and then says nothing of why.
+      def failed(id, problem)
+        recorder.keep_evidence(id, Evidence::Document.broken(problem))
+        recorder.end_stage(id, "failed")
+        id
+      end
+    end
+  end
+end

data/lib/fun_ci/jobs/locks.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require_relative "../pipeline/slot"
+
+module FunCi
+  module Jobs
+    # Each job's worktree and the lock beside it, under a project's
+    # <git-common-dir>/fun-ci/jobs: `<name>` and `<name>.lock`. Whoever holds
+    # a job's lock runs it, so it runs once at a time, and a lock dies with
+    # the process that held it.
+    class Locks
+      def initialize(root)
+        @root = root
+      end
+
+      # The job's worktree, held through its lock as a Pipeline::Slot, or nil
+      # while another process holds it.
+      def take(name)
+        FileUtils.mkdir_p(@root)
+        path = File.join(@root, name)
+        lock = File.new("#{path}.lock", File::RDWR | File::CREAT, 0o644)
+        return Pipeline::Slot.new(path, lock, lock.path) if lock.flock(File::LOCK_EX | File::LOCK_NB)
+
+        lock.close
+        nil
+      end
+    end
+  end
+end

data/lib/fun_ci/jobs/schedule.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Jobs
+    # When each of the jobs a commit starts begins (design.md, Daily and
+    # weekly jobs): one at a time, `spacing` seconds apart, and no sooner
+    # than `spacing` after the latest start of the project's jobs already
+    # running or waiting to (`begun`), so their work is spread over time
+    # rather than taking every core at once. A spacing of 0 starts all now.
+    class Schedule
+      def initialize(spacing, begun, now:)
+        @spacing = spacing
+        @begun = begun
+        @now = now
+      end
+
+      # [job, its start] for each of `jobs`, in their order.
+      def starts(jobs) = jobs.each_with_index.map { |job, index| [job, first + (index * @spacing)] }
+
+      private
+
+      def first = @begun.empty? ? @now : [@now, @begun.max + @spacing].max
+    end
+  end
+end

data/lib/fun_ci/jobs/site.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require_relative "wall_clock_wait"
+
+module FunCi
+  module Jobs
+    # Where a project's jobs run and are recorded: the project's path, the
+    # database, its Pipeline::Worktrees (#check_out), its job Locks,
+    # `clock`, which answers the time now, and `wait`, which waits the
+    # seconds it is given, for a run's turn to start (Schedule).
+    Site = Data.define(:project, :db, :worktrees, :locks, :clock, :wait)
+
+    class Site
+      def initialize(wait: WallClockWait.new, **given) = super
+    end
+  end
+end

data/lib/fun_ci/jobs/standings.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require "time"
+require_relative "folders"
+require_relative "due"
+require_relative "state"
+require_relative "../persistence/job_runs"
+
+module FunCi
+  module Jobs
+    # Where a job stands: its project, the job, its latest run as job_runs
+    # keeps it (nil while it never ran), and Due, which says whether the
+    # next commit starts it and when it is due again.
+    Standing = Data.define(:project, :job, :run, :due)
+
+    class Standing
+      def name = job.name
+      def cadence = job.cadence
+      def state = State.of(run)
+      def needs_you? = State.needs_you?(state)
+      def due_now? = due.now?
+      def due_at = due.at
+
+      # When a run waiting its turn (Schedule) is to start; nil for any other.
+      def starts_at = state == "scheduled" ? Time.parse(run[:started_at]) : nil
+    end
+
+    # Where each of a project's jobs stands at `now`, read once for the
+    # console, the agent commands and the trigger.
+    class Standings
+      def initialize(db, project, now:)
+        @db = db
+        @project = project
+        @now = now
+      end
+
+      # By name.
+      def all
+        runs = Persistence::JobRuns.new(@db, @project)
+        Folders.new(@project).jobs.map { |job| standing(job, runs.latest(job.name)) }
+      end
+
+      private
+
+      def standing(job, run)
+        Standing.new(project: @project, job: job, run: run, due: Due.new(run, job.period, now: @now))
+      end
+    end
+  end
+end

data/lib/fun_ci/jobs/state.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Jobs
+    # A job's state, read once from its latest run as job_runs keeps it
+    # (design.md, Daily and weekly jobs): `due` while it never ran, then
+    # `scheduled` while it waits its turn to start (Schedule), `running`,
+    # `passed`, `failed`, `over_budget`, `cancelled`, or `lost`
+    # when its process died before it said how the run ended. Each surface
+    # (the console, the agent commands) says these in its own words.
+    module State
+      OF_STATUS = { "completed" => "passed", "failed" => "failed", "timed_out" => "over_budget",
+                    "running" => "running", "scheduled" => "scheduled", "cancelled" => "cancelled" }.freeze
+      NEEDS_YOU = %w[failed over_budget lost].freeze
+
+      # run: { status:, completed_at: }, or nil when the job never ran.
+      def self.of(run)
+        return "due" unless run
+        return "lost" if run[:status] == "failed" && run[:completed_at].nil?
+
+        OF_STATUS.fetch(run[:status], "unknown")
+      end
+
+      def self.needs_you?(state) = NEEDS_YOU.include?(state)
+    end
+  end
+end

data/lib/fun_ci/jobs/wall_clock_wait.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Jobs
+    # Waits seconds by the wall clock, a minute at a time, for a job's turn
+    # (Schedule), which can be hours away: sleep's own count stops while the
+    # machine sleeps, and a laptop asleep for part of the wait would start the
+    # job late. It sleeps no more minutes than the wait holds and one, so a
+    # clock that never moves can't keep it waiting for ever.
+    class WallClockWait
+      MINUTE = 60
+
+      def initialize(clock: -> { Time.now }, sleep: Kernel.method(:sleep))
+        @clock = clock
+        @sleep = sleep
+      end
+
+      def call(seconds)
+        until_at = @clock.call + seconds
+        ((seconds / MINUTE).ceil + 1).times do
+          left = until_at - @clock.call
+          break unless left.positive?
+
+          @sleep.call([left, MINUTE].min)
+        end
+      end
+    end
+  end
+end

data/lib/fun_ci/persistence/active_jobs.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "time"
+require_relative "active_runs"
+require "json"
+require_relative "../evidence/document"
+
+module FunCi
+  module Persistence
+    # Daily and weekly jobs' runs still running, of every project, as the
+    # console sees them: to cancel one, and to notice one whose process died.
+    module ActiveJobs
+      # A run is still going while it runs, or waits its turn to (Jobs::Schedule).
+      ACTIVE = "status IN ('running', 'scheduled')"
+
+      # The job run with this id as an ActiveRun, when it still runs; none otherwise.
+      def self.with_id(db, id)
+        db.execute("SELECT id, commit_hash, pid, group_pid, lock_file FROM job_runs " \
+                   "WHERE id = ? AND #{ACTIVE}", [id]).map do |(run_id, sha, pid, group_pid, lock_file)|
+          ActiveRun.new(id: run_id, commit_hash: sha, processes: [pid].compact, stage_groups: [group_pid].compact,
+                        slot_lock: lock_file)
+        end
+      end
+
+      # [id, lock file] of each job run still running.
+      def self.running(db) = db.execute("SELECT id, lock_file FROM job_runs WHERE #{ACTIVE}")
+
+      # What a run whose process died keeps: that it has no result, and why
+      # that may be. It keeps no end, since when it was found dead says
+      # nothing of how long it ran.
+      DIED_FACT = { name: "ended", extractor: "fun-ci",
+                    value: "its process stopped before it said how the run ended (the machine slept or " \
+                           "restarted, or something killed it), so this run has no result" }.freeze
+      DIED = Evidence::Document.new(chosen: [], failures: [], excerpts: [], problems: [], facts: [DIED_FACT])
+
+      def self.cancelled(db, id) = ended(db, id, "cancelled", ACTIVE)
+
+      # A run whose process is gone without recording how it ended: failed,
+      # with no end; one that was still waiting its turn never ran, so it is
+      # cancelled, and the job is due at the next commit.
+      def self.died(db, id)
+        ended(db, id, "cancelled", "status = 'scheduled'")
+        db.execute("UPDATE job_runs SET status = 'failed', evidence = ? WHERE id = ? AND status = 'running'",
+                   [JSON.generate(DIED.to_h), id])
+      end
+
+      # The ids of a project's job's runs still running.
+      def self.running_of(db, project, name)
+        db.execute("SELECT id FROM job_runs WHERE project_path = ? AND job = ? AND #{ACTIVE}",
+                   [project, name]).flatten
+      end
+
+      def self.ended(db, id, status, was)
+        db.execute("UPDATE job_runs SET status = ?, completed_at = ? WHERE id = ? AND #{was}",
+                   [status, Time.now.utc.iso8601(3), id])
+      end
+      private_class_method :ended
+    end
+  end
+end

data/lib/fun_ci/persistence/database.rb

@@ -46,6 +46,16 @@ module FunCi
         )
       SQL
 
+      # Each run of a project's daily and weekly jobs (architecture.md, Daily and weekly jobs).
+      JOB_RUNS_TABLE = <<~SQL
+        CREATE TABLE IF NOT EXISTS job_runs (
+          id INTEGER PRIMARY KEY,
+          project_path TEXT, job TEXT, cadence TEXT, commit_hash TEXT, branch TEXT,
+          status TEXT, started_at TEXT, completed_at TEXT, lock_file TEXT, pid INTEGER, group_pid INTEGER,
+          exit_status INTEGER, signal TEXT, output_tail TEXT, evidence TEXT, budget INTEGER
+        )
+      SQL
+
       # Opening and migrating hold an exclusive lock on a file beside the
       # database, so fun-ci processes starting at once set it up one at a time.
       # SQLite's busy timeout doesn't cover the switch to WAL, and a column can
@@ -63,13 +73,13 @@ module FunCi
                        %w[pipeline_runs waited_at TEXT], %w[stage_jobs exit_status INTEGER],
                        %w[stage_jobs signal TEXT], %w[stage_jobs budget INTEGER], %w[stage_jobs pruned INTEGER],
                        %w[stage_jobs evidence TEXT], %w[pipeline_runs fetch_pgid INTEGER],
-                       %w[pipeline_runs trunk_started_at TEXT]].freeze
+                       %w[pipeline_runs trunk_started_at TEXT], %w[job_runs budget INTEGER]].freeze
+
+      TABLES = [PIPELINE_RUNS_TABLE, STAGE_JOBS_TABLE, TRUNK_CHECKS_TABLE, TRUNK_FETCHES_TABLE, JOB_RUNS_TABLE].freeze
 
       def self.migrate!(db)
         with_setup_lock(db.filename("main")) do
-          [PIPELINE_RUNS_TABLE, STAGE_JOBS_TABLE, TRUNK_CHECKS_TABLE, TRUNK_FETCHES_TABLE].each do |table|
-            db.execute(table)
-          end
+          TABLES.each { |table| db.execute(table) }
           ADDED_COLUMNS.each { |table, column, type| add_column_if_missing(db, table, column, type) }
           db.execute(NAME_DETACHED)
         end

data/lib/fun_ci/persistence/job_recorder.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "json"
+require "time"
+require_relative "raw_outputs"
+
+module FunCi
+  module Persistence
+    # Records a daily or weekly job's run as it runs, through the calls
+    # Pipeline::StageExecution makes of a stage's recorder, so a job's
+    # evidence is collected as a stage's is (architecture.md, Daily and
+    # weekly jobs). `id` is the job run's.
+    class JobRecorder
+      def initialize(db)
+        @db = db
+      end
+
+      # The process that runs the job, which a cancel stops first, and the budget it runs to.
+      def started_by(id, pid, budget: nil) = set(id, pid: pid, budget: budget)
+
+      # The job's script, which leads a process group of its own.
+      def stage_process(id, pid) = set(id, group_pid: pid)
+
+      # Unless the run was cancelled meanwhile.
+      def end_stage(id, status)
+        @db.execute("UPDATE job_runs SET status = ?, completed_at = ? WHERE id = ? AND status = 'running'",
+                    [status, Time.now.utc.iso8601(3), id])
+      end
+
+      # And the output's last lines, as a stage keeps them.
+      def keep_evidence(id, document) = set(id, evidence: JSON.generate(document.to_h), output_tail: document.tail)
+
+      def keep_exit(id, exit_status, signal) = set(id, exit_status: exit_status, signal: signal)
+      def keep_raw(id, text) = RawOutputs.for_jobs(@db.filename("main")).write(id, text)
+
+      # No stage shares a job's worktree.
+      def alongside(_id) = []
+
+      private
+
+      def set(id, **values)
+        @db.execute("UPDATE job_runs SET #{values.keys.map { |column| "#{column} = ?" }.join(", ")} WHERE id = ?",
+                    [*values.values, id])
+      end
+    end
+  end
+end

data/lib/fun_ci/persistence/job_runs.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require "time"
+require_relative "../jobs/due"
+require_relative "raw_outputs"
+require_relative "active_jobs"
+
+module FunCi
+  module Persistence
+    # The runs of one project's daily and weekly jobs (architecture.md, Daily
+    # and weekly jobs), apart from its pipeline runs.
+    class JobRuns
+      COLUMNS = %i[id project_path job cadence commit_hash branch status started_at completed_at lock_file pid
+                   group_pid exit_status signal output_tail evidence budget].freeze
+      SELECT = "SELECT #{COLUMNS.join(", ")} FROM job_runs".freeze
+      KEPT = 10
+
+      def initialize(db, project)
+        @db = db
+        @project = project
+      end
+
+      # Records a run of `job` running, testing `commit` ({ sha:, branch: }),
+      # and answers its id, when the job is due at `now`; nil otherwise. The
+      # job's latest run is read and the new one written in one transaction,
+      # so of two claims at once only one starts it. A claim forgets the
+      # job's runs beyond its newest 10.
+      def claim(job, commit:, lock_file:, now:)
+        @db.transaction(:immediate) do
+          next nil unless Jobs::Due.new(latest(job.name), job.period, now: now).now?
+
+          insert(job, commit, lock_file, now).tap { forget_beyond(job.name) }
+        end
+      end
+
+      # Records the claimed run `id` waiting its turn to start at `at` (Jobs::Schedule).
+      def wait_until(id, at) = update(id, "scheduled", at, "status = 'running'")
+
+      # Records the run `id` running from `now`, its turn come, and answers how
+      # many runs that was: none when it was cancelled while it waited.
+      def start_turn(id, now) = update(id, "running", now, "status = 'scheduled'")
+
+      # The project's newest job runs, newest first.
+      def recent(limit:) = query("WHERE project_path = ? ORDER BY id DESC LIMIT ?", @project, limit)
+
+      # The job's newest run, or nil when it never ran.
+      def latest(name) = query("WHERE project_path = ? AND job = ? ORDER BY id DESC LIMIT 1", @project, name).first
+
+      # Records failed the job's runs still running, whose process is known to be gone.
+      def died(name) = ActiveJobs.running_of(@db, @project, name).each { |id| ActiveJobs.died(@db, id) }
+
+      private
+
+      def insert(job, commit, lock_file, now)
+        @db.execute("INSERT INTO job_runs (project_path, job, cadence, commit_hash, branch, status, started_at, " \
+                    "lock_file) VALUES (?, ?, ?, ?, ?, 'running', ?, ?)",
+                    [@project, job.name, job.cadence, commit[:sha], commit[:branch], stamp(now), lock_file])
+        @db.last_insert_row_id
+      end
+
+      def update(id, status, at, was)
+        @db.execute("UPDATE job_runs SET status = ?, started_at = ? WHERE id = ? AND #{was}", [status, stamp(at), id])
+        @db.changes
+      end
+
+      def forget_beyond(name)
+        ids = @db.execute("SELECT id FROM job_runs WHERE project_path = ? AND job = ? ORDER BY id DESC " \
+                          "LIMIT -1 OFFSET ?", [@project, name, KEPT]).flatten
+        @db.execute("DELETE FROM job_runs WHERE id IN (#{ids.map { "?" }.join(", ")})", ids) if ids.any?
+        RawOutputs.for_jobs(@db.filename("main")).delete(ids)
+      end
+
+      def stamp(time) = time.utc.iso8601(3)
+
+      def query(clause, *params)
+        @db.execute("#{SELECT} #{clause}", params).map { |row| COLUMNS.zip(row).to_h }
+      end
+    end
+  end
+end

data/lib/fun_ci/persistence/pipeline_run.rb

@@ -5,7 +5,8 @@ require "time"
 module FunCi
   module Persistence
     module PipelineRun
-      COLUMNS = %i[id commit_hash branch status pid project_path created_at updated_at trunk_started_at].freeze
+      COLUMNS = %i[id commit_hash branch status pid project_path created_at updated_at trunk_started_at
+                   trigger_pid].freeze
       SELECT = "SELECT #{COLUMNS.join(", ")} FROM pipeline_runs".freeze
 
       def self.create(db, commit_hash:, branch:, project_path: nil)

data/lib/fun_ci/persistence/raw_outputs.rb

@@ -11,6 +11,9 @@ module FunCi
     class RawOutputs
       def self.beside(db_path) = new(File.join(File.dirname(db_path), "raw"))
 
+      # What failed daily and weekly jobs printed, apart, since their ids are their own.
+      def self.for_jobs(db_path) = new(File.join(File.dirname(db_path), "raw-jobs"))
+
       def initialize(dir)
         @dir = dir
       end

data/lib/fun_ci/pipeline/budgets.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Pipeline
+    # Each stage's time budget in seconds (design.md); a worktree just made
+    # gives lint and build the slow suite's, once (SlotRun).
+    DEFAULT_BUDGETS = { "lint" => 30, "build" => 30, "fast" => 10, "slow" => 300 }.freeze
+  end
+end

data/lib/fun_ci/pipeline/pipeline_forker.rb

@@ -5,24 +5,43 @@ require_relative "../persistence/pipeline_recorder"
 require_relative "trigger_params"
 require_relative "../setup/project_config"
 require_relative "../trunk/first_fetch"
+require_relative "../jobs/due_jobs"
+require_relative "../jobs/job_fork"
 
 module FunCi
   module Pipeline
     class PipelineForker
-      # A run started, and what to say before the project's first fetch of its trunk, if this is it.
-      Forked = Data.define(:notice)
+      # A run started, what to say before the project's first fetch of its
+      # trunk, if this is it, and why the daily and weekly jobs didn't start,
+      # if they didn't.
+      Forked = Data.define(:notice, :jobs)
+
+      class Forked
+        def initialize(notice:, jobs: nil) = super
+      end
 
       # Answers a Forked, or false in a project not set up for fun-ci. What the
-      # run prints goes nowhere, so the notice is worked out here, before the fork.
-      def self.fork_pipeline(commit_hash:, branch:, db_path:)
+      # run prints goes nowhere, so the notice is worked out here, before the
+      # fork. The project's due jobs start beside the run, each in a process
+      # of its own (design.md, Daily and weekly jobs), once the run has
+      # started, and trouble starting them never stops it. jobs: what starts
+      # them, given the commit (by its full SHA, as the run keeps it) and the
+      # database's path.
+      def self.fork_pipeline(commit_hash:, branch:, db_path:, jobs: method(:start_due_jobs))
         return false unless Setup::ProjectConfig.new(Dir.pwd).validate.empty?
 
         notice = first_fetch(db_path)
-        pid = fork do
-          run_in_child(commit_hash: commit_hash, branch: branch, db_path: db_path)
-        end
-        Process.detach(pid)
-        Forked.new(notice: notice)
+        commit = Commit.new(sha: Seams.full_sha(commit_hash) || commit_hash, branch: branch)
+        Process.detach(fork { run_in_child(commit_hash: commit.sha, branch: branch, db_path: db_path) })
+        Forked.new(notice: notice, jobs: trouble { jobs.call(commit, db_path) })
+      end
+
+      # What went wrong starting the jobs, or nil.
+      def self.trouble
+        yield
+        nil
+      rescue StandardError => e
+        "fun-ci: the daily and weekly jobs didn't start: #{e.message}"
       end
 
       # The connection closes before the fork, which must not inherit it.
@@ -33,6 +52,19 @@ module FunCi
         db&.close
       end
 
+      # Which jobs are due, and when each one's turn comes, is read before any
+      # fork, which must not inherit the connection. fork: what starts each.
+      def self.start_due_jobs(commit, db_path, fork: Jobs::JobFork.new(project: Dir.pwd, db_path: db_path))
+        due_jobs(db_path).each { |job, at| fork.start(job, commit, at: at) }
+      end
+
+      def self.due_jobs(db_path)
+        db = Persistence::Database.connection(db_path)
+        Jobs::DueJobs.new(Dir.pwd, db, now: Time.now).starts
+      ensure
+        db&.close
+      end
+
       # Trigger requires this file, through TriggerCommand, so it is loaded
       # here, where it is used, rather than above.
       def self.run_in_child(commit_hash:, branch:, db_path:)
@@ -49,7 +81,7 @@ module FunCi
         Trigger.new(project: Dir.pwd, commit: Commit.new(sha: commit_hash, branch: branch),
                     io: Io.new(stdout: File.open(File::NULL, "w")), seams: Seams.new(recorder: recorder))
       end
-      private_class_method :trigger, :first_fetch
+      private_class_method :trigger, :first_fetch, :due_jobs, :trouble
     end
   end
 end

data/lib/fun_ci/pipeline/priorities.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Pipeline
+    # What a daily or weekly job and the slow suite are started under, so
+    # they give way to the stages a push waits for: each a prefix of the
+    # command a stage's shell execs, or "" for none. On macOS nice barely
+    # moves the scheduler, and a QoS clamp of utility lets a job have every
+    # idle core yet yield to the stages; the slow suite is left unclamped
+    # there, since clamped beside clamped jobs it crawls. Elsewhere nice does it.
+    Priorities = Data.define(:job, :slow)
+
+    class Priorities
+      MACOS = new(job: "taskpolicy -c utility ", slow: "")
+      OTHER = new(job: "nice -n 19 ", slow: "nice -n 10 ")
+      # fun-ci's own priority, for both.
+      NONE = new(job: "", slow: "")
+
+      # platform: as RUBY_PLATFORM names it.
+      def self.for(platform) = platform.include?("darwin") ? MACOS : OTHER
+    end
+  end
+end