fun_ci 2.1.1-aarch64-linux → 2.2.0-aarch64-linux

data/lib/fun_ci/evidence/settings_check.rb

@@ -8,7 +8,7 @@ module FunCi
     # What is wrong with the `evidence` key of .fun-ci/config, each as a line
     # `fun-ci check` prints.
     class SettingsCheck
-      KNOWN = %w[budget detect skip mask masking stages].freeze
+      KNOWN = %w[budget detect skip mask masking stages jobs].freeze
       STAGES = %w[all lint build fast slow].freeze
 
       # "2", 2, 2.5 or "2s" as seconds, or nil.
@@ -28,7 +28,7 @@ module FunCi
         return [] if @raw.nil?
         return ["evidence must be a mapping, not #{@raw.inspect}"] unless @raw.is_a?(Hash)
 
-        unknown + budget + stages
+        unknown + budget + stages + jobs
       end
 
       private
@@ -48,12 +48,23 @@ module FunCi
         lists.flat_map { |stage, entries| stage_errors(stage, entries) }
       end
 
+      def jobs
+        lists = @raw.fetch("jobs", {})
+        return ["evidence.jobs must be a mapping of job to entries"] unless lists.is_a?(Hash)
+
+        lists.flat_map { |job, entries| entry_errors("evidence.jobs.#{job}", entries) }
+      end
+
+      def entry_errors(where, entries)
+        Array(entries).filter_map { |entry| refusal(entry) }.map { |message| "#{where}: #{message}" }
+      end
+
       def stage_errors(stage, entries)
         unless STAGES.include?(stage)
           return ["evidence.stages has no stage '#{stage}': use all, lint, build, fast or slow"]
         end
 
-        Array(entries).filter_map { |entry| refusal(entry) }.map { |message| "evidence.stages.#{stage}: #{message}" }
+        entry_errors("evidence.stages.#{stage}", entries)
       end
 
       def refusal(entry)

data/lib/fun_ci/jobs/due.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require "time"
+
+module FunCi
+  module Jobs
+    # Whether a job is due, from its latest run alone ({ status:, started_at: },
+    # or nil when it never ran): when it never ran, when that run was
+    # cancelled, or a period after that run started, unless it still runs
+    # (design.md, Daily and weekly jobs).
+    class Due
+      def initialize(latest, period, now:)
+        @latest = latest
+        @period = period
+        @now = now
+      end
+
+      def now?
+        return true if @latest.nil? || @latest[:status] == "cancelled"
+
+        !running? && again <= @now
+      end
+
+      # When it is due again, or nil while it is due now or still runs.
+      def at = now? || running? ? nil : again
+
+      private
+
+      def running? = @latest[:status] == "running"
+      def again = Time.parse(@latest[:started_at]) + @period
+    end
+  end
+end

data/lib/fun_ci/jobs/due_jobs.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require_relative "standings"
+require_relative "../pipeline/run_canceller"
+
+module FunCi
+  module Jobs
+    # The project's jobs that are due at `now`, read from the database without
+    # taking any lock: who starts a job is decided again, under its lock, by
+    # the process that runs it. A job left running by a process that died is
+    # recorded failed first, or it would never be due again, and no process
+    # would start to find it dead.
+    class DueJobs
+      def initialize(project, db, now:)
+        @project = project
+        @db = db
+        @now = now
+      end
+
+      def list
+        Pipeline::RunCanceller.new.record_dead_jobs(@db)
+        Standings.new(@db, @project, now: @now).all.select(&:due_now?).map(&:job)
+      end
+    end
+  end
+end

data/lib/fun_ci/jobs/folders.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require_relative "job"
+
+module FunCi
+  module Jobs
+    # The jobs a project keeps in .fun-ci/daily/ and .fun-ci/weekly/: each
+    # executable `<name>.sh`. A script that can't run, or a name in both
+    # folders, is no job; #errors says why, for `fun-ci check`.
+    class Folders
+      CADENCES = Job::PERIODS.keys
+      # A name a command line keeps whole, since agents paste it: fun-ci why --job NAME.
+      NAME = /\A[A-Za-z0-9._-]+\z/
+
+      def initialize(project_root)
+        @fun_ci_dir = File.join(project_root, ".fun-ci")
+      end
+
+      # By name.
+      def jobs
+        all = scripts
+        twice = twice(all)
+        all.select { |job| runnable?(job) && NAME.match?(job.name) && !twice.include?(job.name) }.sort_by(&:name)
+      end
+
+      def errors
+        all = scripts
+        all.reject { |job| runnable?(job) }.map { |job| "#{path(job)} is not executable" } +
+          all.reject { |job| NAME.match?(job.name) }.map { |job| misnamed(job) } +
+          twice(all).map { |name| "the job #{name} is in both .fun-ci/daily/ and .fun-ci/weekly/: rename one" }
+      end
+
+      private
+
+      def runnable?(job) = File.executable?(job.script)
+      def path(job) = ".fun-ci/#{job.cadence}/#{job.name}.sh"
+      def misnamed(job) = "the job '#{job.name}' needs a name of letters, digits, '.', '_' and '-': rename #{path(job)}"
+
+      def twice(all) = all.group_by(&:name).select { |_, jobs| jobs.size > 1 }.keys.sort
+
+      def scripts
+        CADENCES.flat_map do |cadence|
+          Dir.glob("*.sh", base: File.join(@fun_ci_dir, cadence)).map do |file|
+            Job.new(name: File.basename(file, ".sh"), cadence: cadence, script: File.join(@fun_ci_dir, cadence, file))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fun_ci/jobs/job.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require "shellwords"
+
+module FunCi
+  module Jobs
+    # A script a project runs at most once a day or once a week, named after
+    # it (design.md, Daily and weekly jobs). `cadence` is "daily" or "weekly".
+    Job = Data.define(:name, :cadence, :script)
+
+    class Job
+      PERIODS = { "daily" => 86_400, "weekly" => 604_800 }.freeze
+      # Every job's, the weekly ones' too.
+      BUDGET = 86_400
+      # How a job is named where a stage would be.
+      STAGE = "jobs/"
+
+      # The name of the job a stage name names, or nil for a pipeline stage's.
+      def self.named_by(stage) = (stage.delete_prefix(STAGE) if stage.start_with?(STAGE))
+
+      # Seconds from one run's start until the job is due again.
+      def period = PERIODS.fetch(cadence)
+
+      # What evidence knows it by: `evidence: jobs: <name>:` in .fun-ci/config.
+      def stage = "#{STAGE}#{name}"
+
+      # The shell command that runs it on a commit, its path quoted.
+      def command(sha) = "#{Shellwords.escape(script)} #{sha}"
+    end
+  end
+end

data/lib/fun_ci/jobs/job_fork.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require_relative "job_run"
+require_relative "../persistence/database"
+require_relative "../pipeline/worktrees"
+
+module FunCi
+  module Jobs
+    # Starts a job in a process of its own (architecture.md, Daily and weekly
+    # jobs). The trigger that forks it exits when its pipeline ends, and the
+    # terminal of the commit may close; a job can run for hours, so it leads a
+    # session of its own, prints nowhere, and opens its own connection, since
+    # a child must not inherit one.
+    module JobFork
+      def self.start(job, commit, project:, db_path:)
+        Process.detach(fork { run_in_child(job, commit, project, db_path) })
+      end
+
+      def self.run_in_child(job, commit, project, db_path)
+        Process.setsid
+        silence
+        db = Persistence::Database.connection(db_path)
+        JobRun.new(job, commit, site(project, db), Pipeline::Seams.new).start
+      ensure
+        db&.close
+      end
+
+      # The process's own descriptors, whatever $stdout and $stderr stand for
+      # (a test's StringIO, a guard's file), onto an open /dev/null, since
+      # reopening by path can't change a stream's access mode.
+      def self.silence
+        File.open(File::NULL, File::RDWR) do |null|
+          (0..2).each { |fd| IO.for_fd(fd, autoclose: false).reopen(null) }
+        end
+      end
+
+      def self.site(project, db)
+        worktrees = Pipeline::Worktrees.new(project)
+        Site.new(project: project, db: db, worktrees: worktrees, locks: Locks.new(worktrees.jobs_root),
+                 clock: -> { Time.now })
+      end
+      private_class_method :run_in_child, :silence, :site
+    end
+  end
+end

data/lib/fun_ci/jobs/job_run.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require_relative "folders"
+require_relative "locks"
+require_relative "site"
+require_relative "../persistence/job_runs"
+require_relative "../persistence/job_recorder"
+require_relative "../pipeline/stage_execution"
+require_relative "../pipeline/trigger_params"
+require_relative "../evidence/document"
+
+module FunCi
+  module Jobs
+    # One run of a job, in the process that runs it (design.md, Daily and
+    # weekly jobs): with the job's lock held, a run left running is known to
+    # be dead and is recorded failed, then the run is claimed if the job is
+    # due, and the job runs on the commit in its own worktree, recorded as a
+    # stage is.
+    class JobRun
+      def initialize(job, commit, site, seams)
+        @job = job
+        @commit = commit
+        @site = site
+        @seams = seams
+      end
+
+      # The run's id, or nil when another process runs the job or it isn't due.
+      def start
+        slot = @site.locks.take(@job.name)
+        slot && holding(slot) { claimed(slot) }
+      end
+
+      private
+
+      def runs = Persistence::JobRuns.new(@site.db, @site.project)
+      def recorder = Persistence::JobRecorder.new(@site.db)
+
+      def holding(slot)
+        yield
+      ensure
+        slot.release
+      end
+
+      def claimed(slot)
+        runs.died(@job.name)
+        id = runs.claim(@job, commit: @commit.to_h, lock_file: slot.lock_file, now: @site.clock.call)
+        id && execute(slot, id)
+      end
+
+      def execute(slot, id)
+        recorder.started_by(id, Process.pid, budget: budget)
+        @site.worktrees.check_out(slot.path, @commit.sha)
+        execution(slot).run(@job.stage, command(slot.path), recorder, id)
+        id
+      rescue StandardError => e
+        failed(id, e.message)
+      end
+
+      # The script holds the job's lock too, so the job reads as alive while
+      # any of it runs, its runner gone or not.
+      # A day for every job, unless a test says otherwise.
+      def seams = @seams.with(time_budgets: { @job.stage => Job::BUDGET }.merge(@seams.time_budgets))
+      def budget = seams.budgets[@job.stage]
+
+      def execution(slot)
+        Pipeline::StageExecution.new(seams: seams, dir: slot.path, commit: @commit,
+                                     launching: { env: { "FUN_CI_JOB" => @job.name }, held: [slot.lock] })
+      end
+
+      # The commit's own copy of the script when it has one, as a pipeline's stages are.
+      def command(dir)
+        committed = File.join(dir, ".fun-ci", @job.cadence, "#{@job.name}.sh")
+        (File.exist?(committed) ? @job.with(script: committed) : @job).command(@commit.sha)
+      end
+
+      # Whatever stopped the run, so it isn't left running: a run left running
+      # is only found dead once its lock is free, and then says nothing of why.
+      def failed(id, problem)
+        recorder.keep_evidence(id, Evidence::Document.broken(problem))
+        recorder.end_stage(id, "failed")
+        id
+      end
+    end
+  end
+end

data/lib/fun_ci/jobs/locks.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require_relative "../pipeline/slot"
+
+module FunCi
+  module Jobs
+    # Each job's worktree and the lock beside it, under a project's
+    # <git-common-dir>/fun-ci/jobs: `<name>` and `<name>.lock`. Whoever holds
+    # a job's lock runs it, so it runs once at a time, and a lock dies with
+    # the process that held it.
+    class Locks
+      def initialize(root)
+        @root = root
+      end
+
+      # The job's worktree, held through its lock as a Pipeline::Slot, or nil
+      # while another process holds it.
+      def take(name)
+        FileUtils.mkdir_p(@root)
+        path = File.join(@root, name)
+        lock = File.new("#{path}.lock", File::RDWR | File::CREAT, 0o644)
+        return Pipeline::Slot.new(path, lock, lock.path) if lock.flock(File::LOCK_EX | File::LOCK_NB)
+
+        lock.close
+        nil
+      end
+    end
+  end
+end

data/lib/fun_ci/jobs/site.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Jobs
+    # Where a project's jobs run and are recorded: the project's path, the
+    # database, its Pipeline::Worktrees (#check_out), its job Locks, and
+    # `clock`, which answers the time now.
+    Site = Data.define(:project, :db, :worktrees, :locks, :clock)
+  end
+end

data/lib/fun_ci/jobs/standings.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require_relative "folders"
+require_relative "due"
+require_relative "state"
+require_relative "../persistence/job_runs"
+
+module FunCi
+  module Jobs
+    # Where a job stands: its project, the job, its latest run as job_runs
+    # keeps it (nil while it never ran), and Due, which says whether the
+    # next commit starts it and when it is due again.
+    Standing = Data.define(:project, :job, :run, :due)
+
+    class Standing
+      def name = job.name
+      def cadence = job.cadence
+      def state = State.of(run)
+      def needs_you? = State.needs_you?(state)
+      def due_now? = due.now?
+      def due_at = due.at
+    end
+
+    # Where each of a project's jobs stands at `now`, read once for the
+    # console, the agent commands and the trigger.
+    class Standings
+      def initialize(db, project, now:)
+        @db = db
+        @project = project
+        @now = now
+      end
+
+      # By name.
+      def all
+        runs = Persistence::JobRuns.new(@db, @project)
+        Folders.new(@project).jobs.map { |job| standing(job, runs.latest(job.name)) }
+      end
+
+      private
+
+      def standing(job, run)
+        Standing.new(project: @project, job: job, run: run, due: Due.new(run, job.period, now: @now))
+      end
+    end
+  end
+end

data/lib/fun_ci/jobs/state.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Jobs
+    # A job's state, read once from its latest run as job_runs keeps it
+    # (design.md, Daily and weekly jobs): `due` while it never ran, then
+    # `running`, `passed`, `failed`, `over_budget`, `cancelled`, or `lost`
+    # when its process died before it said how the run ended. Each surface
+    # (the console, the agent commands) says these in its own words.
+    module State
+      OF_STATUS = { "completed" => "passed", "failed" => "failed", "timed_out" => "over_budget",
+                    "running" => "running", "cancelled" => "cancelled" }.freeze
+      NEEDS_YOU = %w[failed over_budget lost].freeze
+
+      # run: { status:, completed_at: }, or nil when the job never ran.
+      def self.of(run)
+        return "due" unless run
+        return "lost" if run[:status] == "failed" && run[:completed_at].nil?
+
+        OF_STATUS.fetch(run[:status], "unknown")
+      end
+
+      def self.needs_you?(state) = NEEDS_YOU.include?(state)
+    end
+  end
+end

data/lib/fun_ci/persistence/active_jobs.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require "time"
+require_relative "active_runs"
+require "json"
+require_relative "../evidence/document"
+
+module FunCi
+  module Persistence
+    # Daily and weekly jobs' runs still running, of every project, as the
+    # console sees them: to cancel one, and to notice one whose process died.
+    module ActiveJobs
+      # The job run with this id as an ActiveRun, when it still runs; none otherwise.
+      def self.with_id(db, id)
+        db.execute("SELECT id, commit_hash, pid, group_pid, lock_file FROM job_runs " \
+                   "WHERE id = ? AND status = 'running'", [id]).map do |(run_id, sha, pid, group_pid, lock_file)|
+          ActiveRun.new(id: run_id, commit_hash: sha, processes: [pid].compact, stage_groups: [group_pid].compact,
+                        slot_lock: lock_file)
+        end
+      end
+
+      # [id, lock file] of each job run still running.
+      def self.running(db) = db.execute("SELECT id, lock_file FROM job_runs WHERE status = 'running'")
+
+      # What a run whose process died keeps: that it has no result, and why
+      # that may be. It keeps no end, since when it was found dead says
+      # nothing of how long it ran.
+      DIED_FACT = { name: "ended", extractor: "fun-ci",
+                    value: "its process stopped before it said how the run ended (the machine slept or " \
+                           "restarted, or something killed it), so this run has no result" }.freeze
+      DIED = Evidence::Document.new(chosen: [], failures: [], excerpts: [], problems: [], facts: [DIED_FACT])
+
+      def self.cancelled(db, id) = ended(db, id, "cancelled")
+
+      # A run whose process is gone without recording how it ended: failed, with no end.
+      def self.died(db, id)
+        db.execute("UPDATE job_runs SET status = 'failed', evidence = ? WHERE id = ? AND status = 'running'",
+                   [JSON.generate(DIED.to_h), id])
+      end
+
+      # The ids of a project's job's runs still running.
+      def self.running_of(db, project, name)
+        db.execute("SELECT id FROM job_runs WHERE project_path = ? AND job = ? AND status = 'running'",
+                   [project, name]).flatten
+      end
+
+      def self.ended(db, id, status)
+        db.execute("UPDATE job_runs SET status = ?, completed_at = ? WHERE id = ? AND status = 'running'",
+                   [status, Time.now.utc.iso8601(3), id])
+      end
+      private_class_method :ended
+    end
+  end
+end

data/lib/fun_ci/persistence/database.rb

@@ -46,6 +46,16 @@ module FunCi
         )
       SQL
 
+      # Each run of a project's daily and weekly jobs (architecture.md, Daily and weekly jobs).
+      JOB_RUNS_TABLE = <<~SQL
+        CREATE TABLE IF NOT EXISTS job_runs (
+          id INTEGER PRIMARY KEY,
+          project_path TEXT, job TEXT, cadence TEXT, commit_hash TEXT, branch TEXT,
+          status TEXT, started_at TEXT, completed_at TEXT, lock_file TEXT, pid INTEGER, group_pid INTEGER,
+          exit_status INTEGER, signal TEXT, output_tail TEXT, evidence TEXT, budget INTEGER
+        )
+      SQL
+
       # Opening and migrating hold an exclusive lock on a file beside the
       # database, so fun-ci processes starting at once set it up one at a time.
       # SQLite's busy timeout doesn't cover the switch to WAL, and a column can
@@ -63,13 +73,13 @@ module FunCi
                        %w[pipeline_runs waited_at TEXT], %w[stage_jobs exit_status INTEGER],
                        %w[stage_jobs signal TEXT], %w[stage_jobs budget INTEGER], %w[stage_jobs pruned INTEGER],
                        %w[stage_jobs evidence TEXT], %w[pipeline_runs fetch_pgid INTEGER],
-                       %w[pipeline_runs trunk_started_at TEXT]].freeze
+                       %w[pipeline_runs trunk_started_at TEXT], %w[job_runs budget INTEGER]].freeze
+
+      TABLES = [PIPELINE_RUNS_TABLE, STAGE_JOBS_TABLE, TRUNK_CHECKS_TABLE, TRUNK_FETCHES_TABLE, JOB_RUNS_TABLE].freeze
 
       def self.migrate!(db)
         with_setup_lock(db.filename("main")) do
-          [PIPELINE_RUNS_TABLE, STAGE_JOBS_TABLE, TRUNK_CHECKS_TABLE, TRUNK_FETCHES_TABLE].each do |table|
-            db.execute(table)
-          end
+          TABLES.each { |table| db.execute(table) }
           ADDED_COLUMNS.each { |table, column, type| add_column_if_missing(db, table, column, type) }
           db.execute(NAME_DETACHED)
         end

data/lib/fun_ci/persistence/job_recorder.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "json"
+require "time"
+require_relative "raw_outputs"
+
+module FunCi
+  module Persistence
+    # Records a daily or weekly job's run as it runs, through the calls
+    # Pipeline::StageExecution makes of a stage's recorder, so a job's
+    # evidence is collected as a stage's is (architecture.md, Daily and
+    # weekly jobs). `id` is the job run's.
+    class JobRecorder
+      def initialize(db)
+        @db = db
+      end
+
+      # The process that runs the job, which a cancel stops first, and the budget it runs to.
+      def started_by(id, pid, budget: nil) = set(id, pid: pid, budget: budget)
+
+      # The job's script, which leads a process group of its own.
+      def stage_process(id, pid) = set(id, group_pid: pid)
+
+      # Unless the run was cancelled meanwhile.
+      def end_stage(id, status)
+        @db.execute("UPDATE job_runs SET status = ?, completed_at = ? WHERE id = ? AND status = 'running'",
+                    [status, Time.now.utc.iso8601(3), id])
+      end
+
+      # And the output's last lines, as a stage keeps them.
+      def keep_evidence(id, document) = set(id, evidence: JSON.generate(document.to_h), output_tail: document.tail)
+
+      def keep_exit(id, exit_status, signal) = set(id, exit_status: exit_status, signal: signal)
+      def keep_raw(id, text) = RawOutputs.for_jobs(@db.filename("main")).write(id, text)
+
+      # No stage shares a job's worktree.
+      def alongside(_id) = []
+
+      private
+
+      def set(id, **values)
+        @db.execute("UPDATE job_runs SET #{values.keys.map { |column| "#{column} = ?" }.join(", ")} WHERE id = ?",
+                    [*values.values, id])
+      end
+    end
+  end
+end

data/lib/fun_ci/persistence/job_runs.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require "time"
+require_relative "../jobs/due"
+require_relative "raw_outputs"
+require_relative "active_jobs"
+
+module FunCi
+  module Persistence
+    # The runs of one project's daily and weekly jobs (architecture.md, Daily
+    # and weekly jobs), apart from its pipeline runs.
+    class JobRuns
+      COLUMNS = %i[id project_path job cadence commit_hash branch status started_at completed_at lock_file pid
+                   group_pid exit_status signal output_tail evidence budget].freeze
+      SELECT = "SELECT #{COLUMNS.join(", ")} FROM job_runs".freeze
+      KEPT = 10
+
+      def initialize(db, project)
+        @db = db
+        @project = project
+      end
+
+      # Records a run of `job` running, testing `commit` ({ sha:, branch: }),
+      # and answers its id, when the job is due at `now`; nil otherwise. The
+      # job's latest run is read and the new one written in one transaction,
+      # so of two claims at once only one starts it. A claim forgets the
+      # job's runs beyond its newest 10.
+      def claim(job, commit:, lock_file:, now:)
+        @db.transaction(:immediate) do
+          next nil unless Jobs::Due.new(latest(job.name), job.period, now: now).now?
+
+          insert(job, commit, lock_file, now).tap { forget_beyond(job.name) }
+        end
+      end
+
+      # The project's newest job runs, newest first.
+      def recent(limit:) = query("WHERE project_path = ? ORDER BY id DESC LIMIT ?", @project, limit)
+
+      # The job's newest run, or nil when it never ran.
+      def latest(name) = query("WHERE project_path = ? AND job = ? ORDER BY id DESC LIMIT 1", @project, name).first
+
+      # Records failed the job's runs still running, whose process is known to be gone.
+      def died(name) = ActiveJobs.running_of(@db, @project, name).each { |id| ActiveJobs.died(@db, id) }
+
+      private
+
+      def insert(job, commit, lock_file, now)
+        @db.execute("INSERT INTO job_runs (project_path, job, cadence, commit_hash, branch, status, started_at, " \
+                    "lock_file) VALUES (?, ?, ?, ?, ?, 'running', ?, ?)",
+                    [@project, job.name, job.cadence, commit[:sha], commit[:branch], stamp(now), lock_file])
+        @db.last_insert_row_id
+      end
+
+      def forget_beyond(name)
+        ids = @db.execute("SELECT id FROM job_runs WHERE project_path = ? AND job = ? ORDER BY id DESC " \
+                          "LIMIT -1 OFFSET ?", [@project, name, KEPT]).flatten
+        @db.execute("DELETE FROM job_runs WHERE id IN (#{ids.map { "?" }.join(", ")})", ids) if ids.any?
+        RawOutputs.for_jobs(@db.filename("main")).delete(ids)
+      end
+
+      def stamp(time) = time.utc.iso8601(3)
+
+      def query(clause, *params)
+        @db.execute("#{SELECT} #{clause}", params).map { |row| COLUMNS.zip(row).to_h }
+      end
+    end
+  end
+end

data/lib/fun_ci/persistence/raw_outputs.rb

@@ -11,6 +11,9 @@ module FunCi
     class RawOutputs
       def self.beside(db_path) = new(File.join(File.dirname(db_path), "raw"))
 
+      # What failed daily and weekly jobs printed, apart, since their ids are their own.
+      def self.for_jobs(db_path) = new(File.join(File.dirname(db_path), "raw-jobs"))
+
       def initialize(dir)
         @dir = dir
       end