asgard 0.5.1 → 0.6.0

data/lib/asgard/doctor.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "pathname"
-
 require_relative "doctor/task_sections"
 require_relative "doctor/report"
 
@@ -41,7 +39,7 @@ module Asgard
       current = Pathname.new(dir)
       loop do
         candidate = current + ".loki"
-        markers << candidate.to_s if candidate.exist?
+        markers << candidate.to_s if candidate.file?
         parent = current.parent
         break if parent == current
 
@@ -82,9 +80,9 @@ module Asgard
       ImportTracer.doctor = self
       @loaded_files << @marker
 
-      before = Asgard::Base.subclasses.dup
-      load @marker
-      @new_subclasses = (Asgard::Base.subclasses - before + [Tasks]).uniq
+      before = Asgard::Base._asgard_subclasses.dup
+      Asgard.load_root(@marker)
+      @new_subclasses = (Asgard::Base._asgard_subclasses - before + [Tasks]).uniq
     rescue StandardError, ScriptError => e
       @findings << Finding.new(level: :error, message: "load failed: #{e.class}: #{e.message}")
     ensure
@@ -123,7 +121,7 @@ module Asgard
         loc    = caller_locations(1, 1).first
         before = $LOADED_FEATURES.dup
         result = super(path, from: loc)
-        added  = $LOADED_FEATURES - before
+        added  = ImportTracer.loki_files($LOADED_FEATURES - before)
         ImportTracer.doctor&.send(:record_import, ImportTracer.trace_message("import", path, added), added) unless
           loc&.absolute_path&.end_with?("kernel_methods.rb")
         result
@@ -132,11 +130,17 @@ module Asgard
       def import_up(name = ".loki")
         before = $LOADED_FEATURES.dup
         result = super
-        added  = $LOADED_FEATURES - before
+        added  = ImportTracer.loki_files($LOADED_FEATURES - before)
         ImportTracer.doctor&.send(:record_import, ImportTracer.trace_message("import_up", name, added), added)
         result
       end
 
+      # $LOADED_FEATURES also grows with every `require` a .loki file runs
+      # (net/http, uri, ...); only .loki paths were brought in by import.
+      def self.loki_files(added)
+        added.select { |f| f.end_with?(".loki") }
+      end
+
       def self.trace_message(verb, arg, added)
         inspected = arg.inspect
         return "#{verb} #{inspected} -- nothing new (not found or already loaded)" if added.empty?

data/lib/asgard/jobs/commands.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+require "time"
+
+# warn is silent when $VERBOSE is nil and bypasses $stderr in Ruby 4.0 (see
+# base/task_dsl.rb); these messages are for the user, so they always print.
+# rubocop:disable Style/StderrPuts
+
+module Asgard
+  module Jobs
+    # `asgard jobs SUBCOMMAND` — lists, stops and restarts the background
+    # jobs that `sh ..., background:` started. Subclasses Asgard::Base
+    # rather than Tasks so `asgard jobs help` lists only these
+    # subcommands, not every project task.
+    class Commands < Asgard::Base
+      namespace "jobs"
+      default_command :list
+
+      HEADERS  = %w[JOB TASK PID STATUS STARTED COMMAND].freeze
+      KEYS     = %i[name task pid state started command].freeze
+      FLEXIBLE = [5].freeze
+
+      class << self
+        # One `list` row per record, with what the job is doing now.
+        def rows(state) = state.records.map { |record| row(record, state) }
+
+        def row(record, state)
+          name    = record[:name]
+          exited  = state.exit_time(name)
+          status  = Jobs.status(exit_code: state.exit_code(name), verified: state.verified?(record),
+                                leftovers: State.leftovers?(record, exited_at: exited))
+          seconds = (exited - Time.iso8601(record[:started])).round(1) if status.is_a?(Array)
+          { name:, task: record[:task], pid: record[:pid], state: Jobs.describe_status(status, seconds:),
+            started: Time.iso8601(record[:started]).strftime("%Y-%m-%d %H:%M:%S"), command: record[:command] }
+        end
+
+        # One line per record file +states+ left out of the listing.
+        def skipped_lines(states) = states.flat_map(&:invalid_records).map { |path| "jobs: skipped #{path}: not a job record" }
+
+        # The argv that follows a log the way `tail -f` does, from the top.
+        def tail_argv(path) = ["tail", "-n", "+1", "-f", path]
+      end
+
+      desc "tree", "Print a tree of the jobs subcommands", hide: true
+      # Redefined only to re-register tree as hidden in this namespace.
+      def tree = super # rubocop:disable Lint/UselessMethodDefinition
+
+      no_commands do
+        def jobs_root = @jobs_root ||= Asgard.project_root
+
+        def jobs_project = @jobs_project ||= Schedule.project_slug(jobs_root)
+
+        def state = @state ||= State.new(project: jobs_project, root: jobs_root)
+
+        # True when a command should cover every project: asked for with
+        # --all, or there is no .loki here, so no single project to scope to.
+        def every_project? = options[:all] || Asgard.find_task_file.nil?
+
+        # Every project's state, keyed by "Project: slug (/root)".
+        def all_states
+          State.all.to_h do |slug, root|
+            title = ["Project: #{slug} (#{root})", ("[root missing]" unless File.directory?(root))].compact.join(" ")
+            [title, State.new(project: slug, root:)]
+          end
+        end
+
+        def record_for!(name) = state.read_record(name) || abort("jobs: no such job: #{name} (see `asgard jobs list`)")
+
+        def stopper = Stopper.new(timeout: options[:timeout] || Stopper::TIMEOUT)
+
+        # The records `stop` ends, newest start first, and the names that
+        # matched nothing. A task name reaches the jobs of every task it
+        # depends on, which needs this project's .loki loaded.
+        def stop_targets(names)
+          records, unknown =
+            if every_project? then [all_states.values.flat_map(&:records), []]
+            elsif names.empty? then [state.records, []]
+            else Jobs.resolve(names, state.records, closure: ->(task) { Jobs.closure(task, Tasks._deps) })
+            end
+          [records.sort_by { |record| [record[:started].to_s, record[:name].to_s] }.reverse, unknown]
+        end
+
+        def no_jobs_message
+          return "No jobs have been started on this machine." if every_project?
+
+          "No jobs recorded for #{jobs_project}. Start one with `sh \"...\", background: true` inside a task.\n" \
+            "Use `asgard jobs list --all` to see every project's jobs."
+        end
+      end
+
+      desc "list", "Show this project's jobs: running, exited with a status, or gone (--all: every project's)"
+      method_option :all, aliases: "-a", type: :boolean, desc: "List every project's jobs, not just this one's"
+      def list
+        states = every_project? ? all_states : { nil => state }
+        Commands.skipped_lines(states.values).each { |line| $stderr.puts line }
+        groups = states.transform_values { |each_state| Commands.rows(each_state) }.reject { |_, rows| rows.empty? }
+        return puts(no_jobs_message) if groups.empty?
+
+        puts groups.map { |title, rows| Schedule::Table.draw(rows, title:, headers: HEADERS, keys: KEYS, flexible: FLEXIBLE) }.join("\n\n")
+        puts "\nState: #{State.base_dir}"
+      end
+
+      desc "stop [NAME...]", "Stop jobs by job name or by task (and the tasks it depends on); no name stops every job of this project"
+      method_option :all, aliases: "-a", type: :boolean, desc: "Stop every project's jobs"
+      method_option :timeout, aliases: "-t", type: :numeric, desc: "Seconds to wait after TERM before KILL (default #{Stopper::TIMEOUT})"
+      def stop(*names)
+        abort "jobs: stop --all takes no job names; name the jobs without --all" if options[:all] && names.any?
+        records, unmatched = stop_targets(names)
+        idle, unknown = unmatched.partition { |name| Tasks.all_commands.key?(name) }
+        idle.each { |name| puts "#{name} has no running jobs" }
+        unknown.each { |name| $stderr.puts "jobs: no such job or task: #{name}" }
+        puts "No jobs recorded for #{jobs_project}." if records.empty? && unmatched.empty?
+        outcomes = records.map { |record| stopper.stop_and_report(record) }
+        exit 1 if unknown.any? || outcomes.include?(:killed)
+      end
+
+      desc "log NAME", "Print one job's log (stdout and stderr of its latest run; each start empties it)"
+      method_option :follow, aliases: "-f", type: :boolean, desc: "Keep printing new output as it arrives (tail -f)"
+      def log(name)
+        path = record_for!(name)[:log]
+        return $stderr.puts "#{name} has no log yet (#{path})." unless File.exist?(path)
+        return exec(*Commands.tail_argv(path)) if options[:follow]
+
+        IO.copy_stream(path, $stdout)
+      end
+
+      desc "restart NAME", "Stop the job, then run its owning task again"
+      method_option :timeout, aliases: "-t", type: :numeric, desc: "Seconds to wait after TERM before KILL (default #{Stopper::TIMEOUT})"
+      def restart(name)
+        record = record_for!(name)
+        task   = record[:task].to_s
+        abort "jobs: #{name} was not started from a task, so there is nothing to run again" if task.empty?
+        stopper.stop_and_report(record)
+        Tasks._reset_ran!
+        Tasks.start([task])
+      end
+
+      desc "clean", "Forget every job that is no longer running: its record, script, status and log"
+      # Each removal re-checks under the job's start lock (see
+      # State#forget_unless_active), so a start that ran meanwhile wins.
+      def clean
+        invalid = state.invalid_records.map { |path| [File.basename(path, ".json"), " (not a job record)"] }
+        gone    = state.records.reject { |record| state.active?(record) }.map { |record| [record[:name], ""] }
+        removed = (invalid + gone).select { |name, _| state.forget_unless_active(name) }
+        return puts("Nothing to clean for #{jobs_project}.") if removed.empty?
+
+        removed.each { |name, note| puts "removed #{name}#{note}" }
+      end
+    end
+  end
+end
+# rubocop:enable Style/StderrPuts

data/lib/asgard/jobs/declaration.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Asgard
+  # Background jobs: a process a task starts on purpose, by name, that
+  # outlives asgard and that a later `asgard jobs` can list and stop.
+  # `sh "mkdocs serve", background: true` starts one; see
+  # Jobs::Launcher for how, Jobs::State for what is kept on disk and
+  # Jobs::Commands for the management commands. This file holds the pure
+  # rules: naming, records, name resolution and status.
+  module Jobs
+    # A job name a user may choose: letters, digits and underscore, so it
+    # is safe as a file name and can never look like an auto-generated
+    # "<task>-<N>" name.
+    NAME = /\A\w+\z/
+
+    # One started job, as `sh ..., background:` returns it.
+    Job = Data.define(:name, :task, :pid, :log)
+
+    module_function
+
+    # The job name a `background:` value asks for, or nil for the
+    # foreground. +task+ is the task whose body made the call and
+    # +sequence+ its next job number, consulted only for `true`: a named
+    # job never consumes a number, so adding or removing one never
+    # renumbers the rest.
+    def job_name(background, task:, sequence: nil)
+      case background
+      when nil, false   then nil
+      when true         then sequenced_name(task, sequence)
+      when Symbol, String then chosen_name(background)
+      else raise ArgumentError, "background: must be true, a Symbol or a String, not #{background.inspect}"
+      end
+    end
+
+    # How a task name's characters outside \w are spelled in its job name,
+    # which keeps to [\w-] because a record's file name must for `jobs
+    # list` to find it. Every spelling starts with "-" and none with
+    # "-<digit>", so no two task names share a job name and none reads as
+    # the "-<N>" counter: ready? is ready-q-1, never the ready_-1 of ready_.
+    NAME_ESCAPES = { "?" => "-q", "!" => "-b", "=" => "-e", "-" => "--" }.freeze
+
+    def sequenced_name(task, sequence)
+      raise ArgumentError, "sh background: true must be called from a task body" unless task
+
+      :"#{escape_task(task)}-#{sequence}"
+    end
+
+    # Any other character (a non-ASCII letter) is "-u" and six hex digits.
+    def escape_task(task) = task.to_s.gsub(/\W/) { |char| NAME_ESCAPES.fetch(char) { format("-u%06x", char.ord) } }
+
+    def chosen_name(value)
+      name = value.to_s.strip
+      raise ArgumentError, "background: #{value.inspect} is not a job name (letters, digits and _ only)" unless name.match?(NAME)
+
+      name.to_sym
+    end
+
+    # Whether +record+ is one a started job wrote: a Hash with a name, a
+    # pid and a group id above 1 (kill(2) reads -1 and 0 as "everyone"
+    # and "us"), a script path to verify against and a start time. A hand
+    # edit or another tool's JSON fails this and stays out of listings.
+    def valid_record?(record)
+      return false unless record.is_a?(Hash)
+
+      name, pid, pgid, script, started = record.values_at(:name, :pid, :pgid, :script, :started)
+      text?(name) && text?(script) && process_id?(pid) && process_id?(pgid) && time?(started)
+    end
+
+    def text?(value) = value.is_a?(String) && !value.empty?
+
+    def process_id?(id) = id.is_a?(Integer) && id > 1
+
+    def time?(value)
+      value.is_a?(String) && Time.iso8601(value).is_a?(Time)
+    rescue ArgumentError
+      false
+    end
+
+    # The first non-blank line of a script, for the listing.
+    def summary(script) = script.to_s.each_line.map(&:strip).find { |line| !line.empty? }.to_s
+
+    # The record written to disk for a started job.
+    def record(name:, task:, pid:, pgid:, cwd:, script:, log:, command:, started:)
+      { name: name.to_s, task: task.to_s, pid:, pgid:, started:, cwd:, script:, log:, command: }
+    end
+
+    # +task+ plus every task reachable through +deps+ (a `_deps` Hash of
+    # stage lists), in discovery order. `stop servers` reaches the `db-1`
+    # a dependency started.
+    def closure(task, deps)
+      seen  = []
+      queue = [task.to_sym]
+      until queue.empty?
+        current = queue.shift
+        next if seen.include?(current)
+
+        seen << current
+        queue.concat(Array(deps[current]).flatten.map(&:to_sym))
+      end
+      seen
+    end
+
+    # The records +names+ pick out: an exact job name first, else every
+    # record whose task is the name or, through +closure+ (a callable from
+    # task name to task names), one it depends on. Returns [records,
+    # unknown names].
+    def resolve(names, records, closure: nil)
+      found   = []
+      unknown = []
+      names.each do |name|
+        matches = matching_records(name.to_s, records, closure)
+        matches.empty? ? unknown << name.to_s : found.concat(matches)
+      end
+      [found.uniq, unknown]
+    end
+
+    def matching_records(name, records, closure)
+      exact = records.select { |record| record[:name].to_s == name }
+      return exact unless exact.empty?
+
+      tasks = (closure ? closure.call(name) : [name]).map(&:to_s)
+      records.select { |record| tasks.include?(record[:task].to_s) }
+    end
+
+    # What a record's job is doing now: :running while its pid verifies,
+    # [:exited, code] once the trampoline wrote a status (with
+    # :processes_left while processes the script started remain in its
+    # group), :gone when neither (the trampoline died, or the machine
+    # rebooted).
+    def status(exit_code:, verified:, leftovers: false)
+      return [:exited, exit_code, (:processes_left if leftovers)].compact if exit_code
+      return :running if verified
+
+      :gone
+    end
+
+    # The STATUS column: "running", "exited 3 (2.5s)", "exited 0 (2.5s,
+    # processes left)" or "gone".
+    def describe_status(status, seconds: nil)
+      return status.to_s unless status.is_a?(Array)
+
+      _, code, left = status
+      notes = [(format("%.1fs", seconds) if seconds), ("processes left" if left)].compact
+      notes.empty? ? "exited #{code}" : "exited #{code} (#{notes.join(', ')})"
+    end
+  end
+end

data/lib/asgard/jobs/launcher.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "time"
+
+module Asgard
+  module Jobs
+    # Starts a job: a new session with stdio detached, running the job's
+    # shell under a POSIX /bin/sh trampoline that writes the exit status
+    # to a file once nobody is left to wait on it. Everything but `launch`
+    # is pure, and `launch` is three lines.
+    module Launcher
+      # The trampoline body. The trap is load-bearing: a group TERM reaches
+      # this shell too, and an untrapped shell would die at once, before
+      # the status is written. With a handler it finishes waiting for the
+      # child, then runs on to the echo; the child still gets its own TERM
+      # with the default disposition, since handled traps reset on exec.
+      # Never `trap ''`: an ignored signal is inherited across exec. Two
+      # commands, not one, so no shell execs the body away and vanishes.
+      TRAMPOLINE = 'trap ":" TERM INT HUP; "$@"; echo $? > "$ASGARD_JOB_STATUS"'
+
+      module_function
+
+      # /bin/sh by absolute path: never bash and never a registry entry,
+      # so the user's default_shell and the presence of bash do not
+      # matter. The status path travels in the environment, so nothing
+      # here is quoted.
+      def argv(shell_argv) = ["/bin/sh", "-c", TRAMPOLINE, "asgard-job", *shell_argv]
+
+      # The variables added to the task's environment. Python fully
+      # buffers stdout when it is a file, so a print loop would show
+      # nothing in the log until exit without PYTHONUNBUFFERED.
+      def environment(name, status_path, env = ENV)
+        added = { "ASGARD_JOB" => name.to_s, "ASGARD_JOB_STATUS" => status_path }
+        added["PYTHONUNBUFFERED"] = "1" unless env.key?("PYTHONUNBUFFERED")
+        added
+      end
+
+      # stdin from /dev/null, stdout and stderr appended to the log, so
+      # the job never takes SIGTTIN or writes into a terminal that
+      # happens to be attached.
+      # close_others: a descriptor asgard inherited (a CI runner's, a
+      # `3>file`) has no business staying open for the job's lifetime.
+      def spawn_options(log_path) = { in: File::NULL, %i[out err] => [log_path, "a"], close_others: true }
+
+      # Forks, starts a new session (pid = pgid = sid, no controlling
+      # terminal) and execs; returns the child's pid, detached. The parent
+      # waits on a pipe whose write end the child holds until exec closes
+      # it (pipes are close-on-exec), so by the time the pid is recorded it
+      # already leads its own group, and already runs the trampoline: a
+      # `stop` issued at once signals the job, never the parent's group,
+      # and a start issued at once sees the job's own command line.
+      def launch(env, argv, options)
+        reader, writer = IO.pipe
+        pid = fork do
+          reader.close
+          Process.setsid
+          exec(env, *argv, **options)
+        rescue SystemCallError => e
+          # exit!, not raise: no at_exit handler of the parent may run here.
+          $stderr.puts "asgard: could not start job: #{e.message}" # rubocop:disable Style/StderrPuts
+          $stderr.flush # exit! skips Ruby's own flush
+          exit!(127)
+        end
+        writer.close
+        reader.read
+        reader.close
+        Process.detach(pid)
+        pid
+      end
+
+      # Starts +run+ as the job +name+ owned by +task+, or returns the job
+      # already running under that name. A missing shell exits 127 up
+      # front, as foreground `sh` does; the trampoline would otherwise
+      # hide it as a status nobody is watching for. A ps that cannot
+      # verify jobs raises Pid::Unusable first: a job nothing could find
+      # again would be started twice.
+      def start(run, name:, task:, state:, silent: false, out: $stdout)
+        Pid.require_ps!
+        state.with_lock(name) do
+          existing = state.read_record(name)
+          return already_running(existing, out:) if existing && state.verified?(existing)
+
+          run.missing! unless Shell.resolve(run.shell[:path])
+          job = spawn_job(run, name:, task:, state:)
+          out.puts "started #{job.name} (pid #{job.pid}), log: #{job.log}" unless silent
+          job
+        end
+      end
+
+      def already_running(record, out:)
+        out.puts "#{record[:name]} already running (pid #{record[:pid]}, started by #{record[:task]}), log: #{record[:log]}"
+        Job.new(name: record[:name].to_sym, task: record[:task].to_sym, pid: record[:pid], log: record[:log])
+      end
+
+      def spawn_job(run, name:, task:, state:)
+        state.prepare.remove(name)
+        script = state.script_path(name, run.ext)
+        log    = state.log_path(name)
+        File.write(script, "#{run.script}\n")
+        FileUtils.touch(log) # so `jobs log` and a reader never race the child's open
+        pid = launch(environment(name, state.status_path(name)), argv(run.argv(script)), spawn_options(log))
+        state.write_record(Jobs.record(name:, task:, pid:, pgid: pid, cwd: Dir.pwd, script:, log:,
+                                       command: Jobs.summary(run.script), started: Time.now.iso8601(6)))
+        Job.new(name: name.to_sym, task: task&.to_sym, pid:, log:)
+      end
+    end
+  end
+end

data/lib/asgard/jobs/state.rb

@@ -0,0 +1,190 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require_relative "../pid"
+require_relative "../state_dir"
+
+module Asgard
+  module Jobs
+    # Where one project's jobs keep what outlives the asgard run that
+    # started them, under $XDG_STATE_HOME/asgard/jobs/<project-slug>/:
+    # per job name a JSON record, the script as handed to the shell, the
+    # log, and the exit status the trampoline writes when the job ends.
+    # Everything stays until the next start of that name or `jobs clean`,
+    # so the log of a crashed job can still be read.
+    class State
+      # A job's record: a chosen name (\w+) or a sequenced "<task>-<N>".
+      # Dotfiles never match, which is how a half-written tempfile stays out.
+      RECORD = /\A[\w-]+\.json\z/
+
+      attr_reader :dir, :root
+
+      class << self
+        def base_dir(home: Dir.home, env: ENV) = StateDir.base_dir("jobs", home:, env:)
+
+        # Every project with job state under base, as [slug, root] pairs.
+        def all(base: base_dir) = StateDir.all(base:)
+
+        # Whether a record's job is still the process we started: its pid
+        # is alive and runs the trampoline whose command line carries the
+        # script path. A reused pid after a reboot fails this, and so does
+        # one that is now asgard itself, and so does a record that is not
+        # a job's (Jobs.valid_record?). A ps that cannot tell raises
+        # Pid::Unusable rather than calling every job gone.
+        def verified?(record)
+          return false unless Jobs.valid_record?(record)
+
+          Pid.require_ps!
+          Pid.matches?(record[:pid], record[:script], self_ok: false) || survives?(record)
+        end
+
+        # Whether the job outlived its trampoline: some process in its group
+        # still runs the script. A group id is never reused while the group
+        # has members, so a match is still the job we started.
+        def survives?(record, ps: Pid.method(:group_command_lines))
+          pgid   = record[:pgid]
+          script = record[:script].to_s
+          return false unless job_group?(pgid) && !script.empty?
+
+          ps.call(pgid).any? { |command| command.include?(script) }
+        end
+
+        # Whether +pgid+ could be a job's group at all: a group id above 1
+        # that is not asgard's own. Nothing else is ever looked up or
+        # signalled.
+        def job_group?(pgid) = Jobs.process_id?(pgid) && pgid != Process.getpgrp
+
+        # Whether the job still owns processes: it is running, or its
+        # script exited and left processes in its group (`sleep 60 &`).
+        # Those are what `stop` signals and why `clean` keeps the record.
+        def active?(record) = verified?(record) || (Jobs.valid_record?(record) && leftovers?(record, exited_at: exited_at(record)))
+
+        # Whether group +pgid+ still holds a process that started before
+        # the trampoline exited at +exited_at+. The id cannot be reused
+        # while the group has members, but once they are gone a later
+        # group may take it, and every member of such a group is younger
+        # than our exit; one second of slack covers ps's resolution.
+        def leftovers?(record, exited_at:, ps: Pid.method(:group_processes))
+          pgid = record[:pgid]
+          return false unless exited_at && job_group?(pgid)
+
+          ps.call(pgid).any? { |member| member.started <= exited_at + 1 }
+        end
+
+        # The status file of a record's job, beside its script.
+        def status_file(record) = File.join(File.dirname(record[:script]), "#{record[:name]}.status")
+
+        # When the trampoline wrote the job's status, or nil.
+        def exited_at(record) = mtime_of(status_file(record))
+
+        # When +path+ was last written, or nil when it does not exist.
+        def mtime_of(path) = File.exist?(path) ? File.mtime(path) : nil
+
+        # Writes 137 (128 + KILL) as the job's status unless one is
+        # written: a group KILL takes the trampoline with it, so nothing
+        # else records how the job ended. A record that is not a job's
+        # writes nothing.
+        def record_kill(record)
+          return unless Jobs.valid_record?(record)
+
+          File.open(status_file(record), "wx", 0o600) { |file| file.puts(128 + 9) }
+        rescue Errno::EEXIST, Errno::ENOENT
+          nil
+        end
+      end
+
+      def initialize(project:, root:, base: State.base_dir)
+        @root = root
+        @dir  = File.join(base, Schedule.slug(project))
+      end
+
+      def record_path(name)      = File.join(dir, "#{name}.json")
+      def script_path(name, ext) = File.join(dir, "#{name}#{ext}")
+      def log_path(name)         = File.join(dir, "#{name}.log")
+      def status_path(name)      = File.join(dir, "#{name}.status")
+      def lock_path(name)        = File.join(dir, ".#{name}.lock")
+
+      # Runs the block holding an exclusive flock for +name+, so two starts
+      # of one name (two terminals, two parallel dependencies) take turns:
+      # the second sees the first's record and finds the job running. The
+      # lock file is a dotfile, so RECORD and `remove` never see it.
+      def with_lock(name)
+        FileUtils.mkdir_p(dir)
+        File.open(lock_path(name), File::RDWR | File::CREAT, 0o600) do |file|
+          file.flock(File::LOCK_EX)
+          yield
+        end
+      end
+
+      # Creates the directory and records the root.
+      def prepare
+        StateDir.record_root(dir, root)
+        self
+      end
+
+      def write_record(record) = StateDir.write_json(record_path(record[:name]), record)
+
+      # The record for +name+, or nil when there is none or it is not a
+      # job's record (Jobs.valid_record?).
+      def read_record(name) = StateDir.read_json(record_path(name)).then { Jobs.valid_record?(it) ? it : nil }
+
+      # Every job's record, oldest start first. Only "<name>.json" counts,
+      # never the dot-prefixed tempfile write_json is still filling, and a
+      # file that is not a job's record is left out (see invalid_records).
+      def records
+        record_files.filter_map { |path| StateDir.read_json(path) }.select { |record| Jobs.valid_record?(record) }
+                    .sort_by { |record| [record[:started], record[:name]] }
+      end
+
+      # The record files that are not a job's record: unparsable JSON, a
+      # hand edit, another tool's file.
+      def invalid_records = record_files.reject { |path| Jobs.valid_record?(StateDir.read_json(path)) }
+
+      def record_files
+        Dir.children(dir).grep(RECORD).sort.map { |file| File.join(dir, file) }
+      rescue Errno::ENOENT
+        []
+      end
+
+      def verified?(record) = State.verified?(record)
+
+      def active?(record) = State.active?(record)
+
+      # Forgets +name+ (record, script, status and log) unless its job
+      # still owns processes; true when it did. Holds the start lock and
+      # reads the record again inside it, so a start that ran meanwhile
+      # keeps the fresh record it wrote. A record that is not a job's is
+      # forgotten too: it cannot be running.
+      def forget_unless_active(name)
+        with_lock(name) do
+          next false unless File.exist?(record_path(name))
+
+          record = read_record(name)
+          next false if record && active?(record)
+
+          remove(name, keep_log: false)
+          true
+        end
+      end
+
+      # The exit code the trampoline wrote, or nil while none is written.
+      def exit_code(name)
+        path = status_path(name)
+        Integer(File.read(path).strip, exception: false) if File.exist?(path)
+      end
+
+      # When the status was written, or nil.
+      def exit_time(name) = State.mtime_of(status_path(name))
+
+      # Forgets a job: its record, script and status go; the log is kept
+      # but emptied, so a restart appends to a fresh file, unless
+      # keep_log: false deletes it too (`jobs clean`).
+      def remove(name, keep_log: true)
+        log   = log_path(name)
+        files = Dir.exist?(dir) ? Dir.children(dir).select { |file| file.start_with?("#{name}.") }.map { |file| File.join(dir, file) } : []
+        FileUtils.rm_f(keep_log ? files - [log] : files)
+        File.truncate(log, 0) if keep_log && File.exist?(log)
+      end
+    end
+  end
+end