ruby_raider 4.1.0 → 4.2.0

data/lib/utilities/schedules/run_lock.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'securerandom'
+require_relative 'base'
+require_relative 'kind'
+require_relative 'run_log'
+
+module RubyRaider
+  module Schedules
+    # Who is running the project's tests now, decided before rspec spawns (a busy run must never wipe
+    # raider-results/):
+    # - .raider/schedules/run.lock, held with flock among `raider schedule run`s; never removed;
+    # - .raider/schedules/running.json {pid, name, run_id, started_at}, which the desktop is to write for its
+    #   own runs too. A live pid there that is not ours means busy. It is read and written only as a regular file
+    #   (O_NOFOLLOW), made with O_EXCL, and its pid is only checked, never signalled.
+    class RunLock
+      LOCK = 'run.lock'
+      RUNNING = 'running.json'
+      MAX_RUNNING_BYTES = 4096
+
+      def initialize(project)
+        @project = project
+        @lock = nil
+        @run_id = nil
+      end
+
+      # true with the lock held and running.json written; false when another run is going
+      def acquire(name:, run_id:, started_at:)
+        dir = RunLog.private_dir(@project, Kind::TESTS.dir)
+        return false unless lock(File.join(dir, LOCK))
+
+        running = File.join(dir, RUNNING)
+        seen = inspect_running(running)
+        return busy if seen && (seen[:changed] || live?(seen[:data]))
+        return busy if seen && !remove_stale(running, seen[:id])
+
+        write_running(running, name, run_id, started_at) || busy
+      end
+
+      # Removes running.json when it is still ours, then frees the lock
+      def release
+        running = @project.path("#{Kind::TESTS.dir}/#{RUNNING}")
+        File.unlink(running) if @run_id && read_running(running)&.fetch('run_id', nil) == @run_id
+      rescue SystemCallError
+        nil
+      ensure
+        @run_id = nil
+        @lock&.close
+        @lock = nil
+      end
+
+      private
+
+      def lock(path)
+        file = File.open(path, File::RDWR | File::CREAT | RunLog::NOFOLLOW | RunLog::NONBLOCK, 0o600)
+        @lock = file
+        raise Error, "#{Kind::TESTS.dir}/#{LOCK} must be a file" unless file.stat.file?
+
+        file.flock(File::LOCK_EX | File::LOCK_NB) ? true : busy
+      rescue Errno::ELOOP, Errno::EMLINK
+        raise Error, "#{Kind::TESTS.dir}/#{LOCK} must be a file, not a link"
+      rescue Error
+        busy
+        raise
+      end
+
+      def busy
+        @lock&.close
+        @lock = nil
+        false
+      end
+
+      # What is at running.json now: its identity (dev, ino, never through a link), its object when it is a regular
+      # JSON file, and changed when it was replaced between the lstat and the read. nil when there is none.
+      def inspect_running(running)
+        stat = File.lstat(running)
+        id = [stat.dev, stat.ino]
+        data = nil
+        changed = false
+        if stat.file?
+          File.open(running, File::RDONLY | RunLog::NOFOLLOW | RunLog::NONBLOCK) do |file|
+            opened = file.stat
+            changed = [opened.dev, opened.ino] != id
+            data = parse_running(file) unless changed
+          end
+        end
+        { id: id, data: data, changed: changed }
+      rescue Errno::ENOENT
+        nil
+      rescue SystemCallError
+        { id: nil, data: nil, changed: true }
+      end
+
+      # Moves the stale file aside and removes it only when it is still the one inspected. The desktop takes no
+      # lock, so a running.json it wrote in between is put back (link never replaces a newer one) and counts as busy.
+      def remove_stale(running, id)
+        aside = "#{running}.#{SecureRandom.hex(6)}.stale"
+        File.rename(running, aside)
+        stat = File.lstat(aside)
+        return File.unlink(aside).positive? if [stat.dev, stat.ino] == id
+
+        restore(aside, running)
+        false
+      rescue Errno::ENOENT
+        true
+      end
+
+      def restore(aside, running)
+        File.link(aside, running)
+      rescue SystemCallError
+        nil
+      ensure
+        File.unlink(aside)
+      end
+
+      # Another process named by running.json is alive. kill(0) only checks: EPERM means alive but not ours to signal.
+      def live?(data)
+        pid = data&.fetch('pid', nil)
+        return false unless pid.is_a?(Integer) && pid.positive? && pid != Process.pid
+
+        Process.kill(0, pid)
+        true
+      rescue Errno::EPERM
+        true
+      rescue SystemCallError
+        false
+      end
+
+      # Its JSON object, nil when it is missing, a link, not a regular file, too big or not an object
+      def read_running(running)
+        File.open(running, File::RDONLY | RunLog::NOFOLLOW | RunLog::NONBLOCK) { |file| parse_running(file) }
+      rescue SystemCallError
+        nil
+      end
+
+      # The object in an open running.json, nil when it is not a small regular file holding one
+      def parse_running(file)
+        stat = file.stat
+        return nil unless stat.file? && stat.size <= MAX_RUNNING_BYTES
+
+        data = JSON.parse(file.read.to_s)
+        data.is_a?(Hash) ? data : nil
+      rescue JSON::ParserError
+        nil
+      end
+
+      # false when another writer made the file first
+      def write_running(running, name, run_id, started_at)
+        File.open(running, File::WRONLY | File::CREAT | File::EXCL | RunLog::NOFOLLOW, 0o600) do |file|
+          file.write(JSON.generate('pid' => Process.pid, 'name' => name, 'run_id' => run_id, 'started_at' => started_at))
+        end
+        @run_id = run_id
+        true
+      rescue Errno::EEXIST
+        false
+      end
+    end
+  end
+end

data/lib/utilities/schedules/run_log.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'json'
+require 'securerandom'
+require_relative 'base'
+require_relative 'kind'
+
+module RubyRaider
+  module Schedules
+    # <kind dir>/<name>/runs.jsonl: one JSON line per run, oldest first. `raider schedule run` appends to it
+    # (status and counts only, never a message).
+    # :reek:TooManyConstants { max_constants: 7 } -- the read cap, the line cap, the files and the open flags
+    class RunLog
+      MAX_BYTES = 1024 * 1024
+      MAX_LINES = 50
+      FILE = 'runs.jsonl'
+      # A busy run appends without the run lock: appends and the trim at MAX_LINES take this one. It is never
+      # renamed (the trim renames a new runs.jsonl into place), so a lock on it keeps excluding.
+      LOCK = 'runs.jsonl.lock'
+      NOFOLLOW = defined?(File::NOFOLLOW) ? File::NOFOLLOW : 0
+      NONBLOCK = defined?(File::NONBLOCK) ? File::NONBLOCK : 0
+
+      # A 0700 folder <dir>/<name> inside the project; no part of it may be a link. Returns its path.
+      def self.private_dir(project, *parts)
+        path = project.root
+        parts.flat_map { |part| part.split('/') }.each do |part|
+          path = File.join(path, part)
+          FileUtils.mkdir_p(path)
+          raise Error, "#{path.delete_prefix("#{project.root}/")} must be a folder, not a link" unless File.lstat(path).directory?
+        end
+        File.chmod(0o700, path)
+        path
+      end
+
+      def initialize(project, name, dir: Kind::TESTS.dir)
+        @project = project
+        @parts = [dir, name]
+        @file = project.path("#{dir}/#{name}/#{FILE}")
+        @lock = project.path("#{dir}/#{name}/#{LOCK}")
+      end
+
+      # Every readable line in the last MAX_BYTES, oldest first; a broken line is skipped
+      def lines
+        tail.lines.filter_map do |line|
+          parsed = JSON.parse(line)
+          parsed if parsed.is_a?(Hash)
+        rescue JSON::ParserError
+          nil
+        end
+      end
+
+      def last = lines.last
+
+      # The last run that ran: a busy line only says another run held the lock
+      def last_result = lines.reverse.find { |line| line['status'] != 'busy' }
+
+      def find(run_id) = lines.reverse.find { |line| line['run_id'] == run_id }
+
+      # Adds one line, keeping the last MAX_LINES. An append while there is room; otherwise the kept lines are
+      # written to a temp file that replaces the log, so a reader never sees half of it. Never through a link.
+      def append(line)
+        RunLog.private_dir(@project, *@parts)
+        locked do
+          # A line cut short by a crash is ended first, so the new line stays whole
+          raw = tail
+          text = "#{JSON.generate(line)}\n"
+          text = "\n#{text}" unless raw.empty? || raw.end_with?("\n")
+          old = raw.lines
+          old.size < MAX_LINES ? add(text) : replace(old.last(MAX_LINES - 1).join + text)
+        end
+      end
+
+      private
+
+      # Holds runs.jsonl.lock (a regular file, never through a link) while the block reads and writes the log
+      def locked
+        File.open(@lock, File::RDWR | File::CREAT | NOFOLLOW | NONBLOCK, 0o600) do |file|
+          raise Error, "#{LOCK} must be a file" unless file.stat.file?
+
+          file.flock(File::LOCK_EX)
+          yield
+        end
+      rescue Errno::ELOOP, Errno::EMLINK
+        raise Error, "#{LOCK} must be a file, not a link"
+      end
+
+      def add(text)
+        File.open(@file, File::WRONLY | File::APPEND | File::CREAT | NOFOLLOW | NONBLOCK, 0o600) do |file|
+          raise Error, "#{FILE} must be a file" unless file.stat.file?
+
+          file.write(text)
+        end
+      rescue Errno::ELOOP, Errno::EMLINK
+        raise Error, "#{FILE} must be a file, not a link"
+      end
+
+      def replace(text)
+        temp = "#{@file}.#{SecureRandom.hex(6)}.tmp"
+        File.open(temp, File::WRONLY | File::CREAT | File::EXCL | NOFOLLOW, 0o600) { |file| file.write(text) }
+        File.rename(temp, @file)
+      ensure
+        FileUtils.rm_f(temp) if temp && File.exist?(temp)
+      end
+
+      # Read through one handle that is not a link, so the checks and the read see the same file
+      def tail
+        File.open(@file, File::RDONLY | NONBLOCK | NOFOLLOW) do |file|
+          stat = file.stat
+          return '' unless stat.file?
+
+          skip = [stat.size - MAX_BYTES, 0].max
+          file.seek(skip)
+          text = file.read.to_s.force_encoding(Encoding::UTF_8)
+          skip.positive? ? text.partition("\n").last : text
+        end
+      rescue SystemCallError
+        ''
+      end
+    end
+  end
+end

data/lib/utilities/schedules/run_report.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'json'
+require 'securerandom'
+require 'time'
+require_relative 'base'
+require_relative 'run_log'
+
+module RubyRaider
+  module Schedules
+    # raider-results/run.json of the run that just ended, as raider_reporter wrote it when rspec finished.
+    # A run that died before that leaves the previous run's file, so it counts only when its run started at
+    # (or up to SLACK before) the spawn; without a start time, when the file is not older than the spawn.
+    # raider-results/ is only read, never changed, and nothing in it is read through a link.
+    # :reek:TooManyConstants { max_constants: 9 } -- the folder, its files, the caps and the slack
+    class RunReport
+      DIR = 'raider-results'
+      RUN = 'run.json'
+      INDEX = 'index.html'
+      ATTACHMENTS = 'attachments'
+      SLACK = 2
+      MAX_RUN_BYTES = 256 * 1024 * 1024
+      MAX_ATTACHMENT_BYTES = 50 * 1024 * 1024
+      # Past these the attachments are left out of the copy (run.json and index.html still go)
+      MAX_ATTACHMENT_DEPTH = 8
+      MAX_ATTACHMENT_FILES = 1000
+
+      attr_reader :counts
+
+      # The report of the run spawned at since, nil when there is none
+      def self.load(project, since:)
+        dir = project.path(DIR)
+        return nil unless File.lstat(dir).directory?
+
+        bytes, mtime = read_file(File.join(dir, RUN), MAX_RUN_BYTES)
+        data = bytes && JSON.parse(bytes)
+        return nil unless data.is_a?(Hash) && data['results'].is_a?(Array) && fresh?(data['run'], mtime, since)
+
+        new(dir, bytes, data['results'])
+      rescue SystemCallError, JSON::ParserError
+        nil
+      end
+
+      # [content, mtime] of a regular file opened without following a link, nil when it is not one or too big
+      def self.read_file(path, max)
+        File.open(path, File::RDONLY | RunLog::NOFOLLOW | RunLog::NONBLOCK) do |file|
+          stat = file.stat
+          stat.file? && stat.size <= max ? [file.read.to_s, stat.mtime] : nil
+        end
+      rescue SystemCallError
+        nil
+      end
+
+      def self.fresh?(run, mtime, since)
+        started = run.is_a?(Hash) ? run['started_at'] : nil
+        return mtime >= since - SLACK unless started.is_a?(String)
+
+        Time.iso8601(started) >= since - SLACK
+      rescue ArgumentError
+        false
+      end
+
+      def initialize(dir, run_json, results)
+        @dir = dir
+        @run_json = run_json
+        statuses = results.map { |result| result.is_a?(Hash) ? result['status'] : nil }
+        @counts = { 'total' => statuses.size, 'passed' => statuses.count('passed'),
+                    'failed' => statuses.count { |status| %w[failed quarantined].include?(status) },
+                    'skipped' => statuses.count { |status| %w[pending skipped].include?(status) } }.freeze
+      end
+
+      # Copies run.json, index.html and (when they come to at most MAX_ATTACHMENT_BYTES) the attachments to
+      # target, replacing what was there: built in a temp folder beside it, then renamed into place.
+      def copy_to(target)
+        temp = "#{target}.#{SecureRandom.hex(6)}.tmp"
+        Dir.mkdir(temp, 0o700)
+        File.binwrite(File.join(temp, RUN), @run_json)
+        index = RunReport.read_file(File.join(@dir, INDEX), MAX_RUN_BYTES)
+        File.binwrite(File.join(temp, INDEX), index.first) if index
+        copy_attachments(temp)
+        swap(temp, target)
+        true
+      ensure
+        FileUtils.rm_rf(temp) if temp && File.exist?(temp)
+      end
+
+      private
+
+      def copy_attachments(temp)
+        files = regular_files(File.join(@dir, ATTACHMENTS))
+        return unless files && files.sum { |_, size| size } <= MAX_ATTACHMENT_BYTES
+
+        FileUtils.mkdir_p(File.join(temp, ATTACHMENTS))
+        files.each { |relative, size| copy_file(File.join(@dir, ATTACHMENTS, relative), File.join(temp, ATTACHMENTS, relative), size) }
+      end
+
+      # Streams one file that is still a regular file when opened (not a link), at most the size counted for
+      # it, so a file that grew since does not take the copy past the cap; anything else is skipped
+      def copy_file(source, target, size)
+        File.open(source, File::RDONLY | RunLog::NOFOLLOW | RunLog::NONBLOCK) do |from|
+          next unless from.stat.file?
+
+          FileUtils.mkdir_p(File.dirname(target))
+          File.open(target, File::WRONLY | File::CREAT | File::EXCL | RunLog::NOFOLLOW, 0o600) { |to| IO.copy_stream(from, to, size) }
+        end
+      rescue Errno::ELOOP, Errno::EMLINK, Errno::ENOENT
+        nil
+      end
+
+      # {relative path => size} of the regular files under dir (links and anything else skipped); nil once
+      # they pass a cap (bytes, files or folder depth) or when dir is not a folder
+      def regular_files(dir, prefix = nil, found = {}, depth = 0)
+        return nil if depth > MAX_ATTACHMENT_DEPTH || !File.lstat(dir).directory?
+
+        Dir.children(dir).sort.each do |child|
+          relative = prefix ? File.join(prefix, child) : child
+          stat = File.lstat(File.join(dir, child))
+          if stat.directory? then return nil unless regular_files(File.join(dir, child), relative, found, depth + 1)
+          elsif stat.file? then found[relative] = stat.size
+          end
+          return nil if found.size > MAX_ATTACHMENT_FILES || found.sum { |_, size| size } > MAX_ATTACHMENT_BYTES
+        end
+        found
+      rescue SystemCallError
+        nil
+      end
+
+      # The old copy moves aside first, so target is never half old and half new
+      def swap(temp, target)
+        old = "#{target}.#{SecureRandom.hex(6)}.old"
+        File.rename(target, old) if File.symlink?(target) || File.exist?(target)
+        File.rename(temp, target)
+      ensure
+        FileUtils.rm_rf(old) if old && (File.symlink?(old) || File.exist?(old))
+      end
+    end
+  end
+end

data/lib/utilities/schedules/run_wrapper.rb

@@ -0,0 +1,216 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+require 'time'
+require_relative 'base'
+require_relative 'project'
+require_relative 'schedule_store'
+require_relative 'run_log'
+require_relative 'run_lock'
+require_relative 'run_report'
+require_relative 'notifier'
+require_relative 'spawn'
+
+module RubyRaider
+  module Schedules
+    # `raider schedule run NAME`: what the OS entry runs, and what a person can run by hand. It owns the whole
+    # run record (spec §4): it re-validates the entry, takes the lock before rspec spawns (busy never wipes
+    # raider-results/), runs `bundle exec rspec` of the selection with retries and headless set through the
+    # env, counts raider-results/run.json, appends the run line, copies the report, cleans up and notifies.
+    # Exit 0 passed, 1 failed, 2 setup_error (or refused), 3 busy.
+    # :reek:TooManyInstanceVariables { max_instance_variables: 7 } -- the project and the injected collaborators
+    # :reek:TooManyConstants { max_constants: 6 } -- the exit codes, triggers, results folder, log cap and wording
+    class RunWrapper
+      EXIT_CODES = { 'passed' => 0, 'failed' => 1, 'setup_error' => 2, 'busy' => 3 }.freeze
+      TRIGGERS = %w[manual schedule].freeze
+      LOG_BYTES = 1024 * 1024
+      MISSING = 'remove the schedule or add it again'
+      ZERO = { 'total' => 0, 'passed' => 0, 'failed' => 0, 'skipped' => 0 }.freeze
+      # Forward a wrapper INT only when it can't have come from a terminal Ctrl-C, which rspec gets too
+      SPAWNER = ->(env, argv, chdir) { Spawn.call(env, argv, chdir, forward_int: !$stdin.tty?) }
+
+      # spawner: (env, argv, chdir) -> [exit status or nil, stopped]; notifier: nil makes one only when needed
+      def initialize(project, out: $stdout, err: $stderr, spawner: SPAWNER, clock: -> { Time.now }, notifier: nil)
+        @project = project
+        @out = out
+        @err = err
+        @spawner = spawner
+        @clock = clock
+        @notifier = notifier
+        @lock = RunLock.new(project)
+      end
+
+      # The exit code. Raises Error (exit 2, no run line) for an unknown trigger or a schedule that is not in
+      # schedules.yml or no longer valid there.
+      def run(name, trigger: 'manual')
+        raise Error, "Invalid trigger #{Schedules.printable(trigger.inspect)}: use #{TRIGGERS.join(' or ')}" unless TRIGGERS.include?(trigger)
+
+        schedule = load(name)
+        rotate_log(schedule) if trigger == 'schedule'
+        record = { 'run_id' => SecureRandom.hex(8), 'started_at' => stamp, 'trigger' => trigger }
+        outcome = attempt(schedule, record)
+        finish(schedule, record, outcome)
+      end
+
+      private
+
+      def load(name)
+        schedule = ScheduleStore.new(@project).find(name) if Schedules.valid_name?(name)
+        return schedule if schedule
+
+        raise Error, "No test schedule #{Schedules.printable(name.inspect)} in #{ScheduleStore::DIR}/#{Store::FILE} (or it is not valid): " \
+                     'see raider schedule list'
+      end
+
+      # {status, counts, report?, reason?, stopped?}; the lock is held from before the spawn to the run line
+      def attempt(schedule, record)
+        outcome = {}
+        schedule.selection.check!(@project.root)
+        return { status: 'busy' } unless @lock.acquire(name: schedule.name, run_id: record['run_id'], started_at: record['started_at'])
+
+        outcome = spawn(schedule)
+        outcome[:copied] = copy_report(schedule, outcome[:report])
+        # A line that can't be written loses only the line: the stop, counts and exit code stay
+        append_quietly(schedule, record, outcome)
+        outcome[:recorded] = true
+        outcome
+      rescue Error, SystemCallError => e
+        { status: 'setup_error', reason: Schedules.printable(e.message) }
+      rescue SignalException
+        stopped(schedule, record, outcome)
+      ensure
+        @lock.release
+      end
+
+      # INT or TERM outside the spawn's traps (the check, the lock, the report copy) is a stop: finish records it
+      # and the ensure frees the lock and running.json. A run whose line is written already keeps that outcome.
+      def stopped(schedule, record, outcome)
+        return outcome.merge(recorded: true) if outcome[:recorded] || RunLog.new(@project, schedule.name).find(record['run_id'])
+
+        outcome.merge(status: 'setup_error', stopped: true)
+      end
+
+      def spawn(schedule)
+        since = @clock.call
+        code, stopped = @spawner.call(env(schedule), argv(schedule), @project.root)
+        report = RunReport.load(@project, since: since)
+        { status: status(code, stopped, report&.counts), report: report, stopped: stopped }
+      end
+
+      # A stop wins over everything: an interrupted rspec exits 1 even when what it ran passed
+      def status(code, stopped, counts)
+        return 'setup_error' if stopped || counts.nil?
+        return 'passed' if code&.zero? && counts['total'].positive?
+        return 'failed' if code == 1 && counts['failed'].positive?
+
+        'setup_error'
+      end
+
+      def argv(schedule)
+        selection = schedule.selection
+        paths = case selection.kind
+                when 'tag' then ['--tag', selection.tag.to_s]
+                when 'files' then selection.files
+                else []
+                end
+        [schedule.ruby, schedule.bundle, 'exec', 'rspec', *paths]
+      end
+
+      # Spec §4a: the inherited values are dropped first (nil unsets), then retries and headless are set.
+      # config/config.yml is never written.
+      def env(schedule)
+        show = schedule.show_browser
+        gem_env(schedule).merge('BUNDLE_GEMFILE' => @project.path('Gemfile'), 'RETRY_COUNT' => (schedule.retries + 1).to_s,
+                                'RSPEC_RETRY_RETRY_COUNT' => nil, 'HEADLESS' => show ? nil : 'true', 'RAIDER_SHOW_BROWSER' => show ? '1' : nil)
+      end
+
+      # The GEM_HOME and GEM_PATH captured at add, as the OS entry sets them (a key not captured is unset), so
+      # a run by hand from another Ruby uses the scheduled run's gems. Nothing captured: the caller's stay.
+      def gem_env(schedule)
+        captured = schedule.gem_env
+        captured.empty? ? {} : Rule::GEM_ENV.to_h { |key| [key, captured[key]] }
+      end
+
+      # The report's folder relative to the project, nil when there was no report of this run or it could not be copied
+      def copy_report(schedule, report)
+        return nil unless report
+
+        relative = "#{Kind::TESTS.dir}/#{schedule.name}/last"
+        RunLog.private_dir(@project, Kind::TESTS.dir, schedule.name)
+        report.copy_to(@project.path(relative))
+        relative
+      rescue Error, SystemCallError => e
+        @err.puts("Could not copy the report to #{relative}: #{Schedules.printable(e.message)}")
+        nil
+      end
+
+      def append(schedule, record, outcome)
+        line = { 'run_id' => record['run_id'], 'started_at' => record['started_at'], 'finished_at' => stamp, 'status' => outcome[:status],
+                 'counts' => outcome[:report]&.counts || ZERO, 'trigger' => record['trigger'], 'results_dir' => RunReport::DIR }
+        line['report_copy'] = outcome[:copied] if outcome[:copied]
+        RunLog.new(@project, schedule.name).append(line)
+      end
+
+      # Busy and early setup errors get their line here, after the lock is gone; the rest have one already
+      def finish(schedule, record, outcome)
+        status = outcome[:status]
+        append_quietly(schedule, record, outcome) unless outcome[:recorded]
+        @out.puts("Schedule #{Schedules.printable(schedule.name)} #{result_text(schedule, record, outcome)}")
+        notify(schedule, record, outcome) if schedule.notify && !outcome[:stopped] && %w[failed setup_error].include?(status)
+        EXIT_CODES.fetch(status)
+      end
+
+      def append_quietly(schedule, record, outcome)
+        append(schedule, record, outcome)
+      rescue Error, SystemCallError => e
+        @err.puts("Could not record the run: #{Schedules.printable(e.message)}")
+      end
+
+      def result_text(schedule, record, outcome)
+        counts = outcome[:report]&.counts || ZERO
+        return 'was stopped' if outcome[:stopped]
+
+        case outcome[:status]
+        when 'passed' then "passed: #{tests(counts['total'])}"
+        when 'failed' then "failed: #{failures(counts)}"
+        when 'busy' then 'is already running'
+        else "could not start: #{start_reason(schedule, record, outcome, long: true)}"
+        end
+      end
+
+      def failures(counts) = "#{counts['failed']} of #{tests(counts['total'])} failed"
+
+      def tests(total) = total == 1 ? '1 test' : "#{total} tests"
+
+      # The notifier never raises, so a notification can't change the exit code
+      def notify(schedule, record, outcome)
+        counts = outcome[:report]&.counts || ZERO
+        body = outcome[:status] == 'failed' ? failures(counts) : "Could not start: #{start_reason(schedule, record, outcome, long: false)}"
+        (@notifier ||= Notifier.new).notify("Ruby Raider: #{schedule.name} failed", body)
+      end
+
+      # A refusal names itself ("spec/x_spec.rb not found"); a run that started points at its output. Only a
+      # scheduled run outside Windows has a scheduler.log: Windows tasks keep none.
+      def start_reason(schedule, record, outcome, long:)
+        reason = outcome[:reason]
+        return long && reason.end_with?(' not found') ? "#{reason}: #{MISSING}" : reason if reason
+        return "see #{Kind::TESTS.dir}/#{schedule.name}/scheduler.log" if record['trigger'] == 'schedule' && !Gem.win_platform?
+
+        'see the run output'
+      end
+
+      # launchd and cron append to scheduler.log through a handle opened before this process started. Renaming
+      # the file keeps that handle (POSIX), so the rest of this run's output lands in scheduler.log.1 and the
+      # next run starts a new scheduler.log. A link or anything but a regular file is left alone.
+      def rotate_log(schedule)
+        log = @project.path("#{Kind::TESTS.dir}/#{schedule.name}/scheduler.log")
+        stat = File.lstat(log)
+        File.rename(log, "#{log}.1") if stat.file? && stat.size > LOG_BYTES
+      rescue SystemCallError
+        nil
+      end
+
+      def stamp = @clock.call.getutc.iso8601(3)
+    end
+  end
+end