fun_ci 1.2.0 → 2.0.0

data/lib/fun_ci/pipeline/stage_end.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require_relative "../evidence/outcome"
+require_relative "../evidence/document"
+
+module FunCi
+  module Pipeline
+    # Records a stage that has finished, whichever process ran it: a failed
+    # stage's evidence first, so whoever sees the outcome can read why, then
+    # how it exited, then the outcome. Called while the stage still holds its
+    # slot and stage directory, which the collector reads.
+    class StageEnd
+      # What running a stage answered: its output, its Process::Status (nil
+      # when it was killed over budget), and for an overrun, what was found
+      # before the kill.
+      Finished = Data.define(:output, :status, :timed_out, :overrun)
+
+      class Finished
+        def initialize(overrun: nil, **) = super
+      end
+
+      # A failed stage's state in the words `status --json` uses.
+      STATES = { "failed" => "failed", "timed_out" => "over_budget" }.freeze
+
+      def initialize(recorder, job_id, collector)
+        @recorder = recorder
+        @job_id = job_id
+        @collector = collector
+      end
+
+      # Answers the outcome recorded: completed, failed or timed_out.
+      def record(finished)
+        result = outcome(finished)
+        keep_evidence(finished) unless result == "completed"
+        keep_exit(finished.status) if finished.status
+        @recorder.end_stage(@job_id, result)
+        result
+      end
+
+      private
+
+      def outcome(finished)
+        return "timed_out" if finished.timed_out
+
+        finished.status.success? ? "completed" : "failed"
+      end
+
+      # Whatever goes wrong collecting the evidence is kept as a problem, and
+      # the outcome is recorded all the same.
+      def keep_evidence(finished)
+        @recorder.keep_evidence(@job_id, @collector.collect(finished.output, ending(finished)))
+        @recorder.keep_raw(@job_id, @collector.masked(finished.output))
+      rescue StandardError => e
+        @recorder.keep_evidence(@job_id, Evidence::Document.broken("#{e.class}: #{e.message}"))
+      end
+
+      def ending(finished)
+        status = finished.status
+        Evidence::Outcome.new(state: STATES.fetch(outcome(finished)), exit_status: status&.exitstatus,
+                              signal: status&.termsig && Signal.signame(status.termsig),
+                              alongside: @recorder.alongside(@job_id), overrun: finished.overrun)
+      end
+
+      def keep_exit(status)
+        signal = status.termsig && Signal.signame(status.termsig)
+        @recorder.keep_exit(@job_id, status.exitstatus, signal)
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/stage_execution.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require_relative "stage_end"
+require_relative "../evidence/start"
+
+module FunCi
+  module Pipeline
+    # Runs a stage that has been started, in `dir`, with a stage directory of
+    # its own, and records how it ended while it still holds both. The same
+    # for every stage, the slow suite in its forked child included.
+    class StageExecution
+      def initialize(seams:, dir:, commit:)
+        @seams = seams
+        @dir = dir
+        @commit = commit
+      end
+
+      # Answers [the outcome recorded, what the stage printed].
+      def run(stage, command, recorder, job_id)
+        with_stage_dir do |stage_dir|
+          collector = collector(stage, stage_dir)
+          finished = finished(stage, command, stage_dir, collector) { |pid| recorder.stage_process(job_id, pid) }
+          [StageEnd.new(recorder, job_id, collector).record(finished), finished.output]
+        end
+      end
+
+      private
+
+      # A stage over budget is looked at before the kill (acceptance-tests.md, AT-10.17).
+      def finished(stage, command, stage_dir, collector, &)
+        overrun = nil
+        output, status, timed_out = execute(stage, command, stage_dir, lambda { |pgid|
+          overrun = collector.before_kill(pgid)
+        },
+                                            &)
+        StageEnd::Finished.new(output: output, status: status, timed_out: timed_out, overrun: overrun)
+      end
+
+      # The stage learns its name from FUN_CI_STAGE (acceptance-tests.md, AT-10.7).
+      def execute(stage, command, stage_dir, before_kill, &)
+        window = stage_dir.window
+        launch = ProcessRunner::Launch.new(env: { "FUN_CI_STAGE" => stage }, output: window,
+                                           before_kill: before_kill)
+        @seams.executor(@dir).call(command, @seams.budgets[stage], launch, &)
+      ensure
+        window&.close
+      end
+
+      def with_stage_dir
+        stage_dir = @seams.stage_dir.call
+        yield stage_dir
+      ensure
+        stage_dir&.remove
+      end
+
+      # Made before the stage runs, which is when its watched files are stamped.
+      def collector(stage, stage_dir)
+        sources = Evidence::Sources.new(stage: stage, worktree: @dir, stage_dir: stage_dir,
+                                        environment: @seams.environment,
+                                        budget: @seams.budgets[stage], commit: @commit.to_h, started: Time.now,
+                                        processes: @seams.process_table)
+        commands = @seams.extractor_runner.call(dir: @dir, scratch: stage_dir.scratch)
+        Evidence::Start.collector(sources, @seams.clock, commands)
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/stage_runner.rb

@@ -1,85 +1,63 @@
 # frozen_string_literal: true
 
-require "timeout"
-require_relative "process_runner"
+require_relative "trigger_params"
+require_relative "stage_execution"
 
 module FunCi
   module Pipeline
     class StageRunner
-      include ProcessRunner
-
-      def initialize(commit_hash:, stdout:, command_runner: nil, time_budgets: {}, recorder: FunCi::NullRecorder.new)
-        @commit_hash = commit_hash
+      LABELS = { "lint" => "Lint", "build" => "Build", "fast" => "Fast suite", "slow" => "Slow suite" }.freeze
+      ADVICE = {
+        "lint" => "Trim your linter config or split into stages.",
+        "build" => "Keep your build efficient and not let it become a bottleneck.",
+        "fast" => "Your fast tests have gotten too slow. Split or speed them up.",
+        "slow" => "Pare down integration tests, parallelise, or raise the budget."
+      }.freeze
+      FIX_TESTS = "Fix the failing tests above, then try again."
+      NEXT_STEPS = {
+        "lint" => "Fix what the linter reported above, then try again.",
+        "build" => "Fix the build errors above, then try again.",
+        "fast" => FIX_TESTS,
+        "slow" => FIX_TESTS
+      }.freeze
+
+      def initialize(commit:, stdout:, seams: Seams.new, dir: Dir.pwd)
+        @commit = commit
         @stdout = stdout
-        @command_runner = command_runner
-        @time_budgets = time_budgets
-        @recorder = recorder
+        @seams = seams
+        @dir = dir
       end
 
-      def run_stage(config, stage)
-        script = config.script_path(stage)
-        budget = @time_budgets[stage]
-
-        job_id = @recorder.start_stage(stage)
-        output, status, timed_out = run_with_timeout(script, budget)
-
-        if timed_out
-          @recorder.end_stage(job_id, "timed_out")
-          @stdout.puts "#{stage_label(stage)} killed -- exceeded #{budget}s time budget."
-          @stdout.puts budget_advice(stage)
-          return false
-        end
-
-        unless status.success?
-          @recorder.end_stage(job_id, "failed")
-          @stdout.puts output unless output.empty?
-          @stdout.puts "#{stage_label(stage)} failed."
-          return false
-        end
-
-        @recorder.end_stage(job_id, "completed")
-        true
+      # A failed stage's evidence is kept before its outcome is recorded, so
+      # whoever sees the outcome can read why.
+      def passes?(config, stage)
+        job_id = @seams.recorder.start_stage(stage, budget: @seams.budgets[stage])
+        command = config.stage_command(stage, @commit.sha)
+        result, output = StageExecution.new(seams: @seams, dir: @dir, commit: @commit).run(stage, command,
+                                                                                           @seams.recorder, job_id)
+        tell(stage, output, result)
+        result == "completed"
       end
 
       private
 
-      def run_with_timeout(script, budget)
-        cmd = "#{script} #{@commit_hash}"
-
-        if @command_runner
-          begin
-            output, status = @command_runner.call(cmd)
-            [output, status, false]
-          rescue Timeout::Error
-            ["", nil, true]
-          end
-        else
-          run_process_with_timeout(cmd, budget)
-        end
+      def tell(stage, output, result)
+        report_timeout(stage) if result == "timed_out"
+        report_failure(stage, output) if result == "failed"
       end
 
-      def stage_label(stage)
-        case stage
-        when "lint" then "Lint"
-        when "build" then "Build"
-        when "fast" then "Fast suite"
-        when "slow" then "Slow suite"
-        else stage
-        end
+      def report_timeout(stage)
+        @stdout.puts "#{label(stage)} killed -- exceeded #{@seams.budgets[stage]}s time budget."
+        @stdout.puts ADVICE[stage]
       end
 
-      def budget_advice(stage)
-        case stage
-        when "lint"
-          "Trim your linter config or split into stages."
-        when "build"
-          "Keep your build efficient and not let it become a bottleneck."
-        when "fast"
-          "Your fast tests have gotten too slow. Split or speed them up."
-        when "slow"
-          "Pare down integration tests, parallelise, or raise the budget."
-        end
+      def report_failure(stage, output)
+        @stdout.puts output unless output.empty?
+        @stdout.puts "#{label(stage)} failed."
+        @stdout.puts NEXT_STEPS[stage]
       end
+
+      def label(stage) = LABELS.fetch(stage, stage)
     end
   end
 end

data/lib/fun_ci/pipeline/stale_pipeline_canceller.rb

@@ -1,52 +1,35 @@
 # frozen_string_literal: true
 
-require_relative "../persistence/pipeline_run"
+require_relative "../persistence/active_runs"
+require_relative "run_canceller"
 
 module FunCi
   module Pipeline
+    # A newer commit on a branch cancels every run still going for an older
+    # one, unless an agent is waiting on it: its processes are stopped and it
+    # is recorded cancelled (acceptance-tests.md, AT-1.6, AT-9.12).
     class StalePipelineCanceller
-      def initialize(db:, branch:, stdout:, process_killer: nil)
+      def initialize(db:, branch:, stdout:, run_canceller: RunCanceller.new)
         @db = db
         @branch = branch
         @stdout = stdout
-        @process_killer = process_killer || method(:default_process_killer)
+        @run_canceller = run_canceller
       end
 
-      def cancel(new_commit_hash:)
-        run = Persistence::PipelineRun.find_running_with_pid(@db, @branch)
-        return unless run
-
-        kill_process(run[:pid])
-        Persistence::PipelineRun.update_status(@db, run[:id], "cancelled")
-        @stdout.puts "Cancelled stale pipeline for #{run[:commit_hash]}. Starting fresh for #{new_commit_hash}."
-      end
+      # How long an agent's last poll keeps a run it waits on from being cancelled.
+      WAITED_ON_SECONDS = 10
 
-      def store_pid(run_id, pid)
-        Persistence::PipelineRun.store_pid(@db, run_id, pid)
+      def cancel(new_commit_hash:)
+        waited_before = (Time.now - WAITED_ON_SECONDS).utc.iso8601
+        Persistence::ActiveRuns.cancellable(@db, @branch, new_commit: new_commit_hash, waited_before: waited_before)
+                               .each { |run| cancel_run(run, new_commit_hash) }
       end
 
       private
 
-      def kill_process(pid)
-        @process_killer.call(0, pid)
-        safe_signal("TERM", pid)
-        safe_signal("KILL", pid)
-        begin
-          Process.waitpid(pid)
-        rescue Errno::ECHILD, Errno::ESRCH # rubocop:disable Lint/SuppressedException
-        end
-      rescue Errno::ESRCH
-        # Process already dead
-      end
-
-      def safe_signal(signal, pid)
-        @process_killer.call(signal, pid)
-      rescue Errno::ESRCH
-        # Already dead
-      end
-
-      def default_process_killer(signal, pid)
-        Process.kill(signal, pid)
+      def cancel_run(run, new_commit_hash)
+        @run_canceller.cancel(@db, run)
+        @stdout.puts "Cancelled stale pipeline for #{run.commit_hash}. Starting fresh for #{new_commit_hash}."
       end
     end
   end

data/lib/fun_ci/pipeline/tail_segments.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Pipeline
+    # The last `size` bytes of a stream, kept in two IOs of `size` bytes that
+    # take turns: when one fills, the other is emptied and written next.
+    class TailSegments
+      def initialize(open, size)
+        @open = open
+        @size = size
+        @filled = 0
+        @segments = nil
+      end
+
+      def <<(bytes)
+        until bytes.empty?
+          rotate if @filled == @size
+          part = bytes.byteslice(0, @size - @filled)
+          segments.last.write(part)
+          @filled += part.bytesize
+          bytes = bytes.byteslice(part.bytesize..)
+        end
+      end
+
+      def read
+        return +"".b unless @segments
+
+        earlier(@size - @filled) + whole(segments.last)
+      end
+
+      def close = @segments&.each(&:close)
+
+      private
+
+      def segments = @segments ||= [@open.call("tail-0"), @open.call("tail-1")]
+
+      def rotate
+        @segments = segments.reverse
+        segments.last.truncate(0)
+        segments.last.rewind
+        @filled = 0
+      end
+
+      # The earlier segment is full once the tail has wrapped; before that,
+      # nothing came earlier.
+      def earlier(count)
+        io = segments.first
+        return +"".b if count.zero? || io.size < @size
+
+        io.seek(io.size - count)
+        io.read(count)
+      end
+
+      def whole(io)
+        io.rewind
+        io.read
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/test_report.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require "rexml/document"
+
+module FunCi
+  module Pipeline
+    # The failures a JUnit XML report names (acceptance-tests.md, AT-9.7),
+    # each as { file:, line:, test:, message: }, with output: when the failure
+    # kept its own (AT-10.14), its last 4 KB; nil for a report that can't be
+    # read.
+    module TestReport
+      OUTPUT_BYTES = 4096
+
+      def self.junit(xml)
+        REXML::XPath.match(REXML::Document.new(xml), "//testcase[failure or error]").map { |test| junit_failure(test) }
+      rescue REXML::ParseException
+        nil
+      end
+
+      def self.junit_failure(test)
+        with_output({ file: test.attributes["file"], line: test.attributes["line"]&.to_i, test: junit_name(test),
+                      message: junit_message(test.elements["failure"] || test.elements["error"]) },
+                    junit_output(test))
+      end
+
+      # The failure's text, or its message attribute when it has none.
+      def self.junit_message(problem)
+        text = problem.text.to_s.strip
+        text.empty? ? problem.attributes["message"].to_s : text
+      end
+
+      def self.junit_name(test)
+        [test.attributes["classname"], test.attributes["name"]].compact.join("#")
+      end
+
+      def self.junit_output(test)
+        %w[system-out system-err].filter_map { |name| test.elements[name]&.text&.strip }.reject(&:empty?).join("\n")
+      end
+
+      def self.with_output(failure, output)
+        return failure if output.empty?
+
+        failure.merge(output: output.byteslice([output.bytesize - OUTPUT_BYTES, 0].max..).scrub(""))
+      end
+      private_class_method :junit_failure, :junit_message, :junit_name, :junit_output, :with_output
+    end
+  end
+end

data/lib/fun_ci/pipeline/trigger.rb

@@ -1,152 +1,92 @@
 # frozen_string_literal: true
 
-require "open3"
 require_relative "../setup/project_config"
 require_relative "../persistence/pipeline_recorder"
-require_relative "background_wrapper"
-require_relative "stage_runner"
+require_relative "trigger_params"
+require_relative "trigger_command"
+require_relative "slot_run"
 require_relative "stale_pipeline_canceller"
-require_relative "progress_reporter"
-require_relative "pipeline_forker"
-require_relative "process_runner"
+require_relative "workspaces"
 
 module FunCi
   module Pipeline
     class Trigger
-      include ProcessRunner
-      NULL_SHA = ("0" * 40).freeze
-      DEFAULT_BUDGETS = { "lint" => 30, "build" => 30, "fast" => 10, "slow" => 300 }.freeze
-
-      def self.run_from_args(args, stdout: $stdout, stderr: $stderr, recorder: Persistence::NullRecorder.new, pipeline_forker: nil)
-        positional = args.reject { |a| a.start_with?("--") }
-        if positional.length < 2
-          stderr.puts "fun-ci: commit hash and branch name are required."
-          stderr.puts "Usage: fun-ci trigger <commit-hash> <branch>"
-          return 1
-        end
-        commit_hash, branch = positional
-        if args.include?("--no-validate")
-          db_path = recorder.db_path
-          recorder.close
-          (pipeline_forker || PipelineForker.method(:fork_pipeline)).call(
-            commit_hash: commit_hash, branch: branch, db_path: db_path
-          )
-          return 0
-        end
-        new(project_root: Dir.pwd, commit_hash: commit_hash, branch: branch,
-            stdout: stdout, stderr: stderr, recorder: recorder).run
+      NULL_SHA = Workspaces::NULL_SHA
+
+      def self.run_from_args(args, io: Io.new, recorder: Persistence::NullRecorder.new, pipeline_forker: nil)
+        TriggerCommand.new(io: io, recorder: recorder, pipeline_forker: pipeline_forker).run(args)
       end
 
-      attr_writer :command_runner
-
-      def initialize(project_root:, commit_hash:, branch:, stdout: $stdout, stderr: $stderr, command_runner: nil, time_budgets: {}, commit_validator: nil, recorder: Persistence::NullRecorder.new, background_launcher: nil)
-        @project_root = project_root
-        @commit_hash = commit_hash
-        @branch = branch
-        @stdout = stdout
-        @stderr = stderr
-        @command_runner = command_runner
-        @time_budgets = DEFAULT_BUDGETS.merge(time_budgets)
-        @commit_validator = commit_validator || method(:default_commit_validator)
-        @recorder = recorder
-        @background_launcher = background_launcher || method(:default_background_launcher)
+      def initialize(project:, commit:, io: Io.new, seams: Seams.new)
+        @project = project
+        @commit = commit
+        @io = io
+        @seams = seams
       end
 
       def run
-        config = Setup::ProjectConfig.new(@project_root)
+        config = Setup::ProjectConfig.new(@project)
         return handle_config_errors(config) if config.validate.any?
-        unless @commit_hash == NULL_SHA || @commit_validator.call(@commit_hash)
-          @stderr.puts "fun-ci: commit #{@commit_hash} not found in this repository."
-          return 1
-        end
-        cancel_stale_pipelines
-        @recorder.create_run(commit_hash: @commit_hash, branch: @branch, project_path: @project_root)
-        stage_runner = make_stage_runner
-        progress = ProgressReporter.new(stdout: @stdout)
-        results = run_phase_one(stage_runner, config)
-        progress.phase_one_result(results)
-        unless results.values.all?
-          @recorder.fail_run
-          return 1
-        end
-        spawn_slow_suite(config)
-        progress.slow_launched
-        fast_runner = make_stage_runner
-        fast_passed = fast_runner.run_stage(config, "fast")
-        progress.fast_result(fast_passed)
-        unless fast_passed
-          @recorder.fail_run
-          return 1
-        end
-        0
+        return unknown_commit unless known_commit?
+
+        start_run
+        run_in(workspace.acquire(@commit.sha))
+      end
+
+      # The background launcher swaps in a fresh recorder after forking, so
+      # callers release the connection through the trigger, not their own copy.
+      def close
+        recorder.close
       end
 
       private
 
+      def recorder = @seams.recorder
+      def known_commit? = @commit.sha == NULL_SHA || @seams.commit_validator.call(@commit.sha)
+
+      def workspace = @seams.workspace || Workspaces.for(@project, @commit.sha)
+      def branch = Persistence::Branch.new(project: @project, name: @commit.branch)
+
+      # The scripts come from the commit when it has them, so they match the
+      # code they test; a project that keeps .fun-ci/ out of git uses its own.
+      def config_for(slot)
+        committed = Setup::ProjectConfig.new(slot.path)
+        committed.folder_exists? ? committed : Setup::ProjectConfig.new(@project)
+      end
+
       def handle_config_errors(config)
-        config.validate.each { |e| @stdout.puts "fun-ci: #{e}" }
+        config.validate.each { |e| @io.stdout.puts "fun-ci: #{e}" }
         unless config.folder_exists?
-          @stdout.puts "Create .fun-ci/lint.sh, build.sh, fast.sh, and slow.sh to set up this project."
-          @stdout.puts "Commit will proceed without CI."
+          @io.stdout.puts "Create .fun-ci/lint.sh, build.sh, fast.sh, and slow.sh to set up this project."
+          @io.stdout.puts "Commit will proceed without CI."
         end
         0
       end
 
-      def run_phase_one(stage_runner, config)
-        results = {}
-        threads = %w[lint build].map do |stage|
-          Thread.new { results[stage] = stage_runner.run_stage(config, stage) }
-        end
-        threads.each(&:join)
-        results
+      def unknown_commit
+        @io.stderr.puts "fun-ci: commit #{@commit.sha} not found in this repository."
+        1
       end
 
-      def spawn_slow_suite(config)
-        cmd = "#{config.script_path("slow")} #{@commit_hash}"
-        job_id = @recorder.start_stage("slow")
-        budget = @time_budgets["slow"]
-        executor = if @command_runner
-          runner = @command_runner
-          -> do
-            output, status = runner.call(cmd)
-            [output, status, false]
-          rescue Timeout::Error
-            ["", nil, true]
-          end
-        else
-          -> { run_process_with_timeout(cmd, budget) }
-        end
-        @background_launcher.call(
-          db_path: @recorder.db_path, pipeline_run_id: @recorder.pipeline_run_id,
-          job_id: job_id, executor: executor
-        )
+      def start_run
+        recorder.report_trouble_to(@io.stdout)
+        cancel_stale_pipelines
+        recorder.create_run(commit_hash: @commit.sha, branch: @commit.branch, project_path: @project)
       end
 
-      def make_stage_runner
-        StageRunner.new(commit_hash: @commit_hash, stdout: @stdout,
-          command_runner: @command_runner, time_budgets: @time_budgets, recorder: @recorder)
+      # After forking the slow suite the run holds a recorder of its own.
+      def run_in(slot)
+        slot_run = SlotRun.new(commit: @commit, io: @io, seams: @seams, slot: slot)
+        slot_run.run(config_for(slot)).tap { @seams = slot_run.seams }
       end
 
       def cancel_stale_pipelines
-        return unless @recorder.db
-        StalePipelineCanceller.new(db: @recorder.db, branch: @branch, stdout: @stdout)
-          .cancel(new_commit_hash: @commit_hash)
-      end
+        return unless recorder.db
 
-      def default_background_launcher(db_path:, pipeline_run_id:, job_id:, executor:)
-        @recorder.close
-        pid = fork do
-          recorder = Persistence::DbRecorder.for_background(db_path, pipeline_run_id)
-          BackgroundWrapper.new(recorder: recorder, job_id: job_id, executor: executor).run
-          recorder.close
+        recorder.tolerating do
+          StalePipelineCanceller.new(db: recorder.db, branch: branch, stdout: @io.stdout)
+                                .cancel(new_commit_hash: @commit.sha)
         end
-        @recorder = Persistence::DbRecorder.for_background(db_path, pipeline_run_id)
-        Process.detach(pid)
-        Persistence::PipelineRun.store_pid(@recorder.db, pipeline_run_id, pid)
-      end
-
-      def default_commit_validator(hash)
-        Open3.capture2e("git", "cat-file", "-t", hash).last.success?
       end
     end
   end