fun_ci 2.0.0-aarch64-linux

data/lib/fun_ci/pipeline/stage_execution.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require_relative "stage_end"
+require_relative "../evidence/start"
+
+module FunCi
+  module Pipeline
+    # Runs a stage that has been started, in `dir`, with a stage directory of
+    # its own, and records how it ended while it still holds both. The same
+    # for every stage, the slow suite in its forked child included.
+    class StageExecution
+      def initialize(seams:, dir:, commit:)
+        @seams = seams
+        @dir = dir
+        @commit = commit
+      end
+
+      # Answers [the outcome recorded, what the stage printed].
+      def run(stage, command, recorder, job_id)
+        with_stage_dir do |stage_dir|
+          collector = collector(stage, stage_dir)
+          finished = finished(stage, command, stage_dir, collector) { |pid| recorder.stage_process(job_id, pid) }
+          [StageEnd.new(recorder, job_id, collector).record(finished), finished.output]
+        end
+      end
+
+      private
+
+      # A stage over budget is looked at before the kill (acceptance-tests.md, AT-10.17).
+      def finished(stage, command, stage_dir, collector, &)
+        overrun = nil
+        output, status, timed_out = execute(stage, command, stage_dir, lambda { |pgid|
+          overrun = collector.before_kill(pgid)
+        },
+                                            &)
+        StageEnd::Finished.new(output: output, status: status, timed_out: timed_out, overrun: overrun)
+      end
+
+      # The stage learns its name from FUN_CI_STAGE (acceptance-tests.md, AT-10.7).
+      def execute(stage, command, stage_dir, before_kill, &)
+        window = stage_dir.window
+        launch = ProcessRunner::Launch.new(env: { "FUN_CI_STAGE" => stage }, output: window,
+                                           before_kill: before_kill)
+        @seams.executor(@dir).call(command, @seams.budgets[stage], launch, &)
+      ensure
+        window&.close
+      end
+
+      def with_stage_dir
+        stage_dir = @seams.stage_dir.call
+        yield stage_dir
+      ensure
+        stage_dir&.remove
+      end
+
+      # Made before the stage runs, which is when its watched files are stamped.
+      def collector(stage, stage_dir)
+        sources = Evidence::Sources.new(stage: stage, worktree: @dir, stage_dir: stage_dir,
+                                        environment: @seams.environment,
+                                        budget: @seams.budgets[stage], commit: @commit.to_h, started: Time.now,
+                                        processes: @seams.process_table)
+        commands = @seams.extractor_runner.call(dir: @dir, scratch: stage_dir.scratch)
+        Evidence::Start.collector(sources, @seams.clock, commands)
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/stage_runner.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require_relative "trigger_params"
+require_relative "stage_execution"
+
+module FunCi
+  module Pipeline
+    class StageRunner
+      LABELS = { "lint" => "Lint", "build" => "Build", "fast" => "Fast suite", "slow" => "Slow suite" }.freeze
+      ADVICE = {
+        "lint" => "Trim your linter config or split into stages.",
+        "build" => "Keep your build efficient and not let it become a bottleneck.",
+        "fast" => "Your fast tests have gotten too slow. Split or speed them up.",
+        "slow" => "Pare down integration tests, parallelise, or raise the budget."
+      }.freeze
+      FIX_TESTS = "Fix the failing tests above, then try again."
+      NEXT_STEPS = {
+        "lint" => "Fix what the linter reported above, then try again.",
+        "build" => "Fix the build errors above, then try again.",
+        "fast" => FIX_TESTS,
+        "slow" => FIX_TESTS
+      }.freeze
+
+      def initialize(commit:, stdout:, seams: Seams.new, dir: Dir.pwd)
+        @commit = commit
+        @stdout = stdout
+        @seams = seams
+        @dir = dir
+      end
+
+      # A failed stage's evidence is kept before its outcome is recorded, so
+      # whoever sees the outcome can read why.
+      def passes?(config, stage)
+        job_id = @seams.recorder.start_stage(stage, budget: @seams.budgets[stage])
+        command = config.stage_command(stage, @commit.sha)
+        result, output = StageExecution.new(seams: @seams, dir: @dir, commit: @commit).run(stage, command,
+                                                                                           @seams.recorder, job_id)
+        tell(stage, output, result)
+        result == "completed"
+      end
+
+      private
+
+      def tell(stage, output, result)
+        report_timeout(stage) if result == "timed_out"
+        report_failure(stage, output) if result == "failed"
+      end
+
+      def report_timeout(stage)
+        @stdout.puts "#{label(stage)} killed -- exceeded #{@seams.budgets[stage]}s time budget."
+        @stdout.puts ADVICE[stage]
+      end
+
+      def report_failure(stage, output)
+        @stdout.puts output unless output.empty?
+        @stdout.puts "#{label(stage)} failed."
+        @stdout.puts NEXT_STEPS[stage]
+      end
+
+      def label(stage) = LABELS.fetch(stage, stage)
+    end
+  end
+end

data/lib/fun_ci/pipeline/stale_pipeline_canceller.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require_relative "../persistence/active_runs"
+require_relative "run_canceller"
+
+module FunCi
+  module Pipeline
+    # A newer commit on a branch cancels every run still going for an older
+    # one, unless an agent is waiting on it: its processes are stopped and it
+    # is recorded cancelled (acceptance-tests.md, AT-1.6, AT-9.12).
+    class StalePipelineCanceller
+      def initialize(db:, branch:, stdout:, run_canceller: RunCanceller.new)
+        @db = db
+        @branch = branch
+        @stdout = stdout
+        @run_canceller = run_canceller
+      end
+
+      # How long an agent's last poll keeps a run it waits on from being cancelled.
+      WAITED_ON_SECONDS = 10
+
+      def cancel(new_commit_hash:)
+        waited_before = (Time.now - WAITED_ON_SECONDS).utc.iso8601
+        Persistence::ActiveRuns.cancellable(@db, @branch, new_commit: new_commit_hash, waited_before: waited_before)
+                               .each { |run| cancel_run(run, new_commit_hash) }
+      end
+
+      private
+
+      def cancel_run(run, new_commit_hash)
+        @run_canceller.cancel(@db, run)
+        @stdout.puts "Cancelled stale pipeline for #{run.commit_hash}. Starting fresh for #{new_commit_hash}."
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/tail_segments.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Pipeline
+    # The last `size` bytes of a stream, kept in two IOs of `size` bytes that
+    # take turns: when one fills, the other is emptied and written next.
+    class TailSegments
+      def initialize(open, size)
+        @open = open
+        @size = size
+        @filled = 0
+        @segments = nil
+      end
+
+      def <<(bytes)
+        until bytes.empty?
+          rotate if @filled == @size
+          part = bytes.byteslice(0, @size - @filled)
+          segments.last.write(part)
+          @filled += part.bytesize
+          bytes = bytes.byteslice(part.bytesize..)
+        end
+      end
+
+      def read
+        return +"".b unless @segments
+
+        earlier(@size - @filled) + whole(segments.last)
+      end
+
+      def close = @segments&.each(&:close)
+
+      private
+
+      def segments = @segments ||= [@open.call("tail-0"), @open.call("tail-1")]
+
+      def rotate
+        @segments = segments.reverse
+        segments.last.truncate(0)
+        segments.last.rewind
+        @filled = 0
+      end
+
+      # The earlier segment is full once the tail has wrapped; before that,
+      # nothing came earlier.
+      def earlier(count)
+        io = segments.first
+        return +"".b if count.zero? || io.size < @size
+
+        io.seek(io.size - count)
+        io.read(count)
+      end
+
+      def whole(io)
+        io.rewind
+        io.read
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/test_report.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require "rexml/document"
+
+module FunCi
+  module Pipeline
+    # The failures a JUnit XML report names (acceptance-tests.md, AT-9.7),
+    # each as { file:, line:, test:, message: }, with output: when the failure
+    # kept its own (AT-10.14), its last 4 KB; nil for a report that can't be
+    # read.
+    module TestReport
+      OUTPUT_BYTES = 4096
+
+      def self.junit(xml)
+        REXML::XPath.match(REXML::Document.new(xml), "//testcase[failure or error]").map { |test| junit_failure(test) }
+      rescue REXML::ParseException
+        nil
+      end
+
+      def self.junit_failure(test)
+        with_output({ file: test.attributes["file"], line: test.attributes["line"]&.to_i, test: junit_name(test),
+                      message: junit_message(test.elements["failure"] || test.elements["error"]) },
+                    junit_output(test))
+      end
+
+      # The failure's text, or its message attribute when it has none.
+      def self.junit_message(problem)
+        text = problem.text.to_s.strip
+        text.empty? ? problem.attributes["message"].to_s : text
+      end
+
+      def self.junit_name(test)
+        [test.attributes["classname"], test.attributes["name"]].compact.join("#")
+      end
+
+      def self.junit_output(test)
+        %w[system-out system-err].filter_map { |name| test.elements[name]&.text&.strip }.reject(&:empty?).join("\n")
+      end
+
+      def self.with_output(failure, output)
+        return failure if output.empty?
+
+        failure.merge(output: output.byteslice([output.bytesize - OUTPUT_BYTES, 0].max..).scrub(""))
+      end
+      private_class_method :junit_failure, :junit_message, :junit_name, :junit_output, :with_output
+    end
+  end
+end

data/lib/fun_ci/pipeline/trigger.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require_relative "../setup/project_config"
+require_relative "../persistence/pipeline_recorder"
+require_relative "trigger_params"
+require_relative "trigger_command"
+require_relative "slot_run"
+require_relative "stale_pipeline_canceller"
+require_relative "workspaces"
+
+module FunCi
+  module Pipeline
+    class Trigger
+      NULL_SHA = Workspaces::NULL_SHA
+
+      def self.run_from_args(args, io: Io.new, recorder: Persistence::NullRecorder.new, pipeline_forker: nil)
+        TriggerCommand.new(io: io, recorder: recorder, pipeline_forker: pipeline_forker).run(args)
+      end
+
+      def initialize(project:, commit:, io: Io.new, seams: Seams.new)
+        @project = project
+        @commit = commit
+        @io = io
+        @seams = seams
+      end
+
+      def run
+        config = Setup::ProjectConfig.new(@project)
+        return handle_config_errors(config) if config.validate.any?
+        return unknown_commit unless known_commit?
+
+        start_run
+        run_in(workspace.acquire(@commit.sha))
+      end
+
+      # The background launcher swaps in a fresh recorder after forking, so
+      # callers release the connection through the trigger, not their own copy.
+      def close
+        recorder.close
+      end
+
+      private
+
+      def recorder = @seams.recorder
+      def known_commit? = @commit.sha == NULL_SHA || @seams.commit_validator.call(@commit.sha)
+
+      def workspace = @seams.workspace || Workspaces.for(@project, @commit.sha)
+      def branch = Persistence::Branch.new(project: @project, name: @commit.branch)
+
+      # The scripts come from the commit when it has them, so they match the
+      # code they test; a project that keeps .fun-ci/ out of git uses its own.
+      def config_for(slot)
+        committed = Setup::ProjectConfig.new(slot.path)
+        committed.folder_exists? ? committed : Setup::ProjectConfig.new(@project)
+      end
+
+      def handle_config_errors(config)
+        config.validate.each { |e| @io.stdout.puts "fun-ci: #{e}" }
+        unless config.folder_exists?
+          @io.stdout.puts "Create .fun-ci/lint.sh, build.sh, fast.sh, and slow.sh to set up this project."
+          @io.stdout.puts "Commit will proceed without CI."
+        end
+        0
+      end
+
+      def unknown_commit
+        @io.stderr.puts "fun-ci: commit #{@commit.sha} not found in this repository."
+        1
+      end
+
+      def start_run
+        recorder.report_trouble_to(@io.stdout)
+        cancel_stale_pipelines
+        recorder.create_run(commit_hash: @commit.sha, branch: @commit.branch, project_path: @project)
+      end
+
+      # After forking the slow suite the run holds a recorder of its own.
+      def run_in(slot)
+        slot_run = SlotRun.new(commit: @commit, io: @io, seams: @seams, slot: slot)
+        slot_run.run(config_for(slot)).tap { @seams = slot_run.seams }
+      end
+
+      def cancel_stale_pipelines
+        return unless recorder.db
+
+        recorder.tolerating do
+          StalePipelineCanceller.new(db: recorder.db, branch: branch, stdout: @io.stdout)
+                                .cancel(new_commit_hash: @commit.sha)
+        end
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/trigger_command.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require_relative "trigger_params"
+require_relative "pipeline_forker"
+
+module FunCi
+  module Pipeline
+    # `fun-ci trigger [--background] <commit-hash> <branch>`. --no-validate is
+    # the 1.x name for --background, kept for one release.
+    class TriggerCommand
+      BACKGROUND = %w[--background --no-validate].freeze
+      DEPRECATED = "fun-ci: --no-validate is now --background; the old name goes in 2.1."
+
+      def initialize(io:, recorder:, pipeline_forker:, project: Dir.pwd)
+        @io = io
+        @recorder = recorder
+        @pipeline_forker = pipeline_forker || PipelineForker.method(:fork_pipeline)
+        @project = project
+      end
+
+      # Closes the database connection it was given on every path.
+      def run(args)
+        sha, branch = args.reject { |a| a.start_with?("--") }
+        return usage unless branch
+
+        @io.stderr.puts DEPRECATED if args.include?("--no-validate")
+        commit = Commit.new(sha: sha, branch: branch)
+        args.intersect?(BACKGROUND) ? fork_pipeline(commit) : run_pipeline(commit)
+      ensure
+        @recorder.close
+      end
+
+      private
+
+      # After forking the slow suite the trigger holds a recorder of its own.
+      def run_pipeline(commit)
+        trigger = Trigger.new(project: @project, commit: commit, io: @io, seams: Seams.new(recorder: @recorder))
+        trigger.run
+      ensure
+        trigger&.close
+      end
+
+      def usage
+        @io.stderr.puts "fun-ci: commit hash and branch name are required."
+        @io.stderr.puts "Usage: fun-ci trigger <commit-hash> <branch>"
+        1
+      end
+
+      # The forker answers whether it started a run; one that did says how an
+      # agent gets its verdict (acceptance-tests.md, AT-9.14).
+      def fork_pipeline(commit)
+        db_path = @recorder.db_path
+        @recorder.close
+        started = @pipeline_forker.call(commit_hash: commit.sha, branch: commit.branch, db_path: db_path)
+        say_how_to_wait(commit.sha[0, 7]) if started
+        0
+      end
+
+      def say_how_to_wait(sha) = @io.stdout.puts("fun-ci: testing #{sha}. Verdict: fun-ci wait #{sha} --need all")
+    end
+  end
+end

data/lib/fun_ci/pipeline/trigger_params.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require "open3"
+require_relative "../persistence/pipeline_recorder"
+require_relative "command_executor"
+require_relative "stage_dir"
+require_relative "git_environment"
+require_relative "../evidence/command_runner"
+require_relative "../evidence/process_table"
+
+module FunCi
+  module Pipeline
+    DEFAULT_BUDGETS = { "lint" => 30, "build" => 30, "fast" => 10, "slow" => 300 }.freeze
+
+    Commit = Data.define(:sha, :branch)
+
+    Io = Data.define(:stdout, :stderr) do
+      def initialize(stdout: $stdout, stderr: $stderr) = super
+    end
+
+    # The collaborators a pipeline run can have replaced. Each one left out
+    # gets the real thing. `environment` is the stages' environment as a hash,
+    # which masking and injected command runners see; `clock` answers seconds
+    # for the evidence budget; `extractor_runner` makes what runs a project's
+    # own extractors (`run:` entries), given dir:, env: and scratch:;
+    # `process_table` answers what ps lists, for a stage over budget.
+    Seams = Data.define(:command_runner, :time_budgets, :commit_validator, :recorder, :background_launcher, :workspace,
+                        :stage_dir, :environment, :clock, :extractor_runner, :process_table)
+
+    # Reopened rather than given as a block to Data.define, so tools that read
+    # the source (mutineer) see these as Seams' methods.
+    class Seams
+      def self.defaults
+        { command_runner: nil, time_budgets: {}, recorder: Persistence::NullRecorder.new, background_launcher: nil,
+          workspace: nil, commit_validator: method(:commit_exists?), stage_dir: StageDir.method(:create),
+          environment: ENV.to_h, clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) },
+          extractor_runner: Evidence::CommandRunner.method(:new),
+          process_table: Evidence::ProcessTable.method(:now) }
+      end
+
+      def self.commit_exists?(sha) = Open3.capture2e(GitEnvironment::CLEAN, "git", "cat-file", "-t", sha).last.success?
+
+      def initialize(**given) = super(**self.class.defaults.merge(given))
+      def budgets = DEFAULT_BUDGETS.merge(time_budgets)
+      def executor(dir) = CommandExecutor.new(command_runner, dir, environment)
+    end
+  end
+end

data/lib/fun_ci/pipeline/workspaces.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require_relative "../setup/project_config"
+require_relative "worktree_pool"
+require_relative "in_place"
+
+module FunCi
+  module Pipeline
+    # Where a pipeline for a commit runs: a slot from the project's worktree
+    # pool, sized by .fun-ci/config, or, for a root commit's null SHA, which
+    # no worktree can check out, the project directory itself.
+    module Workspaces
+      NULL_SHA = ("0" * 40).freeze
+
+      def self.for(project, sha)
+        return InPlace.new(project) if sha == NULL_SHA
+
+        WorktreePool.new(Worktrees.new(project), size: Setup::ProjectConfig.new(project).worktree_slots)
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/worktree_pool.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require_relative "slot"
+require_relative "worktrees"
+
+module FunCi
+  module Pipeline
+    # Hands each pipeline a worktree slot of its own under
+    # <git-common-dir>/fun-ci/worktrees, checked out at its commit, waiting
+    # while every slot is taken.
+    class WorktreePool
+      DEFAULT_SIZE = 2
+
+      attr_reader :size
+
+      def initialize(worktrees, size: DEFAULT_SIZE, waiter: -> { sleep 0.25 })
+        @worktrees = worktrees
+        @size = size
+        @waiter = waiter
+      end
+
+      def acquire(sha)
+        slot = wait_for_free_slot
+        @worktrees.check_out(slot.path, sha)
+        slot
+      rescue StandardError
+        slot&.release
+        raise
+      end
+
+      private
+
+      def wait_for_free_slot
+        loop do
+          slot = free_slot
+          return slot if slot
+
+          @waiter.call
+        end
+      end
+
+      def free_slot
+        root = @worktrees.root
+        FileUtils.mkdir_p(root)
+        @size.times.lazy.filter_map { |index| try_lock(File.join(root, "slot-#{index}")) }.first
+      end
+
+      # The lock stays open for as long as the slot is held, so no block.
+      def try_lock(path)
+        lock = File.new("#{path}.lock", File::RDWR | File::CREAT, 0o644)
+        return Slot.new(path, stamped(lock), lock.path) if lock.flock(File::LOCK_EX | File::LOCK_NB)
+
+        lock.close
+        nil
+      end
+
+      def stamped(lock)
+        lock.truncate(0)
+        lock.write(Process.pid.to_s)
+        lock.flush
+        lock
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/worktree_prune.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require_relative "worktrees"
+
+module FunCi
+  module Pipeline
+    # Removes every slot of a project's worktree pool, and git's entries for
+    # them, when no run holds one. It takes each slot's lock first, so a run
+    # starting meanwhile waits until the slot is gone and then checks it out
+    # afresh.
+    class WorktreePrune
+      class Busy < StandardError; end
+
+      def initialize(worktrees)
+        @worktrees = worktrees
+      end
+
+      # How many slots it removed. Raises Busy, removing nothing, while a run
+      # holds a slot.
+      def run
+        locks = take_locks
+        locks.each { |lock| FileUtils.rm_rf(lock.path.delete_suffix(".lock")) }
+        @worktrees.prune
+        locks.size
+      ensure
+        locks&.each(&:close)
+      end
+
+      private
+
+      # Every slot's lock, or none: on the first slot a run holds, it lets go
+      # of the locks it took before raising Busy.
+      def take_locks
+        slot_paths.each_with_object([]) do |path, locks|
+          locks << File.new("#{path}.lock", File::RDWR | File::CREAT, 0o644)
+          next if locks.last.flock(File::LOCK_EX | File::LOCK_NB)
+
+          locks.each(&:close)
+          raise Busy, "a pipeline is running in #{path}"
+        end
+      end
+
+      def slot_paths
+        Dir.glob(File.join(@worktrees.root, "slot-*")).map { |path| path.delete_suffix(".lock") }.uniq
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/worktrees.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require "open3"
+require_relative "git_environment"
+
+module FunCi
+  module Pipeline
+    # The git commands a worktree slot needs, run against the project's repository.
+    class Worktrees
+      class GitError < StandardError; end
+
+      def initialize(project_root)
+        @project_root = project_root
+      end
+
+      def root
+        File.join(File.expand_path(git(@project_root, "rev-parse", "--git-common-dir").strip, @project_root),
+                  "fun-ci", "worktrees")
+      end
+
+      # Leaves +path+ as a clean detached checkout of +sha+. Ignored files stay,
+      # so caches such as vendor/bundle survive between runs in the same slot.
+      def check_out(path, sha)
+        return add(path, sha) unless File.exist?(File.join(path, ".git"))
+
+        git(path, "checkout", "--detach", "--force", sha)
+        git(path, "clean", "-fd")
+      end
+
+      # Has git forget the worktrees whose directories are gone.
+      def prune = git(@project_root, "worktree", "prune")
+
+      private
+
+      def add(path, sha) = git(@project_root, "worktree", "add", "--detach", "--force", path, sha)
+
+      def git(dir, *args)
+        output, status = Open3.capture2e(GitEnvironment::CLEAN, "git", *args, chdir: dir)
+        raise GitError, "git #{args.join(" ")}: #{output}" unless status.success?
+
+        output
+      end
+    end
+  end
+end