fun_ci 2.0.0-x86_64-linux

data/lib/fun_ci/pipeline/head_part.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Pipeline
+    # The first `size` bytes of a stream, kept in an IO opened when first written.
+    class HeadPart
+      def initialize(open, size)
+        @open = open
+        @size = size
+        @closed = false
+      end
+
+      # Writes what fits, and answers the rest.
+      def take(bytes)
+        room = [@size - written, 0].max
+        io.write(bytes.byteslice(0, room)) if room.positive? && !bytes.empty?
+        bytes.byteslice(room..) || "".b
+      end
+
+      def read
+        return +"".b unless @io
+
+        @io.rewind
+        @io.read
+      end
+
+      def close
+        @io&.close
+        @closed = true
+      end
+
+      def closed? = @closed
+
+      private
+
+      def written = @io ? @io.size : 0
+      def io = @io ||= @open.call("head")
+    end
+  end
+end

data/lib/fun_ci/pipeline/in_place.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require_relative "slot"
+
+module FunCi
+  module Pipeline
+    # A workspace that is the project directory itself, with nothing to lock.
+    # For a root commit's null SHA, which no worktree can check out.
+    InPlace = Data.define(:path)
+
+    class InPlace
+      Unlocked = Struct.new(:closed?) { def close = nil }
+
+      def acquire(_sha) = Slot.new(path, Unlocked.new(false))
+    end
+  end
+end

data/lib/fun_ci/pipeline/output_window.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "stringio"
+require_relative "head_part"
+require_relative "tail_segments"
+
+module FunCi
+  module Pipeline
+    # What a stage printed, kept as its first `head` bytes and its last
+    # `tail` bytes, written as it comes so no more than a chunk is held in
+    # memory. Both cuts fall on line ends, unless a line is longer than the
+    # part it is in, and a marker line says how many bytes were dropped.
+    class OutputWindow
+      Sizes = Data.define(:head, :tail)
+      REAL = Sizes.new(head: 1_048_576, tail: 7_340_032)
+
+      def self.in_memory(sizes = REAL) = new(->(_name) { StringIO.new(+"".b) }, sizes)
+
+      def self.in(dir, sizes = REAL)
+        new(->(name) { File.open(File.join(dir, name), "w+b") }, sizes)
+      end
+
+      # `open` answers a new, empty read-write IO for a name.
+      def initialize(open, sizes)
+        @sizes = sizes
+        @written = 0
+        @head = HeadPart.new(open, sizes.head)
+        @tail = TailSegments.new(open, sizes.tail + 1)
+      end
+
+      def <<(chunk)
+        bytes = chunk.b
+        rest = @head.take(bytes)
+        @tail << rest unless rest.empty?
+        @written += bytes.bytesize
+        self
+      end
+
+      def text
+        head = @head.read
+        return head + @tail.read if @written <= @sizes.head + @sizes.tail
+
+        cut(head, @tail.read)
+      end
+
+      def close
+        @head.close
+        @tail.close
+      end
+
+      def closed? = @head.closed?
+
+      private
+
+      def cut(head, tail)
+        kept_head = head.rindex("\n") ? head[0..head.rindex("\n")] : head
+        kept_tail = line_start(tail)
+        dropped = @written - kept_head.bytesize - kept_tail.bytesize
+        "#{kept_head}[fun-ci: #{dropped} bytes dropped here]\n".b + kept_tail
+      end
+
+      # The tail holds one byte more than it keeps, to tell whether it starts
+      # on a line start. With no line start in it but after its last byte, it
+      # is the end of one long line, and is kept from mid-line.
+      def line_start(tail)
+        first_end = tail.index("\n")
+        return tail.byteslice(1..) if first_end.nil? || first_end.zero? || first_end == tail.bytesize - 1
+
+        tail.byteslice((first_end + 1)..)
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/pipeline_forker.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require_relative "../persistence/database"
+require_relative "../persistence/pipeline_recorder"
+require_relative "trigger_params"
+require_relative "../setup/project_config"
+
+module FunCi
+  module Pipeline
+    class PipelineForker
+      # Answers whether it started a run: none in a project not set up for fun-ci.
+      def self.fork_pipeline(commit_hash:, branch:, db_path:)
+        return false unless Setup::ProjectConfig.new(Dir.pwd).validate.empty?
+
+        pid = fork do
+          run_in_child(commit_hash: commit_hash, branch: branch, db_path: db_path)
+        end
+        Process.detach(pid)
+      end
+
+      # Trigger requires this file, through TriggerCommand, so it is loaded
+      # here, where it is used, rather than above.
+      def self.run_in_child(commit_hash:, branch:, db_path:)
+        require_relative "trigger"
+        recorder = Persistence::DbRecorder.new(Persistence::Database.connection(db_path))
+        trigger = trigger(commit_hash, branch, recorder)
+        trigger.run
+        trigger.close
+      end
+
+      # The slow suite forks from here as it does in the foreground, so the
+      # fast suite, whose verdict a push waits for, runs beside it.
+      def self.trigger(commit_hash, branch, recorder)
+        Trigger.new(project: Dir.pwd, commit: Commit.new(sha: commit_hash, branch: branch),
+                    io: Io.new(stdout: File.open(File::NULL, "w")), seams: Seams.new(recorder: recorder))
+      end
+      private_class_method :trigger
+    end
+  end
+end

data/lib/fun_ci/pipeline/process_runner.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+require_relative "gate"
+require_relative "git_environment"
+require_relative "output_window"
+
+module FunCi
+  module Pipeline
+    module ProcessRunner
+      # The command waits on fd 9 until the caller has been told its pid, so a
+      # cancel that reads the recorded pid can never miss a stage that has
+      # started; exec keeps the pid and the process group. If the caller dies
+      # first, the pipe closes and the command never runs.
+      GATED = "read _ <&9 || exit 125; exec 9<&-; exec %s"
+
+      # Yields the pid of the process it starts, which leads a process group
+      # of its own, so the caller can stop the command and all it spawned.
+      # A command over budget answers what it printed before the kill, which
+      # is the only clue to what it was stuck on (acceptance-tests.md, AT-9.5).
+      # `timer` answers whether the reading finished within the budget.
+      BUDGET = ->(reading, budget) { reading.join(budget) }
+      # How long a killed command's last output may take to drain.
+      DRAIN_SECONDS = 1
+
+      # Where the command runs, what it finds in its environment besides
+      # fun-ci's own, the window what it prints is written to, and what to do
+      # with its pid when it runs over budget, before it is killed.
+      Launch = Data.define(:chdir, :env, :output, :before_kill)
+
+      class Launch
+        def initialize(**given)
+          super(chdir: Dir.pwd, env: {}, output: OutputWindow.in_memory, before_kill: nil, **given)
+        end
+      end
+
+      def run_process_with_timeout(cmd, budget, launch: Launch.new, timer: BUDGET, &)
+        reader, writer = IO.pipe
+        pid = start(cmd, writer, launch, &)
+        printed = launch.output
+        reading = Thread.new { read_until_closed(reader, printed) }
+        timer.call(reading, budget) ? process_finished(pid, text(reading.value)) : over_budget(pid, reading, launch)
+      ensure
+        ignoring_errors { reader&.close }
+      end
+
+      private
+
+      def start(cmd, writer, launch)
+        gate = Gate.create
+        pid = spawn_gated(cmd, writer, gate, launch)
+        [writer, gate.child_end].each(&:close)
+        yield pid if block_given?
+        gate.open
+        pid
+      end
+
+      def spawn_gated(cmd, writer, gate, launch)
+        Process.spawn(GitEnvironment::CLEAN.merge(launch.env), format(GATED, cmd),
+                      out: writer, err: writer, 9 => gate.child_end, pgroup: true, chdir: launch.chdir)
+      end
+
+      # The reader may be closed while this thread still reads, once a killed
+      # command's output has drained or stopped coming.
+      def read_until_closed(reader, printed)
+        loop { printed << reader.readpartial(65_536) }
+      rescue IOError
+        printed
+      end
+
+      # What looks at the stage before the kill does so while its output is still read.
+      def over_budget(pid, reading, launch)
+        launch.before_kill&.call(pid)
+        kill_process_group(pid)
+        [drained(reading, launch.output), nil, true]
+      end
+
+      def drained(reading, printed)
+        reading.join(DRAIN_SECONDS)
+        text(printed)
+      end
+
+      def text(window) = window.text.force_encoding(Encoding.default_external)
+
+      def process_finished(pid, output)
+        _, status = Process.waitpid2(pid)
+        [output, status, false]
+      end
+
+      def kill_process_group(pid)
+        %w[TERM KILL].each { |signal| ignoring_errors { Process.kill(signal, -pid) } }
+        ignoring_errors { Process.waitpid(pid) }
+      end
+
+      def ignoring_errors
+        yield
+      rescue StandardError
+        # A process that already exited or was already reaped needs nothing.
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/progress_reporter.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Pipeline
+    class ProgressReporter
+      def initialize(stdout:)
+        @stdout = stdout
+      end
+
+      def phase_one_result(results)
+        markers = results.map { |stage, passed| "#{stage} #{result_marker(passed)}" }
+        summary = results.values.all? ? "phase 1 passed" : "phase 1 failed"
+        @stdout.puts "fun-ci: #{markers.join("  ")}  (#{summary})"
+      end
+
+      def fast_result(passed)
+        @stdout.puts "fun-ci: fast #{result_marker(passed)}"
+      end
+
+      def slow_launched
+        @stdout.puts "fun-ci: slow (running in background)"
+      end
+
+      private
+
+      def result_marker(passed)
+        passed ? "ok" : "FAIL"
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/run_canceller.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require_relative "../persistence/active_runs"
+require_relative "../persistence/run_status"
+require_relative "../persistence/write_trouble"
+require_relative "slot_lock"
+
+module FunCi
+  module Pipeline
+    # Stops every process of a run. The run's own processes go first, so none
+    # of them records the run as failed once its stages die; then each stage's
+    # process group. A run whose slot nobody holds any more has died already,
+    # and the pids it recorded may belong to other processes now, so it is
+    # left alone; so is this process.
+    class RunCanceller
+      def initialize(killer: Process.method(:kill), slot_held: SlotLock.method(:held?))
+        @killer = killer
+        @slot_held = slot_held
+      end
+
+      # Stops the run and records it, and its unfinished stages, cancelled.
+      def cancel(db, run)
+        stop(run) && stop_late_stages(db, run)
+        Persistence::ActiveRuns.cancelled(db, run)
+      end
+
+      # Records failed each slow stage whose forked process is gone without
+      # having recorded its result (acceptance-tests.md, AT-8.3), and settles
+      # its run.
+      def record_dead(db)
+        Persistence::ActiveRuns.slow_suites(db).reject { |_, _, pid| exists?(pid) }.each do |job_id, run_id, _|
+          Persistence::ActiveRuns.slow_suite_died(db, job_id)
+          Persistence::RunStatus.settle(db, run_id)
+        end
+      rescue SQLite3::BusyException, *Persistence::WriteTrouble::UNWRITABLE
+        # The next poll looks again.
+      end
+
+      def stop(run)
+        return unless alive?(run)
+
+        (run.processes - [Process.pid] + run.stage_groups.map(&:-@)).each { |pid| kill(pid) }
+      end
+
+      private
+
+      # A stage its process started after `run` was read. A run's process
+      # records a stage's group before it lets the stage start, so once those
+      # processes are dead, looking again finds every stage they started.
+      def stop_late_stages(db, run)
+        Persistence::ActiveRuns.with_id(db, run.id).each do |now|
+          (now.stage_groups - run.stage_groups).each { |group| kill(-group) }
+        end
+      end
+
+      def alive?(run) = run.slot_lock.nil? || @slot_held.call(run.slot_lock)
+
+      # Whether a process with this pid exists; one this user may not signal does.
+      def exists?(pid)
+        @killer.call(0, pid)
+        true
+      rescue Errno::ESRCH
+        false
+      rescue Errno::EPERM
+        true
+      end
+
+      def kill(pid)
+        @killer.call("KILL", pid)
+      rescue Errno::ESRCH
+        # Gone already.
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/slot.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Pipeline
+    # A worktree a pipeline has to itself, held through an flock on its lock
+    # file. The fast suite and the slow suite each hold it; it frees when the
+    # last of them lets go. A process that dies lets go with it.
+    class Slot
+      attr_reader :path, :lock_file
+
+      # +lock_file+ names the lock on disk; nil for a slot with nothing to lock.
+      def initialize(path, lock, lock_file = nil)
+        @path = path
+        @lock = lock
+        @lock_file = lock_file
+        @holders = 1
+      end
+
+      def share
+        @holders += 1
+        self
+      end
+
+      def release
+        @holders -= 1
+        @lock.close if @holders.zero? && !@lock.closed?
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/slot_lock.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Pipeline
+    # Whether some process holds a slot's lock, which is whether the run that
+    # took the slot is still alive: an flock dies with its holders.
+    module SlotLock
+      def self.held?(path)
+        File.exist?(path) && File.open(path) { |lock| !lock.flock(File::LOCK_EX | File::LOCK_NB) }
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/slot_run.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require_relative "trigger_params"
+require_relative "background_fork"
+require_relative "stage_runner"
+require_relative "stage_execution"
+require_relative "progress_reporter"
+
+module FunCi
+  module Pipeline
+    # The stages of one pipeline, run in the slot it was given: lint and build
+    # side by side, then the slow suite in the background while the fast suite
+    # runs. The slot frees when the last stage holding it finishes.
+    class SlotRun
+      attr_reader :seams
+
+      def initialize(commit:, io:, seams:, slot:)
+        @commit = commit
+        @io = io
+        @seams = seams
+        @slot = slot
+      end
+
+      def run(config)
+        recorder.slot_taken(@slot.lock_file) if @slot.lock_file
+        return released(1) unless phase_one_passed?(config)
+
+        launch_slow_suite(config)
+        progress.slow_launched
+        released(fast_passed?(config) ? 0 : 1)
+      end
+
+      private
+
+      def recorder = @seams.recorder
+      def progress = ProgressReporter.new(stdout: @io.stdout)
+      def stage_runner = StageRunner.new(commit: @commit, stdout: @io.stdout, seams: @seams, dir: @slot.path)
+
+      # This process's part is over; the slow suite may still be running.
+      def released(exit_code)
+        @slot.release
+        recorder.foreground_done
+        exit_code
+      end
+
+      def phase_one_passed?(config)
+        results = %w[lint build].to_h { |stage| [stage, Thread.new { stage_runner.passes?(config, stage) }] }
+                                .transform_values(&:value)
+        progress.phase_one_result(results)
+        results.values.all?
+      end
+
+      def fast_passed?(config)
+        passed = stage_runner.passes?(config, "fast")
+        progress.fast_result(passed)
+        passed
+      end
+
+      def launch_slow_suite(config)
+        job_id = recorder.start_stage("slow", budget: @seams.budgets["slow"])
+        launch(db_path: recorder.db_path, pipeline_run_id: recorder.pipeline_run_id, job_id: job_id,
+               executor: slow_suite(config, @slot.share))
+      end
+
+      # Called with the recorder and job to record the slow suite with, in
+      # whichever process runs it; the slot is let go once it is recorded.
+      def slow_suite(config, slot)
+        execution = StageExecution.new(seams: @seams, dir: slot.path, commit: @commit)
+        command = config.stage_command("slow", @commit.sha)
+        ->(recorder, job_id) { holding(slot) { execution.run("slow", command, recorder, job_id) } }
+      end
+
+      def holding(slot)
+        yield
+      ensure
+        slot.release
+      end
+
+      def launch(**)
+        return @seams.background_launcher.call(**) if @seams.background_launcher
+
+        @seams = @seams.with(recorder: BackgroundFork.new(recorder, @slot).launch(**))
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/stage_dir.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require "tmpdir"
+require "fileutils"
+require_relative "output_window"
+require_relative "../persistence/state_dir"
+
+module FunCi
+  module Pipeline
+    # A stage's own directory, named for the process that made it, in the
+    # state directory: it holds the window its output is written to, unmasked,
+    # until the stage is recorded (AT-10.6), and what a project's extractor
+    # reads and writes.
+    class StageDir
+      def self.default_root = File.join(Persistence::StateDir.path(ENV), "stages")
+
+      # Removes the directories of processes that died before removing their own.
+      def self.create(root = default_root)
+        FileUtils.mkdir_p(root)
+        Dir.children(root).reject { |name| alive?(name.to_i) }.each { |name| FileUtils.rm_rf(File.join(root, name)) }
+        new(Dir.mktmpdir("#{Process.pid}-", root))
+      end
+
+      def self.alive?(pid)
+        Process.kill(0, pid)
+        true
+      rescue Errno::ESRCH
+        false
+      rescue Errno::EPERM
+        true
+      end
+      private_class_method :alive?
+
+      def initialize(path)
+        @path = path
+      end
+
+      # The output as the window kept it, written for a project's extractor to read.
+      def output_file(text)
+        File.join(@path, "output.log").tap { |file| File.binwrite(file, text) }
+      end
+
+      # Where a project's extractor writes what it needs to (its stdin and stderr).
+      def scratch = @path
+      def window(sizes = OutputWindow::REAL) = OutputWindow.in(@path, sizes)
+
+      def remove = FileUtils.rm_rf(@path)
+    end
+  end
+end

data/lib/fun_ci/pipeline/stage_end.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require_relative "../evidence/outcome"
+require_relative "../evidence/document"
+
+module FunCi
+  module Pipeline
+    # Records a stage that has finished, whichever process ran it: a failed
+    # stage's evidence first, so whoever sees the outcome can read why, then
+    # how it exited, then the outcome. Called while the stage still holds its
+    # slot and stage directory, which the collector reads.
+    class StageEnd
+      # What running a stage answered: its output, its Process::Status (nil
+      # when it was killed over budget), and for an overrun, what was found
+      # before the kill.
+      Finished = Data.define(:output, :status, :timed_out, :overrun)
+
+      class Finished
+        def initialize(overrun: nil, **) = super
+      end
+
+      # A failed stage's state in the words `status --json` uses.
+      STATES = { "failed" => "failed", "timed_out" => "over_budget" }.freeze
+
+      def initialize(recorder, job_id, collector)
+        @recorder = recorder
+        @job_id = job_id
+        @collector = collector
+      end
+
+      # Answers the outcome recorded: completed, failed or timed_out.
+      def record(finished)
+        result = outcome(finished)
+        keep_evidence(finished) unless result == "completed"
+        keep_exit(finished.status) if finished.status
+        @recorder.end_stage(@job_id, result)
+        result
+      end
+
+      private
+
+      def outcome(finished)
+        return "timed_out" if finished.timed_out
+
+        finished.status.success? ? "completed" : "failed"
+      end
+
+      # Whatever goes wrong collecting the evidence is kept as a problem, and
+      # the outcome is recorded all the same.
+      def keep_evidence(finished)
+        @recorder.keep_evidence(@job_id, @collector.collect(finished.output, ending(finished)))
+        @recorder.keep_raw(@job_id, @collector.masked(finished.output))
+      rescue StandardError => e
+        @recorder.keep_evidence(@job_id, Evidence::Document.broken("#{e.class}: #{e.message}"))
+      end
+
+      def ending(finished)
+        status = finished.status
+        Evidence::Outcome.new(state: STATES.fetch(outcome(finished)), exit_status: status&.exitstatus,
+                              signal: status&.termsig && Signal.signame(status.termsig),
+                              alongside: @recorder.alongside(@job_id), overrun: finished.overrun)
+      end
+
+      def keep_exit(status)
+        signal = status.termsig && Signal.signame(status.termsig)
+        @recorder.keep_exit(@job_id, status.exitstatus, signal)
+      end
+    end
+  end
+end