fun_ci 1.2.1 → 2.0.1

data/lib/fun_ci/pipeline/command_executor.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require "timeout"
+require_relative "process_runner"
+
+module FunCi
+  module Pipeline
+    # Runs a stage command within its budget and answers [output, status, timed_out].
+    # An injected command runner replaces the real process and signals a blown
+    # budget by raising Timeout::Error. A real process inherits fun-ci's own
+    # environment; an injected runner sees `environment` in its place.
+    class CommandExecutor
+      include ProcessRunner
+
+      def initialize(command_runner, dir = Dir.pwd, environment = {})
+        @command_runner = command_runner
+        @dir = dir
+        @environment = environment
+      end
+
+      # Yields the pid of the process the command runs in, when there is one.
+      # The launch's `env` is added to the command's environment; an injected
+      # runner that takes a second argument is given the whole of it. What
+      # the command prints is written to the launch's `output`, and answered
+      # as the window keeps it; its `before_kill` is called with the pid of
+      # a command over budget, while it still runs.
+      def call(cmd, budget, launch = Launch.new, &)
+        launch = launch.with(chdir: @dir)
+        return run_process_with_timeout(cmd, budget, launch: launch, &) unless @command_runner
+
+        injected(cmd, launch, &)
+      end
+
+      private
+
+      # An injected runner signals a blown budget by raising Timeout::Error,
+      # which is its moment before the kill.
+      def injected(cmd, launch)
+        started = nil
+        printed, status = run_injected(cmd, launch.env) { |pid| (started = pid) && (yield pid if block_given?) }
+        [text(launch.output << printed), status, false]
+      rescue Timeout::Error
+        launch.before_kill&.call(started)
+        ["", nil, true]
+      end
+
+      # A lambda's own, or those of an object's #call.
+      def parameters
+        @command_runner.respond_to?(:parameters) ? @command_runner.parameters : @command_runner.method(:call).parameters
+      end
+
+      def run_injected(cmd, env, &)
+        takes_env = parameters.count { |kind, _| %i[req opt].include?(kind) } > 1
+        takes_env ? @command_runner.call(cmd, @environment.merge(env), &) : @command_runner.call(cmd, &)
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/gate.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "io/nonblock"
+
+module FunCi
+  module Pipeline
+    # A pipe a child process waits on until the caller opens it. Ruby makes
+    # pipes non-blocking, and a child shares that setting with the parent, so
+    # the child's end is made blocking: otherwise a child reading before the
+    # caller has opened the gate fails instead of waiting.
+    class Gate
+      attr_reader :child_end
+
+      def self.create
+        child_end, opener = IO.pipe
+        child_end.nonblock = false
+        new(child_end, opener)
+      end
+
+      def initialize(child_end, opener)
+        @child_end = child_end
+        @opener = opener
+      end
+
+      # Lets the child through. A child that has gone already needs nothing.
+      def open
+        @opener.puts
+      rescue Errno::EPIPE
+        nil
+      ensure
+        @opener.close
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/git_environment.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Pipeline
+    # Git runs hooks with variables naming the repository and index that ran
+    # them. Passed on, they would point fun-ci's git commands and the stage
+    # scripts in a worktree at the checkout's index instead of their own.
+    module GitEnvironment
+      REPOSITORY = %w[GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE GIT_PREFIX GIT_COMMON_DIR GIT_OBJECT_DIRECTORY
+                      GIT_ALTERNATE_OBJECT_DIRECTORIES GIT_NAMESPACE].freeze
+
+      # An environment for spawn that unsets each of them.
+      CLEAN = REPOSITORY.to_h { |name| [name, nil] }.freeze
+    end
+  end
+end

data/lib/fun_ci/pipeline/head_part.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Pipeline
+    # The first `size` bytes of a stream, kept in an IO opened when first written.
+    class HeadPart
+      def initialize(open, size)
+        @open = open
+        @size = size
+        @closed = false
+      end
+
+      # Writes what fits, and answers the rest.
+      def take(bytes)
+        room = [@size - written, 0].max
+        io.write(bytes.byteslice(0, room)) if room.positive? && !bytes.empty?
+        bytes.byteslice(room..) || "".b
+      end
+
+      def read
+        return +"".b unless @io
+
+        @io.rewind
+        @io.read
+      end
+
+      def close
+        @io&.close
+        @closed = true
+      end
+
+      def closed? = @closed
+
+      private
+
+      def written = @io ? @io.size : 0
+      def io = @io ||= @open.call("head")
+    end
+  end
+end

data/lib/fun_ci/pipeline/in_place.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require_relative "slot"
+
+module FunCi
+  module Pipeline
+    # A workspace that is the project directory itself, with nothing to lock.
+    # For a root commit's null SHA, which no worktree can check out.
+    InPlace = Data.define(:path)
+
+    class InPlace
+      Unlocked = Struct.new(:closed?) { def close = nil }
+
+      def acquire(_sha) = Slot.new(path, Unlocked.new(false))
+    end
+  end
+end

data/lib/fun_ci/pipeline/output_window.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "stringio"
+require_relative "head_part"
+require_relative "tail_segments"
+
+module FunCi
+  module Pipeline
+    # What a stage printed, kept as its first `head` bytes and its last
+    # `tail` bytes, written as it comes so no more than a chunk is held in
+    # memory. Both cuts fall on line ends, unless a line is longer than the
+    # part it is in, and a marker line says how many bytes were dropped.
+    class OutputWindow
+      Sizes = Data.define(:head, :tail)
+      REAL = Sizes.new(head: 1_048_576, tail: 7_340_032)
+
+      def self.in_memory(sizes = REAL) = new(->(_name) { StringIO.new(+"".b) }, sizes)
+
+      def self.in(dir, sizes = REAL)
+        new(->(name) { File.open(File.join(dir, name), "w+b") }, sizes)
+      end
+
+      # `open` answers a new, empty read-write IO for a name.
+      def initialize(open, sizes)
+        @sizes = sizes
+        @written = 0
+        @head = HeadPart.new(open, sizes.head)
+        @tail = TailSegments.new(open, sizes.tail + 1)
+      end
+
+      def <<(chunk)
+        bytes = chunk.b
+        rest = @head.take(bytes)
+        @tail << rest unless rest.empty?
+        @written += bytes.bytesize
+        self
+      end
+
+      def text
+        head = @head.read
+        return head + @tail.read if @written <= @sizes.head + @sizes.tail
+
+        cut(head, @tail.read)
+      end
+
+      def close
+        @head.close
+        @tail.close
+      end
+
+      def closed? = @head.closed?
+
+      private
+
+      def cut(head, tail)
+        kept_head = head.rindex("\n") ? head[0..head.rindex("\n")] : head
+        kept_tail = line_start(tail)
+        dropped = @written - kept_head.bytesize - kept_tail.bytesize
+        "#{kept_head}[fun-ci: #{dropped} bytes dropped here]\n".b + kept_tail
+      end
+
+      # The tail holds one byte more than it keeps, to tell whether it starts
+      # on a line start. With no line start in it but after its last byte, it
+      # is the end of one long line, and is kept from mid-line.
+      def line_start(tail)
+        first_end = tail.index("\n")
+        return tail.byteslice(1..) if first_end.nil? || first_end.zero? || first_end == tail.bytesize - 1
+
+        tail.byteslice((first_end + 1)..)
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/pipeline_forker.rb

@@ -2,37 +2,39 @@
 
 require_relative "../persistence/database"
 require_relative "../persistence/pipeline_recorder"
-require_relative "background_wrapper"
+require_relative "trigger_params"
+require_relative "../setup/project_config"
 
 module FunCi
   module Pipeline
     class PipelineForker
+      # Answers whether it started a run: none in a project not set up for fun-ci.
       def self.fork_pipeline(commit_hash:, branch:, db_path:)
+        return false unless Setup::ProjectConfig.new(Dir.pwd).validate.empty?
+
         pid = fork do
           run_in_child(commit_hash: commit_hash, branch: branch, db_path: db_path)
         end
         Process.detach(pid)
       end
 
+      # Trigger requires this file, through TriggerCommand, so it is loaded
+      # here, where it is used, rather than above.
       def self.run_in_child(commit_hash:, branch:, db_path:)
-        db = Persistence::Database.connection(db_path)
-        recorder = Persistence::DbRecorder.new(db)
-        Trigger.new(
-          project_root: Dir.pwd,
-          commit_hash: commit_hash,
-          branch: branch,
-          stdout: File.open(File::NULL, "w"),
-          recorder: recorder,
-          background_launcher: method(:sync_launcher)
-        ).run
-        recorder.close
+        require_relative "trigger"
+        recorder = Persistence::DbRecorder.new(Persistence::Database.connection(db_path))
+        trigger = trigger(commit_hash, branch, recorder)
+        trigger.run
+        trigger.close
       end
 
-      def self.sync_launcher(db_path:, pipeline_run_id:, job_id:, executor:)
-        recorder = Persistence::DbRecorder.for_background(db_path, pipeline_run_id)
-        BackgroundWrapper.new(recorder: recorder, job_id: job_id, executor: executor).run
-        recorder.close
+      # The slow suite forks from here as it does in the foreground, so the
+      # fast suite, whose verdict a push waits for, runs beside it.
+      def self.trigger(commit_hash, branch, recorder)
+        Trigger.new(project: Dir.pwd, commit: Commit.new(sha: commit_hash, branch: branch),
+                    io: Io.new(stdout: File.open(File::NULL, "w")), seams: Seams.new(recorder: recorder))
       end
+      private_class_method :trigger
     end
   end
 end

data/lib/fun_ci/pipeline/process_runner.rb

@@ -1,27 +1,100 @@
 # frozen_string_literal: true
 
+require_relative "gate"
+require_relative "git_environment"
+require_relative "output_window"
+
 module FunCi
   module Pipeline
     module ProcessRunner
-      def run_process_with_timeout(cmd, budget)
-        r, w = IO.pipe
-        pid = Process.spawn(cmd, out: w, err: w, pgroup: true)
-        w.close
-
-        reader = Thread.new { r.read }
-
-        if reader.join(budget)
-          output = reader.value
-          _, status = Process.waitpid2(pid)
-          [output, status, false]
-        else
-          Process.kill("TERM", -pid) rescue nil
-          Process.kill("KILL", -pid) rescue nil
-          Process.waitpid(pid) rescue nil
-          ["", nil, true]
+      # The command waits on fd 9 until the caller has been told its pid, so a
+      # cancel that reads the recorded pid can never miss a stage that has
+      # started; exec keeps the pid and the process group. If the caller dies
+      # first, the pipe closes and the command never runs.
+      GATED = "read _ <&9 || exit 125; exec 9<&-; exec %s"
+
+      # Yields the pid of the process it starts, which leads a process group
+      # of its own, so the caller can stop the command and all it spawned.
+      # A command over budget answers what it printed before the kill, which
+      # is the only clue to what it was stuck on (acceptance-tests.md, AT-9.5).
+      # `timer` answers whether the reading finished within the budget.
+      BUDGET = ->(reading, budget) { reading.join(budget) }
+      # How long a killed command's last output may take to drain.
+      DRAIN_SECONDS = 1
+
+      # Where the command runs, what it finds in its environment besides
+      # fun-ci's own, the window what it prints is written to, and what to do
+      # with its pid when it runs over budget, before it is killed.
+      Launch = Data.define(:chdir, :env, :output, :before_kill)
+
+      class Launch
+        def initialize(**given)
+          super(chdir: Dir.pwd, env: {}, output: OutputWindow.in_memory, before_kill: nil, **given)
         end
+      end
+
+      def run_process_with_timeout(cmd, budget, launch: Launch.new, timer: BUDGET, &)
+        reader, writer = IO.pipe
+        pid = start(cmd, writer, launch, &)
+        printed = launch.output
+        reading = Thread.new { read_until_closed(reader, printed) }
+        timer.call(reading, budget) ? process_finished(pid, text(reading.value)) : over_budget(pid, reading, launch)
       ensure
-        r&.close rescue nil
+        ignoring_errors { reader&.close }
+      end
+
+      private
+
+      def start(cmd, writer, launch)
+        gate = Gate.create
+        pid = spawn_gated(cmd, writer, gate, launch)
+        [writer, gate.child_end].each(&:close)
+        yield pid if block_given?
+        gate.open
+        pid
+      end
+
+      def spawn_gated(cmd, writer, gate, launch)
+        Process.spawn(GitEnvironment::CLEAN.merge(launch.env), format(GATED, cmd),
+                      out: writer, err: writer, 9 => gate.child_end, pgroup: true, chdir: launch.chdir)
+      end
+
+      # The reader may be closed while this thread still reads, once a killed
+      # command's output has drained or stopped coming.
+      def read_until_closed(reader, printed)
+        loop { printed << reader.readpartial(65_536) }
+      rescue IOError
+        printed
+      end
+
+      # What looks at the stage before the kill does so while its output is still read.
+      def over_budget(pid, reading, launch)
+        launch.before_kill&.call(pid)
+        kill_process_group(pid)
+        [drained(reading, launch.output), nil, true]
+      end
+
+      def drained(reading, printed)
+        reading.join(DRAIN_SECONDS)
+        text(printed)
+      end
+
+      def text(window) = window.text.force_encoding(Encoding.default_external)
+
+      def process_finished(pid, output)
+        _, status = Process.waitpid2(pid)
+        [output, status, false]
+      end
+
+      def kill_process_group(pid)
+        %w[TERM KILL].each { |signal| ignoring_errors { Process.kill(signal, -pid) } }
+        ignoring_errors { Process.waitpid(pid) }
+      end
+
+      def ignoring_errors
+        yield
+      rescue StandardError
+        # A process that already exited or was already reaped needs nothing.
       end
     end
   end

data/lib/fun_ci/pipeline/run_canceller.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require_relative "../persistence/active_runs"
+require_relative "../persistence/run_status"
+require_relative "../persistence/write_trouble"
+require_relative "slot_lock"
+
+module FunCi
+  module Pipeline
+    # Stops every process of a run. The run's own processes go first, so none
+    # of them records the run as failed once its stages die; then each stage's
+    # process group. A run whose slot nobody holds any more has died already,
+    # and the pids it recorded may belong to other processes now, so it is
+    # left alone; so is this process.
+    class RunCanceller
+      def initialize(killer: Process.method(:kill), slot_held: SlotLock.method(:held?))
+        @killer = killer
+        @slot_held = slot_held
+      end
+
+      # Stops the run and records it, and its unfinished stages, cancelled.
+      def cancel(db, run)
+        stop(run) && stop_late_stages(db, run)
+        Persistence::ActiveRuns.cancelled(db, run)
+      end
+
+      # Records failed each slow stage whose forked process is gone without
+      # having recorded its result (acceptance-tests.md, AT-8.3), and settles
+      # its run.
+      def record_dead(db)
+        Persistence::ActiveRuns.slow_suites(db).reject { |_, _, pid| exists?(pid) }.each do |job_id, run_id, _|
+          Persistence::ActiveRuns.slow_suite_died(db, job_id)
+          Persistence::RunStatus.settle(db, run_id)
+        end
+      rescue SQLite3::BusyException, *Persistence::WriteTrouble::UNWRITABLE
+        # The next poll looks again.
+      end
+
+      def stop(run)
+        return unless alive?(run)
+
+        (run.processes - [Process.pid] + run.stage_groups.map(&:-@)).each { |pid| kill(pid) }
+      end
+
+      private
+
+      # A stage its process started after `run` was read. A run's process
+      # records a stage's group before it lets the stage start, so once those
+      # processes are dead, looking again finds every stage they started.
+      def stop_late_stages(db, run)
+        Persistence::ActiveRuns.with_id(db, run.id).each do |now|
+          (now.stage_groups - run.stage_groups).each { |group| kill(-group) }
+        end
+      end
+
+      def alive?(run) = run.slot_lock.nil? || @slot_held.call(run.slot_lock)
+
+      # Whether a process with this pid exists; one this user may not signal does.
+      def exists?(pid)
+        @killer.call(0, pid)
+        true
+      rescue Errno::ESRCH
+        false
+      rescue Errno::EPERM
+        true
+      end
+
+      def kill(pid)
+        @killer.call("KILL", pid)
+      rescue Errno::ESRCH
+        # Gone already.
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/slot.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Pipeline
+    # A worktree a pipeline has to itself, held through an flock on its lock
+    # file. The fast suite and the slow suite each hold it; it frees when the
+    # last of them lets go. A process that dies lets go with it.
+    class Slot
+      attr_reader :path, :lock_file
+
+      # +lock_file+ names the lock on disk; nil for a slot with nothing to lock.
+      def initialize(path, lock, lock_file = nil)
+        @path = path
+        @lock = lock
+        @lock_file = lock_file
+        @holders = 1
+      end
+
+      def share
+        @holders += 1
+        self
+      end
+
+      def release
+        @holders -= 1
+        @lock.close if @holders.zero? && !@lock.closed?
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/slot_lock.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Pipeline
+    # Whether some process holds a slot's lock, which is whether the run that
+    # took the slot is still alive: an flock dies with its holders.
+    module SlotLock
+      def self.held?(path)
+        File.exist?(path) && File.open(path) { |lock| !lock.flock(File::LOCK_EX | File::LOCK_NB) }
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/slot_run.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require_relative "trigger_params"
+require_relative "background_fork"
+require_relative "stage_runner"
+require_relative "stage_execution"
+require_relative "progress_reporter"
+
+module FunCi
+  module Pipeline
+    # The stages of one pipeline, run in the slot it was given: lint and build
+    # side by side, then the slow suite in the background while the fast suite
+    # runs. The slot frees when the last stage holding it finishes.
+    class SlotRun
+      attr_reader :seams
+
+      def initialize(commit:, io:, seams:, slot:)
+        @commit = commit
+        @io = io
+        @seams = seams
+        @slot = slot
+      end
+
+      def run(config)
+        recorder.slot_taken(@slot.lock_file) if @slot.lock_file
+        return released(1) unless phase_one_passed?(config)
+
+        launch_slow_suite(config)
+        progress.slow_launched
+        released(fast_passed?(config) ? 0 : 1)
+      end
+
+      private
+
+      def recorder = @seams.recorder
+      def progress = ProgressReporter.new(stdout: @io.stdout)
+      def stage_runner = StageRunner.new(commit: @commit, stdout: @io.stdout, seams: @seams, dir: @slot.path)
+
+      # This process's part is over; the slow suite may still be running.
+      def released(exit_code)
+        @slot.release
+        recorder.foreground_done
+        exit_code
+      end
+
+      def phase_one_passed?(config)
+        results = %w[lint build].to_h { |stage| [stage, Thread.new { stage_runner.passes?(config, stage) }] }
+                                .transform_values(&:value)
+        progress.phase_one_result(results)
+        results.values.all?
+      end
+
+      def fast_passed?(config)
+        passed = stage_runner.passes?(config, "fast")
+        progress.fast_result(passed)
+        passed
+      end
+
+      def launch_slow_suite(config)
+        job_id = recorder.start_stage("slow", budget: @seams.budgets["slow"])
+        launch(db_path: recorder.db_path, pipeline_run_id: recorder.pipeline_run_id, job_id: job_id,
+               executor: slow_suite(config, @slot.share))
+      end
+
+      # Called with the recorder and job to record the slow suite with, in
+      # whichever process runs it; the slot is let go once it is recorded.
+      def slow_suite(config, slot)
+        execution = StageExecution.new(seams: @seams, dir: slot.path, commit: @commit)
+        command = config.stage_command("slow", @commit.sha)
+        ->(recorder, job_id) { holding(slot) { execution.run("slow", command, recorder, job_id) } }
+      end
+
+      def holding(slot)
+        yield
+      ensure
+        slot.release
+      end
+
+      def launch(**)
+        return @seams.background_launcher.call(**) if @seams.background_launcher
+
+        @seams = @seams.with(recorder: BackgroundFork.new(recorder, @slot).launch(**))
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/stage_dir.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require "tmpdir"
+require "fileutils"
+require_relative "output_window"
+require_relative "../persistence/state_dir"
+
+module FunCi
+  module Pipeline
+    # A stage's own directory, named for the process that made it, in the
+    # state directory: it holds the window its output is written to, unmasked,
+    # until the stage is recorded (AT-10.6), and what a project's extractor
+    # reads and writes.
+    class StageDir
+      def self.default_root = File.join(Persistence::StateDir.path(ENV), "stages")
+
+      # Removes the directories of processes that died before removing their own.
+      def self.create(root = default_root)
+        FileUtils.mkdir_p(root)
+        Dir.children(root).reject { |name| alive?(name.to_i) }.each { |name| FileUtils.rm_rf(File.join(root, name)) }
+        new(Dir.mktmpdir("#{Process.pid}-", root))
+      end
+
+      def self.alive?(pid)
+        Process.kill(0, pid)
+        true
+      rescue Errno::ESRCH
+        false
+      rescue Errno::EPERM
+        true
+      end
+      private_class_method :alive?
+
+      def initialize(path)
+        @path = path
+      end
+
+      # The output as the window kept it, written for a project's extractor to read.
+      def output_file(text)
+        File.join(@path, "output.log").tap { |file| File.binwrite(file, text) }
+      end
+
+      # Where a project's extractor writes what it needs to (its stdin and stderr).
+      def scratch = @path
+      def window(sizes = OutputWindow::REAL) = OutputWindow.in(@path, sizes)
+
+      def remove = FileUtils.rm_rf(@path)
+    end
+  end
+end