fun_ci 1.0.0 → 1.2.0

data/lib/fun_ci/persistence/database.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "sqlite3"
+
+module FunCi
+  module Persistence
+    module Database
+      def self.connection(db_path)
+        db = SQLite3::Database.new(db_path)
+        db.busy_timeout = 5000
+        db.execute("PRAGMA journal_mode=WAL")
+        db.execute("PRAGMA foreign_keys=ON")
+        db
+      end
+
+      def self.migrate!(db)
+        db.execute(<<~SQL)
+          CREATE TABLE IF NOT EXISTS pipeline_runs (
+            id INTEGER PRIMARY KEY,
+            commit_hash TEXT,
+            branch TEXT,
+            status TEXT DEFAULT 'scheduled',
+            pid INTEGER,
+            created_at TEXT,
+            updated_at TEXT
+          )
+        SQL
+
+        add_column_if_missing(db, "pipeline_runs", "pid", "INTEGER")
+        add_column_if_missing(db, "pipeline_runs", "project_path", "TEXT")
+
+        db.execute(<<~SQL)
+          CREATE TABLE IF NOT EXISTS stage_jobs (
+            id INTEGER PRIMARY KEY,
+            pipeline_run_id INTEGER REFERENCES pipeline_runs(id),
+            stage TEXT,
+            status TEXT DEFAULT 'scheduled',
+            started_at TEXT,
+            completed_at TEXT
+          )
+        SQL
+      end
+
+      def self.add_column_if_missing(db, table, column, type)
+        columns = db.execute("PRAGMA table_info(#{table})").map { |row| row[1] }
+        return if columns.include?(column)
+        db.execute("ALTER TABLE #{table} ADD COLUMN #{column} #{type}")
+      end
+      private_class_method :add_column_if_missing
+    end
+  end
+end

data/lib/fun_ci/persistence/pipeline_recorder.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require_relative "pipeline_run"
+require_relative "stage_job"
+
+module FunCi
+  module Persistence
+    class NullRecorder
+      def create_run(commit_hash:, branch:, project_path: nil) = nil
+      def start_stage(stage) = nil
+      def end_stage(job_id, status) = nil
+      def complete_run = nil
+      def fail_run = nil
+      def db = nil
+      def db_path = nil
+      def pipeline_run_id = nil
+      def close = nil
+    end
+
+    class DbRecorder
+      attr_reader :db, :db_path, :pipeline_run_id
+
+      def self.for_background(db_path, pipeline_run_id)
+        db = Database.connection(db_path)
+        new(db, pipeline_run_id: pipeline_run_id)
+      end
+
+      def initialize(db, pipeline_run_id: nil)
+        @db = db
+        @db_path = db.filename("main")
+        @pipeline_run_id = pipeline_run_id
+      end
+
+      def close
+        @db.close rescue nil
+      end
+
+      def create_run(commit_hash:, branch:, project_path: nil)
+        @pipeline_run_id = PipelineRun.create(@db, commit_hash: commit_hash, branch: branch, project_path: project_path)
+      end
+
+      def start_stage(stage)
+        return nil unless @pipeline_run_id
+        ensure_running
+        job_id = StageJob.create(@db, pipeline_run_id: @pipeline_run_id, stage: stage)
+        StageJob.update_status(@db, job_id, "running")
+        job_id
+      end
+
+      def end_stage(job_id, status)
+        return unless job_id
+        StageJob.update_status(@db, job_id, status)
+      end
+
+      def complete_run
+        return unless @pipeline_run_id
+        PipelineRun.update_status(@db, @pipeline_run_id, "completed")
+      end
+
+      def fail_run
+        return unless @pipeline_run_id
+        PipelineRun.update_status(@db, @pipeline_run_id, "failed")
+      end
+
+      private
+
+      def ensure_running
+        run = PipelineRun.find(@db, @pipeline_run_id)
+        return unless run && run[:status] == "scheduled"
+        PipelineRun.update_status(@db, @pipeline_run_id, "running")
+      end
+    end
+  end
+end

data/lib/fun_ci/persistence/pipeline_run.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "time"
+
+module FunCi
+  module Persistence
+    module PipelineRun
+      def self.create(db, commit_hash:, branch:, project_path: nil)
+        now = Time.now.utc.iso8601
+        db.execute(
+          "INSERT INTO pipeline_runs (commit_hash, branch, project_path, status, created_at, updated_at) VALUES (?, ?, ?, 'scheduled', ?, ?)",
+          [commit_hash, branch, project_path, now, now]
+        )
+        db.last_insert_row_id
+      end
+
+      def self.find(db, id)
+        row = db.execute("SELECT id, commit_hash, branch, status, pid, project_path, created_at, updated_at FROM pipeline_runs WHERE id = ?", [id]).first
+        return nil unless row
+        row_to_hash(row)
+      end
+
+      def self.find_by_branch(db, branch)
+        rows = db.execute("SELECT id, commit_hash, branch, status, pid, project_path, created_at, updated_at FROM pipeline_runs WHERE branch = ? ORDER BY id DESC", [branch])
+        rows.map { |row| row_to_hash(row) }
+      end
+
+      def self.find_by_commit(db, commit_hash)
+        rows = db.execute("SELECT id, commit_hash, branch, status, pid, project_path, created_at, updated_at FROM pipeline_runs WHERE commit_hash = ? ORDER BY id DESC", [commit_hash])
+        rows.map { |row| row_to_hash(row) }
+      end
+
+      def self.store_pid(db, id, pid)
+        db.execute("UPDATE pipeline_runs SET pid = ? WHERE id = ?", [pid, id])
+      end
+
+      def self.find_running_with_pid(db, branch)
+        row = db.execute("SELECT id, commit_hash, branch, status, pid, project_path, created_at, updated_at FROM pipeline_runs WHERE branch = ? AND status = 'running' AND pid IS NOT NULL ORDER BY id DESC LIMIT 1", [branch]).first
+        return nil unless row
+        row_to_hash(row)
+      end
+
+      def self.update_status(db, id, new_status)
+        now = Time.now.utc.iso8601
+        db.execute("UPDATE pipeline_runs SET status = ?, updated_at = ? WHERE id = ?", [new_status, now, id])
+      end
+
+      def self.recent(db, limit: 20)
+        rows = db.execute("SELECT id, commit_hash, branch, status, pid, project_path, created_at, updated_at FROM pipeline_runs ORDER BY id DESC LIMIT ?", [limit])
+        rows.map { |row| row_to_hash(row) }
+      end
+
+      def self.row_to_hash(row)
+        { id: row[0], commit_hash: row[1], branch: row[2], status: row[3], pid: row[4], project_path: row[5], created_at: row[6], updated_at: row[7] }
+      end
+      private_class_method :row_to_hash
+    end
+  end
+end

data/lib/fun_ci/persistence/stage_job.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "time"
+
+module FunCi
+  module Persistence
+    module StageJob
+      TERMINAL_STATUSES = %w[completed failed timed_out cancelled].freeze
+
+      def self.create(db, pipeline_run_id:, stage:)
+        db.execute(
+          "INSERT INTO stage_jobs (pipeline_run_id, stage, status) VALUES (?, ?, 'scheduled')",
+          [pipeline_run_id, stage]
+        )
+        db.last_insert_row_id
+      end
+
+      def self.find(db, id)
+        row = db.execute("SELECT id, pipeline_run_id, stage, status, started_at, completed_at FROM stage_jobs WHERE id = ?", [id]).first
+        return nil unless row
+        row_to_hash(row)
+      end
+
+      def self.update_status(db, id, new_status)
+        now = Time.now.utc.iso8601
+        if new_status == "running"
+          db.execute("UPDATE stage_jobs SET status = ?, started_at = ? WHERE id = ?", [new_status, now, id])
+        elsif TERMINAL_STATUSES.include?(new_status)
+          db.execute("UPDATE stage_jobs SET status = ?, completed_at = ? WHERE id = ?", [new_status, now, id])
+        else
+          db.execute("UPDATE stage_jobs SET status = ? WHERE id = ?", [new_status, id])
+        end
+      end
+
+      def self.elapsed_duration(job)
+        return nil unless job[:started_at] && job[:completed_at]
+        Time.parse(job[:completed_at]) - Time.parse(job[:started_at])
+      end
+
+      def self.row_to_hash(row)
+        { id: row[0], pipeline_run_id: row[1], stage: row[2], status: row[3], started_at: row[4], completed_at: row[5] }
+      end
+      private_class_method :row_to_hash
+    end
+  end
+end

data/lib/fun_ci/{state_machine.rb → persistence/state_machine.rb}

@@ -1,18 +1,8 @@
 # frozen_string_literal: true
 
 module FunCi
-  # Enforced state machine for pipeline runs and stage jobs.
-  #
-  # States: scheduled, running, completed, failed, timed_out, cancelled
-  #
-  # Valid transitions:
-  #   scheduled -> running
-  #   scheduled -> cancelled
-  #   running   -> completed
-  #   running   -> failed
-  #   running   -> timed_out
-  #   running   -> cancelled
-  class StateMachine
+  module Persistence
+    class StateMachine
     class InvalidTransition < StandardError; end
 
     STATES = %i[scheduled running completed failed timed_out cancelled].freeze
@@ -55,5 +45,6 @@ module FunCi
 
       raise ArgumentError, "Unknown state: #{state}. Valid states: #{STATES.join(", ")}"
     end
+    end
   end
 end

data/lib/fun_ci/pipeline/background_wrapper.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Pipeline
+    class BackgroundWrapper
+      def initialize(recorder:, job_id:, executor:)
+        @recorder = recorder
+        @job_id = job_id
+        @executor = executor
+      end
+
+      def run
+        _output, status, timed_out = @executor.call
+        if timed_out
+          @recorder.end_stage(@job_id, "timed_out")
+          @recorder.fail_run
+        elsif status.success?
+          @recorder.end_stage(@job_id, "completed")
+          @recorder.complete_run
+        else
+          @recorder.end_stage(@job_id, "failed")
+          @recorder.fail_run
+        end
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/pipeline_forker.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require_relative "../persistence/database"
+require_relative "../persistence/pipeline_recorder"
+require_relative "background_wrapper"
+
+module FunCi
+  module Pipeline
+    class PipelineForker
+      def self.fork_pipeline(commit_hash:, branch:, db_path:)
+        pid = fork do
+          run_in_child(commit_hash: commit_hash, branch: branch, db_path: db_path)
+        end
+        Process.detach(pid)
+      end
+
+      def self.run_in_child(commit_hash:, branch:, db_path:)
+        db = Persistence::Database.connection(db_path)
+        recorder = Persistence::DbRecorder.new(db)
+        Trigger.new(
+          project_root: Dir.pwd,
+          commit_hash: commit_hash,
+          branch: branch,
+          stdout: File.open(File::NULL, "w"),
+          recorder: recorder,
+          background_launcher: method(:sync_launcher)
+        ).run
+        recorder.close
+      end
+
+      def self.sync_launcher(db_path:, pipeline_run_id:, job_id:, executor:)
+        recorder = Persistence::DbRecorder.for_background(db_path, pipeline_run_id)
+        BackgroundWrapper.new(recorder: recorder, job_id: job_id, executor: executor).run
+        recorder.close
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/process_runner.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Pipeline
+    module ProcessRunner
+      def run_process_with_timeout(cmd, budget)
+        r, w = IO.pipe
+        pid = Process.spawn(cmd, out: w, err: w, pgroup: true)
+        w.close
+
+        reader = Thread.new { r.read }
+
+        if reader.join(budget)
+          output = reader.value
+          _, status = Process.waitpid2(pid)
+          [output, status, false]
+        else
+          Process.kill("TERM", -pid) rescue nil
+          Process.kill("KILL", -pid) rescue nil
+          Process.waitpid(pid) rescue nil
+          ["", nil, true]
+        end
+      ensure
+        r&.close rescue nil
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/progress_reporter.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Pipeline
+    class ProgressReporter
+      def initialize(stdout:)
+        @stdout = stdout
+      end
+
+      def phase_one_result(results)
+        markers = results.map { |stage, passed| "#{stage} #{result_marker(passed)}" }
+        summary = results.values.all? ? "phase 1 passed" : "phase 1 failed"
+        @stdout.puts "fun-ci: #{markers.join("  ")}  (#{summary})"
+      end
+
+      def fast_result(passed)
+        @stdout.puts "fun-ci: fast #{result_marker(passed)}"
+      end
+
+      def slow_launched
+        @stdout.puts "fun-ci: slow (running in background)"
+      end
+
+      private
+
+      def result_marker(passed)
+        passed ? "ok" : "FAIL"
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/stage_runner.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require "timeout"
+require_relative "process_runner"
+
+module FunCi
+  module Pipeline
+    class StageRunner
+      include ProcessRunner
+
+      def initialize(commit_hash:, stdout:, command_runner: nil, time_budgets: {}, recorder: FunCi::NullRecorder.new)
+        @commit_hash = commit_hash
+        @stdout = stdout
+        @command_runner = command_runner
+        @time_budgets = time_budgets
+        @recorder = recorder
+      end
+
+      def run_stage(config, stage)
+        script = config.script_path(stage)
+        budget = @time_budgets[stage]
+
+        job_id = @recorder.start_stage(stage)
+        output, status, timed_out = run_with_timeout(script, budget)
+
+        if timed_out
+          @recorder.end_stage(job_id, "timed_out")
+          @stdout.puts "#{stage_label(stage)} killed -- exceeded #{budget}s time budget."
+          @stdout.puts budget_advice(stage)
+          return false
+        end
+
+        unless status.success?
+          @recorder.end_stage(job_id, "failed")
+          @stdout.puts output unless output.empty?
+          @stdout.puts "#{stage_label(stage)} failed."
+          return false
+        end
+
+        @recorder.end_stage(job_id, "completed")
+        true
+      end
+
+      private
+
+      def run_with_timeout(script, budget)
+        cmd = "#{script} #{@commit_hash}"
+
+        if @command_runner
+          begin
+            output, status = @command_runner.call(cmd)
+            [output, status, false]
+          rescue Timeout::Error
+            ["", nil, true]
+          end
+        else
+          run_process_with_timeout(cmd, budget)
+        end
+      end
+
+      def stage_label(stage)
+        case stage
+        when "lint" then "Lint"
+        when "build" then "Build"
+        when "fast" then "Fast suite"
+        when "slow" then "Slow suite"
+        else stage
+        end
+      end
+
+      def budget_advice(stage)
+        case stage
+        when "lint"
+          "Trim your linter config or split into stages."
+        when "build"
+          "Keep your build efficient and not let it become a bottleneck."
+        when "fast"
+          "Your fast tests have gotten too slow. Split or speed them up."
+        when "slow"
+          "Pare down integration tests, parallelise, or raise the budget."
+        end
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/stale_pipeline_canceller.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require_relative "../persistence/pipeline_run"
+
+module FunCi
+  module Pipeline
+    class StalePipelineCanceller
+      def initialize(db:, branch:, stdout:, process_killer: nil)
+        @db = db
+        @branch = branch
+        @stdout = stdout
+        @process_killer = process_killer || method(:default_process_killer)
+      end
+
+      def cancel(new_commit_hash:)
+        run = Persistence::PipelineRun.find_running_with_pid(@db, @branch)
+        return unless run
+
+        kill_process(run[:pid])
+        Persistence::PipelineRun.update_status(@db, run[:id], "cancelled")
+        @stdout.puts "Cancelled stale pipeline for #{run[:commit_hash]}. Starting fresh for #{new_commit_hash}."
+      end
+
+      def store_pid(run_id, pid)
+        Persistence::PipelineRun.store_pid(@db, run_id, pid)
+      end
+
+      private
+
+      def kill_process(pid)
+        @process_killer.call(0, pid)
+        safe_signal("TERM", pid)
+        safe_signal("KILL", pid)
+        begin
+          Process.waitpid(pid)
+        rescue Errno::ECHILD, Errno::ESRCH # rubocop:disable Lint/SuppressedException
+        end
+      rescue Errno::ESRCH
+        # Process already dead
+      end
+
+      def safe_signal(signal, pid)
+        @process_killer.call(signal, pid)
+      rescue Errno::ESRCH
+        # Already dead
+      end
+
+      def default_process_killer(signal, pid)
+        Process.kill(signal, pid)
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/trigger.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+require "open3"
+require_relative "../setup/project_config"
+require_relative "../persistence/pipeline_recorder"
+require_relative "background_wrapper"
+require_relative "stage_runner"
+require_relative "stale_pipeline_canceller"
+require_relative "progress_reporter"
+require_relative "pipeline_forker"
+require_relative "process_runner"
+
+module FunCi
+  module Pipeline
+    class Trigger
+      include ProcessRunner
+      NULL_SHA = ("0" * 40).freeze
+      DEFAULT_BUDGETS = { "lint" => 30, "build" => 30, "fast" => 10, "slow" => 300 }.freeze
+
+      def self.run_from_args(args, stdout: $stdout, stderr: $stderr, recorder: Persistence::NullRecorder.new, pipeline_forker: nil)
+        positional = args.reject { |a| a.start_with?("--") }
+        if positional.length < 2
+          stderr.puts "fun-ci: commit hash and branch name are required."
+          stderr.puts "Usage: fun-ci trigger <commit-hash> <branch>"
+          return 1
+        end
+        commit_hash, branch = positional
+        if args.include?("--no-validate")
+          db_path = recorder.db_path
+          recorder.close
+          (pipeline_forker || PipelineForker.method(:fork_pipeline)).call(
+            commit_hash: commit_hash, branch: branch, db_path: db_path
+          )
+          return 0
+        end
+        new(project_root: Dir.pwd, commit_hash: commit_hash, branch: branch,
+            stdout: stdout, stderr: stderr, recorder: recorder).run
+      end
+
+      attr_writer :command_runner
+
+      def initialize(project_root:, commit_hash:, branch:, stdout: $stdout, stderr: $stderr, command_runner: nil, time_budgets: {}, commit_validator: nil, recorder: Persistence::NullRecorder.new, background_launcher: nil)
+        @project_root = project_root
+        @commit_hash = commit_hash
+        @branch = branch
+        @stdout = stdout
+        @stderr = stderr
+        @command_runner = command_runner
+        @time_budgets = DEFAULT_BUDGETS.merge(time_budgets)
+        @commit_validator = commit_validator || method(:default_commit_validator)
+        @recorder = recorder
+        @background_launcher = background_launcher || method(:default_background_launcher)
+      end
+
+      def run
+        config = Setup::ProjectConfig.new(@project_root)
+        return handle_config_errors(config) if config.validate.any?
+        unless @commit_hash == NULL_SHA || @commit_validator.call(@commit_hash)
+          @stderr.puts "fun-ci: commit #{@commit_hash} not found in this repository."
+          return 1
+        end
+        cancel_stale_pipelines
+        @recorder.create_run(commit_hash: @commit_hash, branch: @branch, project_path: @project_root)
+        stage_runner = make_stage_runner
+        progress = ProgressReporter.new(stdout: @stdout)
+        results = run_phase_one(stage_runner, config)
+        progress.phase_one_result(results)
+        unless results.values.all?
+          @recorder.fail_run
+          return 1
+        end
+        spawn_slow_suite(config)
+        progress.slow_launched
+        fast_runner = make_stage_runner
+        fast_passed = fast_runner.run_stage(config, "fast")
+        progress.fast_result(fast_passed)
+        unless fast_passed
+          @recorder.fail_run
+          return 1
+        end
+        0
+      end
+
+      private
+
+      def handle_config_errors(config)
+        config.validate.each { |e| @stdout.puts "fun-ci: #{e}" }
+        unless config.folder_exists?
+          @stdout.puts "Create .fun-ci/lint.sh, build.sh, fast.sh, and slow.sh to set up this project."
+          @stdout.puts "Commit will proceed without CI."
+        end
+        0
+      end
+
+      def run_phase_one(stage_runner, config)
+        results = {}
+        threads = %w[lint build].map do |stage|
+          Thread.new { results[stage] = stage_runner.run_stage(config, stage) }
+        end
+        threads.each(&:join)
+        results
+      end
+
+      def spawn_slow_suite(config)
+        cmd = "#{config.script_path("slow")} #{@commit_hash}"
+        job_id = @recorder.start_stage("slow")
+        budget = @time_budgets["slow"]
+        executor = if @command_runner
+          runner = @command_runner
+          -> do
+            output, status = runner.call(cmd)
+            [output, status, false]
+          rescue Timeout::Error
+            ["", nil, true]
+          end
+        else
+          -> { run_process_with_timeout(cmd, budget) }
+        end
+        @background_launcher.call(
+          db_path: @recorder.db_path, pipeline_run_id: @recorder.pipeline_run_id,
+          job_id: job_id, executor: executor
+        )
+      end
+
+      def make_stage_runner
+        StageRunner.new(commit_hash: @commit_hash, stdout: @stdout,
+          command_runner: @command_runner, time_budgets: @time_budgets, recorder: @recorder)
+      end
+
+      def cancel_stale_pipelines
+        return unless @recorder.db
+        StalePipelineCanceller.new(db: @recorder.db, branch: @branch, stdout: @stdout)
+          .cancel(new_commit_hash: @commit_hash)
+      end
+
+      def default_background_launcher(db_path:, pipeline_run_id:, job_id:, executor:)
+        @recorder.close
+        pid = fork do
+          recorder = Persistence::DbRecorder.for_background(db_path, pipeline_run_id)
+          BackgroundWrapper.new(recorder: recorder, job_id: job_id, executor: executor).run
+          recorder.close
+        end
+        @recorder = Persistence::DbRecorder.for_background(db_path, pipeline_run_id)
+        Process.detach(pid)
+        Persistence::PipelineRun.store_pid(@recorder.db, pipeline_run_id, pid)
+      end
+
+      def default_commit_validator(hash)
+        Open3.capture2e("git", "cat-file", "-t", hash).last.success?
+      end
+    end
+  end
+end