fun_ci 1.2.1 → 2.0.0

data/lib/fun_ci/pipeline/trigger_command.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require_relative "trigger_params"
+require_relative "pipeline_forker"
+
+module FunCi
+  module Pipeline
+    # `fun-ci trigger [--background] <commit-hash> <branch>`. --no-validate is
+    # the 1.x name for --background, kept for one release.
+    class TriggerCommand
+      BACKGROUND = %w[--background --no-validate].freeze
+      DEPRECATED = "fun-ci: --no-validate is now --background; the old name goes in 2.1."
+
+      def initialize(io:, recorder:, pipeline_forker:, project: Dir.pwd)
+        @io = io
+        @recorder = recorder
+        @pipeline_forker = pipeline_forker || PipelineForker.method(:fork_pipeline)
+        @project = project
+      end
+
+      # Closes the database connection it was given on every path.
+      def run(args)
+        sha, branch = args.reject { |a| a.start_with?("--") }
+        return usage unless branch
+
+        @io.stderr.puts DEPRECATED if args.include?("--no-validate")
+        commit = Commit.new(sha: sha, branch: branch)
+        args.intersect?(BACKGROUND) ? fork_pipeline(commit) : run_pipeline(commit)
+      ensure
+        @recorder.close
+      end
+
+      private
+
+      # After forking the slow suite the trigger holds a recorder of its own.
+      def run_pipeline(commit)
+        trigger = Trigger.new(project: @project, commit: commit, io: @io, seams: Seams.new(recorder: @recorder))
+        trigger.run
+      ensure
+        trigger&.close
+      end
+
+      def usage
+        @io.stderr.puts "fun-ci: commit hash and branch name are required."
+        @io.stderr.puts "Usage: fun-ci trigger <commit-hash> <branch>"
+        1
+      end
+
+      # The forker answers whether it started a run; one that did says how an
+      # agent gets its verdict (acceptance-tests.md, AT-9.14).
+      def fork_pipeline(commit)
+        db_path = @recorder.db_path
+        @recorder.close
+        started = @pipeline_forker.call(commit_hash: commit.sha, branch: commit.branch, db_path: db_path)
+        say_how_to_wait(commit.sha[0, 7]) if started
+        0
+      end
+
+      def say_how_to_wait(sha) = @io.stdout.puts("fun-ci: testing #{sha}. Verdict: fun-ci wait #{sha} --need all")
+    end
+  end
+end

data/lib/fun_ci/pipeline/trigger_params.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require "open3"
+require_relative "../persistence/pipeline_recorder"
+require_relative "command_executor"
+require_relative "stage_dir"
+require_relative "git_environment"
+require_relative "../evidence/command_runner"
+require_relative "../evidence/process_table"
+
+module FunCi
+  module Pipeline
+    DEFAULT_BUDGETS = { "lint" => 30, "build" => 30, "fast" => 10, "slow" => 300 }.freeze
+
+    Commit = Data.define(:sha, :branch)
+
+    Io = Data.define(:stdout, :stderr) do
+      def initialize(stdout: $stdout, stderr: $stderr) = super
+    end
+
+    # The collaborators a pipeline run can have replaced. Each one left out
+    # gets the real thing. `environment` is the stages' environment as a hash,
+    # which masking and injected command runners see; `clock` answers seconds
+    # for the evidence budget; `extractor_runner` makes what runs a project's
+    # own extractors (`run:` entries), given dir:, env: and scratch:;
+    # `process_table` answers what ps lists, for a stage over budget.
+    Seams = Data.define(:command_runner, :time_budgets, :commit_validator, :recorder, :background_launcher, :workspace,
+                        :stage_dir, :environment, :clock, :extractor_runner, :process_table)
+
+    # Reopened rather than given as a block to Data.define, so tools that read
+    # the source (mutineer) see these as Seams' methods.
+    class Seams
+      def self.defaults
+        { command_runner: nil, time_budgets: {}, recorder: Persistence::NullRecorder.new, background_launcher: nil,
+          workspace: nil, commit_validator: method(:commit_exists?), stage_dir: StageDir.method(:create),
+          environment: ENV.to_h, clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) },
+          extractor_runner: Evidence::CommandRunner.method(:new),
+          process_table: Evidence::ProcessTable.method(:now) }
+      end
+
+      def self.commit_exists?(sha) = Open3.capture2e(GitEnvironment::CLEAN, "git", "cat-file", "-t", sha).last.success?
+
+      def initialize(**given) = super(**self.class.defaults.merge(given))
+      def budgets = DEFAULT_BUDGETS.merge(time_budgets)
+      def executor(dir) = CommandExecutor.new(command_runner, dir, environment)
+    end
+  end
+end

data/lib/fun_ci/pipeline/workspaces.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require_relative "../setup/project_config"
+require_relative "worktree_pool"
+require_relative "in_place"
+
+module FunCi
+  module Pipeline
+    # Where a pipeline for a commit runs: a slot from the project's worktree
+    # pool, sized by .fun-ci/config, or, for a root commit's null SHA, which
+    # no worktree can check out, the project directory itself.
+    module Workspaces
+      NULL_SHA = ("0" * 40).freeze
+
+      def self.for(project, sha)
+        return InPlace.new(project) if sha == NULL_SHA
+
+        WorktreePool.new(Worktrees.new(project), size: Setup::ProjectConfig.new(project).worktree_slots)
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/worktree_pool.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require_relative "slot"
+require_relative "worktrees"
+
+module FunCi
+  module Pipeline
+    # Hands each pipeline a worktree slot of its own under
+    # <git-common-dir>/fun-ci/worktrees, checked out at its commit, waiting
+    # while every slot is taken.
+    class WorktreePool
+      DEFAULT_SIZE = 2
+
+      attr_reader :size
+
+      def initialize(worktrees, size: DEFAULT_SIZE, waiter: -> { sleep 0.25 })
+        @worktrees = worktrees
+        @size = size
+        @waiter = waiter
+      end
+
+      def acquire(sha)
+        slot = wait_for_free_slot
+        @worktrees.check_out(slot.path, sha)
+        slot
+      rescue StandardError
+        slot&.release
+        raise
+      end
+
+      private
+
+      def wait_for_free_slot
+        loop do
+          slot = free_slot
+          return slot if slot
+
+          @waiter.call
+        end
+      end
+
+      def free_slot
+        root = @worktrees.root
+        FileUtils.mkdir_p(root)
+        @size.times.lazy.filter_map { |index| try_lock(File.join(root, "slot-#{index}")) }.first
+      end
+
+      # The lock stays open for as long as the slot is held, so no block.
+      def try_lock(path)
+        lock = File.new("#{path}.lock", File::RDWR | File::CREAT, 0o644)
+        return Slot.new(path, stamped(lock), lock.path) if lock.flock(File::LOCK_EX | File::LOCK_NB)
+
+        lock.close
+        nil
+      end
+
+      def stamped(lock)
+        lock.truncate(0)
+        lock.write(Process.pid.to_s)
+        lock.flush
+        lock
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/worktree_prune.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require_relative "worktrees"
+
+module FunCi
+  module Pipeline
+    # Removes every slot of a project's worktree pool, and git's entries for
+    # them, when no run holds one. It takes each slot's lock first, so a run
+    # starting meanwhile waits until the slot is gone and then checks it out
+    # afresh.
+    class WorktreePrune
+      class Busy < StandardError; end
+
+      def initialize(worktrees)
+        @worktrees = worktrees
+      end
+
+      # How many slots it removed. Raises Busy, removing nothing, while a run
+      # holds a slot.
+      def run
+        locks = take_locks
+        locks.each { |lock| FileUtils.rm_rf(lock.path.delete_suffix(".lock")) }
+        @worktrees.prune
+        locks.size
+      ensure
+        locks&.each(&:close)
+      end
+
+      private
+
+      # Every slot's lock, or none: on the first slot a run holds, it lets go
+      # of the locks it took before raising Busy.
+      def take_locks
+        slot_paths.each_with_object([]) do |path, locks|
+          locks << File.new("#{path}.lock", File::RDWR | File::CREAT, 0o644)
+          next if locks.last.flock(File::LOCK_EX | File::LOCK_NB)
+
+          locks.each(&:close)
+          raise Busy, "a pipeline is running in #{path}"
+        end
+      end
+
+      def slot_paths
+        Dir.glob(File.join(@worktrees.root, "slot-*")).map { |path| path.delete_suffix(".lock") }.uniq
+      end
+    end
+  end
+end

data/lib/fun_ci/pipeline/worktrees.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require "open3"
+require_relative "git_environment"
+
+module FunCi
+  module Pipeline
+    # The git commands a worktree slot needs, run against the project's repository.
+    class Worktrees
+      class GitError < StandardError; end
+
+      def initialize(project_root)
+        @project_root = project_root
+      end
+
+      def root
+        File.join(File.expand_path(git(@project_root, "rev-parse", "--git-common-dir").strip, @project_root),
+                  "fun-ci", "worktrees")
+      end
+
+      # Leaves +path+ as a clean detached checkout of +sha+. Ignored files stay,
+      # so caches such as vendor/bundle survive between runs in the same slot.
+      def check_out(path, sha)
+        return add(path, sha) unless File.exist?(File.join(path, ".git"))
+
+        git(path, "checkout", "--detach", "--force", sha)
+        git(path, "clean", "-fd")
+      end
+
+      # Has git forget the worktrees whose directories are gone.
+      def prune = git(@project_root, "worktree", "prune")
+
+      private
+
+      def add(path, sha) = git(@project_root, "worktree", "add", "--detach", "--force", path, sha)
+
+      def git(dir, *args)
+        output, status = Open3.capture2e(GitEnvironment::CLEAN, "git", *args, chdir: dir)
+        raise GitError, "git #{args.join(" ")}: #{output}" unless status.success?
+
+        output
+      end
+    end
+  end
+end

data/lib/fun_ci/setup/agent_instructions.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Setup
+    # What `fun-ci init` tells a coding agent working in the project, in the
+    # instructions file agents read (acceptance-tests.md, AT-9.15): AGENTS.md,
+    # or CLAUDE.md when that is the only one. The marker keeps it to once.
+    module AgentInstructions
+      MARKER = "<!-- fun-ci: agent instructions -->"
+      SECTION = <<~MD.freeze
+        ## fun-ci
+
+        #{MARKER}
+        This project runs fun-ci on every commit. After each commit, run the
+        `fun-ci wait` command it prints, in the background, and act on its exit
+        code: 0 passed, 1 failed, 2 over budget, 3 undecided, 4 superseded.
+        `fun-ci wait --need all` must exit 0 before the work is called done.
+        `fun-ci runs` lists recent runs, and `fun-ci status` says where one stands.
+      MD
+
+      def self.file_for(names) = names.include?("CLAUDE.md") && !names.include?("AGENTS.md") ? "CLAUDE.md" : "AGENTS.md"
+
+      # The text with the section added, or nil when it has it already.
+      def self.merged(text)
+        return nil if text.include?(MARKER)
+
+        text.strip.empty? ? SECTION : "#{text.rstrip}\n\n#{SECTION}"
+      end
+    end
+  end
+end

data/lib/fun_ci/setup/commands.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require_relative "installer"
+require_relative "hook_writer"
+require_relative "git_hooks"
+require_relative "setup_checker"
+require_relative "legacy_hooks"
+
+module FunCi
+  module Setup
+    # `fun-ci init`, `install-hooks` and `check`, in the project directory.
+    # `hooks_dir` answers where git runs a project's hooks from.
+    class Commands
+      def initialize(project_root, stdout, hooks_dir: GitHooks.method(:dir))
+        @project_root = project_root
+        @stdout = stdout
+        @hooks_dir = hooks_dir
+      end
+
+      def init(args)
+        code = Installer.run(project_root: @project_root, stdout: @stdout)
+        return code if !code.zero? || !args.include?("--everything")
+
+        code = install_hooks([])
+        return code unless code.zero?
+
+        check([])
+      end
+
+      def install_hooks(args)
+        LegacyHooks.new(@project_root).remove(@stdout)
+        install(args.any? ? [args.first] : HookScript.types)
+      end
+
+      def check(_args) = SetupChecker.run(project_root: @project_root, stdout: @stdout)
+
+      private
+
+      def install(types)
+        hooks_dir = @hooks_dir.call(@project_root)
+        types.each do |type|
+          code = HookWriter.run(hooks_dir: hooks_dir, hook_type: type, stdout: @stdout)
+          return code unless code.zero?
+        end
+        0
+      end
+    end
+  end
+end

data/lib/fun_ci/setup/git_hooks.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "open3"
+require_relative "../pipeline/git_environment"
+
+module FunCi
+  module Setup
+    # Where git runs a project's hooks from: its git directory's hooks, or
+    # where core.hooksPath points, and from a linked worktree the
+    # repository's own. Git is asked, as .git may be a file or be ignored.
+    module GitHooks
+      # The directory, or nil outside a git repository.
+      def self.dir(project_root)
+        out, status = Open3.capture2(Pipeline::GitEnvironment::CLEAN, "git", "rev-parse", "--git-path", "hooks",
+                                     chdir: project_root, err: File::NULL)
+        status.success? ? File.expand_path(out.strip, project_root) : nil
+      end
+    end
+  end
+end

data/lib/fun_ci/setup/hook_script.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module FunCi
+  module Setup
+    # The script fun-ci writes into each git hook it manages.
+    module HookScript
+      MARKER = "# fun-ci-managed-hook"
+      NULL_SHA = "0" * 40
+
+      GUARD = <<~SH.freeze
+        #!/bin/sh
+        #{MARKER}
+        if ! command -v fun-ci >/dev/null 2>&1; then
+          echo "fun-ci is not installed or not on PATH, so this runs without CI. Install it with: gem install fun_ci" >&2
+          exit 0
+        fi
+      SH
+
+      # After a commit: the commit just made, in the background.
+      POST_COMMIT = <<~SH.freeze
+        COMMIT=$(git rev-parse HEAD 2>/dev/null || echo "#{NULL_SHA}")
+        BRANCH=$(git branch --show-current 2>/dev/null || echo "unknown")
+        fun-ci trigger --background "$COMMIT" "$BRANCH"
+      SH
+
+      # Before a push: the fast verdict of each commit git says it pushes
+      # (acceptance-tests.md, AT-9.13). A deleted ref pushes the null SHA, and
+      # exit 5 is a project not set up for fun-ci, which pushes without CI.
+      PRE_PUSH = <<~SH.freeze
+        status=0
+        while read -r local_ref local_sha remote_ref remote_sha; do
+          [ "$local_sha" = "#{NULL_SHA}" ] && continue
+          fun-ci wait "$local_sha" --need fast
+          code=$?
+          [ "$code" -eq 0 ] || [ "$code" -eq 5 ] || status=1
+        done
+        exit $status
+      SH
+
+      BODIES = { "post-commit" => POST_COMMIT, "pre-push" => PRE_PUSH }.freeze
+
+      def self.types = BODIES.keys
+      def self.for(hook_type) = GUARD + BODIES.fetch(hook_type)
+      def self.managed?(script) = script.include?(MARKER)
+    end
+  end
+end

data/lib/fun_ci/setup/hook_writer.rb

@@ -1,39 +1,26 @@
 # frozen_string_literal: true
 
 require "fileutils"
+require_relative "hook_script"
 
 module FunCi
   module Setup
+    # Writes a hook into `hooks_dir`, where git runs the project's hooks
+    # from (GitHooks.dir); nil outside a git repository.
     class HookWriter
-      ALLOWED_HOOKS = %w[pre-commit pre-push].freeze
-      MARKER = "# fun-ci-managed-hook"
-
-      HOOK_COMMANDS = {
-        "pre-commit" => "fun-ci trigger --no-validate",
-        "pre-push" => "fun-ci trigger"
-      }.freeze
-
-      HOOK_TEMPLATE = <<~SH
-        #!/bin/sh
-        #{MARKER}
-        COMMIT=$(git rev-parse HEAD 2>/dev/null || echo "0000000000000000000000000000000000000000")
-        BRANCH=$(git branch --show-current 2>/dev/null || echo "unknown")
-        %<command>s "$COMMIT" "$BRANCH"
-      SH
-
-      def self.run(project_root:, hook_type:, stdout: $stdout)
-        new(project_root: project_root, hook_type: hook_type, stdout: stdout).run
+      def self.run(hooks_dir:, hook_type:, stdout: $stdout)
+        new(hooks_dir: hooks_dir, hook_type: hook_type, stdout: stdout).run
       end
 
-      def initialize(project_root:, hook_type:, stdout:)
-        @project_root = project_root
+      def initialize(hooks_dir:, hook_type:, stdout:)
+        @hooks_dir = hooks_dir
         @hook_type = hook_type
         @stdout = stdout
       end
 
       def run
         return reject("Not a git repository: no .git/ found.") unless git_repo?
-        return reject("Unknown hook type: #{@hook_type}") unless ALLOWED_HOOKS.include?(@hook_type)
+        return reject("Unknown hook type: #{@hook_type}") unless HookScript.types.include?(@hook_type)
         return skip("Hook #{@hook_type} already exists from another tool, so it was left alone.") if foreign_hook?
 
         write_hook
@@ -43,21 +30,16 @@ module FunCi
 
       private
 
-      def git_repo?
-        Dir.exist?(File.join(@project_root, ".git"))
-      end
-
-      def hook_path
-        File.join(@project_root, ".git", "hooks", @hook_type)
-      end
+      def git_repo? = !@hooks_dir.nil?
+      def hook_path = File.join(@hooks_dir, @hook_type)
 
       def foreign_hook?
-        File.exist?(hook_path) && !File.read(hook_path).include?(MARKER)
+        File.exist?(hook_path) && !HookScript.managed?(File.read(hook_path))
       end
 
       def write_hook
-        FileUtils.mkdir_p(File.join(@project_root, ".git", "hooks"))
-        File.write(hook_path, format(HOOK_TEMPLATE, command: HOOK_COMMANDS[@hook_type]))
+        FileUtils.mkdir_p(@hooks_dir)
+        File.write(hook_path, HookScript.for(@hook_type))
         File.chmod(0o755, hook_path)
       end
 

data/lib/fun_ci/setup/installer.rb

@@ -3,51 +3,67 @@
 require_relative "project_detector"
 require_relative "template_writer"
 require_relative "maven_linter_detector"
+require_relative "agent_instructions"
 
 module FunCi
   module Setup
     class Installer
-      def self.run(project_root:, stdout: $stdout, pom_reader: nil)
-        new(project_root: project_root, stdout: stdout, pom_reader: pom_reader).run
+      def self.run(project_root:, stdout: $stdout)
+        new(project_root: project_root, stdout: stdout).run
       end
 
-      def initialize(project_root:, stdout:, pom_reader: nil)
+      def initialize(project_root:, stdout:)
         @project_root = project_root
         @stdout = stdout
-        @pom_reader = pom_reader || ->(path) { File.read(path) }
       end
 
+      # A project set up for fun-ci, new or not, also tells agents what to do.
       def run
-        if Dir.exist?(File.join(@project_root, ".fun-ci"))
-          @stdout.puts ".fun-ci/ already exists, so init did nothing."
-          return 0
-        end
+        code = set_up
+        tell_agents if code.zero?
+        code
+      end
 
-        filenames = Dir.children(@project_root)
-        detected = ProjectDetector.new(filenames).detect
+      private
 
-        if detected == :unknown
-          @stdout.puts "Could not detect project type. Create .fun-ci/ manually."
-          return 1
-        end
+      def set_up
+        return report(".fun-ci/ already exists, so init left it as it is.", 0) if initialised?
 
-        @stdout.puts "Detected: #{detected.to_s.tr("_", " ")}"
+        detected = ProjectDetector.new(Dir.children(@project_root)).detect
+        return report("Could not detect project type. Create .fun-ci/ manually.", 1) if detected == :unknown
 
-        lint_override = detect_maven_linter(detected)
-        TemplateWriter.new(detected, @project_root, lint_override: lint_override).write
+        write_templates(detected)
+      end
+
+      def tell_agents
+        file = AgentInstructions.file_for(Dir.children(@project_root))
+        path = File.join(@project_root, file)
+        merged = AgentInstructions.merged(File.exist?(path) ? File.read(path) : "")
+        return unless merged
 
-        @stdout.puts "Created .fun-ci/ with template scripts."
-        0
+        File.write(path, merged)
+        @stdout.puts "Told agents what to do after a commit, in #{file}."
       end
 
-      private
+      def initialised?
+        Dir.exist?(File.join(@project_root, ".fun-ci"))
+      end
+
+      def write_templates(detected)
+        @stdout.puts "Detected: #{detected.to_s.tr("_", " ")}"
+        TemplateWriter.new(detected, @project_root, lint_override: detect_maven_linter(detected)).write
+        report("Created .fun-ci/ with template scripts.", 0)
+      end
+
+      def report(message, exit_code)
+        @stdout.puts message
+        exit_code
+      end
 
       def detect_maven_linter(detected)
         return nil unless detected == :jvm_maven
 
-        pom_path = File.join(@project_root, "pom.xml")
-        pom_content = @pom_reader.call(pom_path)
-        command = MavenLinterDetector.new(pom_content).lint_command
+        command = MavenLinterDetector.new(File.read(File.join(@project_root, "pom.xml"))).lint_command
         command == MavenLinterDetector::DEFAULT_COMMAND ? nil : command
       end
     end

data/lib/fun_ci/setup/legacy_hooks.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require_relative "hook_script"
+
+module FunCi
+  module Setup
+    # fun-ci 1.x ran the background pipeline from a pre-commit hook, before
+    # the commit existed. The hook fun-ci wrote goes on upgrade, and is warned
+    # about until then; one someone wrote by hand that still calls fun-ci is
+    # warned about, never touched.
+    class LegacyHooks
+      WARNING = ".git/hooks/pre-commit calls fun-ci, which now runs after the commit: " \
+                "take fun-ci out of it and run `fun-ci install-hooks`"
+      NOT_UPGRADED = "fun-ci 1.x's pre-commit hook is still installed, so each run tests the commit before yours " \
+                     "and yours gets none: run `fun-ci install-hooks`"
+
+      def initialize(project_root)
+        @hook = File.join(project_root, ".git", "hooks", "pre-commit")
+      end
+
+      def remove(stdout)
+        return unless File.exist?(@hook) && HookScript.managed?(File.read(@hook))
+
+        File.delete(@hook)
+        stdout.puts "Removed fun-ci's 1.x pre-commit hook; the background pipeline now runs from post-commit."
+      end
+
+      def warnings
+        return [] unless File.exist?(@hook)
+
+        script = File.read(@hook)
+        return [NOT_UPGRADED] if HookScript.managed?(script)
+
+        script.include?("fun-ci") ? [WARNING] : []
+      end
+    end
+  end
+end