openlanes 3.0.0

data/lib/openlanes/evals/evidence.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module OpenLanes
+  module Evals
+    # Everything a check may look at: the folder, where it ended, who was
+    # asked to write it, what the run spent and what it changed.
+    #
+    # @!attribute [r] folder
+    #   @return [String] the plan folder, which may not exist
+    # @!attribute [r] state
+    #   @return [Symbol, nil]
+    # @!attribute [r] agent
+    #   @return [OpenLanes::Agents::Agent]
+    # @!attribute [r] run
+    #   @return [OpenLanes::Evals::Run]
+    # @!attribute [r] changes
+    #   @return [OpenLanes::Evals::Changes, nil] nil when no repository was recorded
+    Evidence = Data.define(:folder, :state, :agent, :run, :changes) do
+      # @param name [String] a file in the plan folder
+      # @return [String, nil]
+      def read(name)
+        path = File.join(folder, name)
+        File.file?(path) ? File.read(path, encoding: "UTF-8") : nil
+      end
+
+      # A document without its ledger blocks, narrowed to one section when
+      # asked. The ledger is the harness's bookkeeping, not the agent's prose,
+      # and counting it would let a run pad itself by signing.
+      #
+      # @param name [String]
+      # @param section [String, nil] a heading's text
+      # @return [String, nil] nil when the file or the section is missing
+      def text(name, section = nil)
+        body = read(name) or return nil
+        body = Plans::Ledger.stripped(body)
+        section ? Evidence.section(body, section) : body
+      end
+
+      # The folder as the state machine would see it under +status+'s name,
+      # so the status's own invariant can be asked of a folder whose name
+      # carries no state, such as a recording's `plan/`.
+      #
+      # @param status [OpenLanes::Plans::Status]
+      # @return [OpenLanes::Plans::Subject]
+      def subject_as(status)
+        Plans.subject(Plans::Feature.new(ordinal: Plans.ordinal("1"),
+          status:,
+          slug:    "eval",
+          dirname: File.basename(folder),
+          path:    folder))
+      end
+
+      # @param name [String]
+      # @return [Regexp] a Markdown heading with that text, any level
+      def self.heading(name) = /\A[ \t]{0,3}\#{1,6}[ \t]+#{Regexp.escape(name)}(?!\w)/i
+
+      # The body under a heading, up to the next heading of the same or a
+      # higher level.
+      #
+      # @param text [String]
+      # @param name [String]
+      # @return [String, nil]
+      def self.section(text, name)
+        lines = text.lines
+        start = lines.index { |l| l.match?(heading(name)) } or return nil
+        level = lines[start][/\A[ \t]*(#+)/, 1].size
+        lines.drop(start + 1).take_while { |l| (l[/\A[ \t]{0,3}(#+)[ \t]/, 1]&.size || 7) > level }.join
+      end
+    end
+  end
+end

data/lib/openlanes/evals/live.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "tmpdir"
+require "yaml"
+
+module OpenLanes
+  module Evals
+    # Runs the real agent on a case's fixture, in a repository nobody else
+    # uses, under a token cap, and scores what it leaves behind.
+    #
+    # The steps mirror what the runner does around one agent, and reuse its
+    # pieces rather than imitating them: {Execution.executor} runs the agent
+    # with the same autonomy boundary, and `resync dirs` renames the folder to
+    # the state its contents justify, as the runner does after every round.
+    # What it cannot reproduce is publishing. A build agent's 🟢 needs a pull
+    # request on GitHub, so build cases check signatures and diffs, not state.
+    class Live
+      # The default ceiling for one live case. Cheap enough to run by hand,
+      # high enough that a deep research case is not killed for doing its job.
+      DEFAULT_MAX_TOKENS = 300_000
+
+      # The one plan in the fixture repository.
+      ORDINAL = "001.00"
+
+      # Who the fixture commit is from: a made-up identity, so a commit in a
+      # temp repository can never be mistaken for a person's.
+      AUTHOR = ["-c", "user.name=OpenLanes Eval", "-c", "user.email=eval@example.com",
+                "-c", "commit.gpgsign=false", "-c", "core.hooksPath=/dev/null"].freeze
+
+      # @!attribute [r] score
+      #   @return [OpenLanes::Evals::Score]
+      # @!attribute [r] result
+      #   @return [OpenLanes::Execution::Executor::Result]
+      # @!attribute [r] root
+      #   @return [String, nil] the repository, when it was kept
+      Outcome = Data.define(:score, :result, :root)
+
+      # @param kase [OpenLanes::Evals::Case] at the depth to run it
+      # @param registry [OpenLanes::Agents::Registry]
+      # @param max_tokens [Integer] enforced by the executor's meter
+      # @param timeout [Integer, nil] seconds; nil leaves the agent's own clock
+      # @param spawn [#call, nil] the executor's seam; nil starts the real CLI
+      # @param trace_dir [String, nil] where the NDJSON trace goes
+      # @param keep [Boolean] leave the repository on disk for inspection
+      def initialize(kase, registry: Agents.registry, max_tokens: DEFAULT_MAX_TOKENS, timeout: nil, spawn: nil,
+        trace_dir: nil, keep: false)
+        @kase = kase
+        @registry = registry
+        @agent = registry.find(kase.agent) or raise Invalid, "#{kase.path}: names no agent called #{kase.agent}"
+        @max_tokens = max_tokens
+        @timeout = timeout
+        @spawn = spawn
+        @trace_dir = trace_dir
+        @keep = keep
+      end
+
+      # @return [OpenLanes::Evals::Live::Outcome]
+      def call
+        root = Dir.mktmpdir("openlanes-eval-")
+        begin
+          plans = build(root)
+          result = executor(root).call(@agent, start(plans), root:)
+          Lifecycle.resync_dirs(tree: Plans.tree(plans)).call(commit: true)
+          found = Plans.tree(plans).find(ORDINAL)
+          run = Run.new(seconds: result.seconds, tokens: result.fresh)
+          score = Scorer.new(@kase, registry: @registry)
+                        .call(folder: found&.feature&.path || File.join(plans, "missing"), run:, repo: root)
+          Outcome.new(score:, result:, root: @keep ? root : nil)
+        ensure
+          FileUtils.rm_rf(root) unless @keep
+        end
+      end
+
+      private
+
+      # A repository holding the fixture, committed, so the executor's
+      # after-check has a HEAD to hold the agent to.
+      #
+      # @return [String] the `.plans` directory
+      def build(root)
+        git(root, "init", "-q")
+        @kase.fixture.repo.each { |name, body| write(File.join(root, name), body) }
+        plans = File.join(root, OpenLanes::PLANS_DIR)
+        status = Plans.status(@kase.fixture.state)
+        folder = File.join(plans, Plans.plan_dirname(ORDINAL, status, @kase.fixture.slug))
+        @kase.fixture.files.each { |name, body| write(File.join(folder, name), body) }
+        stamp_depth(File.join(folder, Plans::Spec::FILENAME))
+        git(root, "add", "-A")
+        git(root, *AUTHOR, "commit", "-q", "-m", "Eval fixture #{@kase}")
+        plans
+      end
+
+      # The depth reaches the agent the way it would in a real plan: as
+      # `depth:` in spec.md's frontmatter, which the profile turns into effort.
+      #
+      # @return [void]
+      def stamp_depth(path)
+        meta, body = Frontmatter.split(File.read(path, encoding: "UTF-8"))
+        meta = meta.merge("depth" => @kase.depth.to_s)
+        File.write(path, "#{YAML.dump(meta)}---\n#{body}")
+      end
+
+      # The runner moves a folder to the agent's `starts_as` the moment it is
+      # dispatched, so r2d2 works a 🟡 plan and hansolo a 👀 one. Same here.
+      #
+      # @return [OpenLanes::Plans::Subject]
+      def start(plans)
+        subject = Plans.tree(plans).find(ORDINAL)
+        target = @agent.starts_as
+        return subject unless target && subject.machine.may?(target)
+
+        subject.machine.promote!(target)
+        Plans.tree(plans).find(ORDINAL)
+      end
+
+      # @return [OpenLanes::Execution::Executor]
+      def executor(root)
+        options = { root:, max_tokens: @max_tokens, fresh_budget: true, timeout: @timeout }
+        options[:spawn] = @spawn if @spawn
+        options[:trace_dir] = @trace_dir if @trace_dir
+        Execution.executor(**options)
+      end
+
+      # @return [void]
+      def write(path, body)
+        FileUtils.mkdir_p(File.dirname(path))
+        File.write(path, body)
+      end
+
+      # @return [void]
+      # @raise [OpenLanes::Error] when git refuses
+      def git(root, *args)
+        return if system("git", "-C", root, *args, out: File::NULL, err: File::NULL)
+
+        raise Error, "git #{args.last(2).join(" ")} failed in #{root}"
+      end
+    end
+  end
+end

data/lib/openlanes/evals/recording.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module OpenLanes
+  module Evals
+    # A plan folder as some run of the agent left it, kept so the scorers can
+    # be checked without running anything.
+    #
+    #   <case>/pass/plan/       the plan folder
+    #   <case>/pass/repo/       the repository around it, for coding agents
+    #   <case>/pass/run.json    {"seconds": .., "tokens": .., "state": ..}
+    #
+    # The folder sits in `plan/` rather than at the top so that `run.json` and
+    # `repo/` are never mistaken for files the agent wrote.
+    #
+    # @!attribute [r] dir
+    #   @return [String]
+    Recording = Data.define(:dir) do
+      # @param dir [String]
+      # @return [OpenLanes::Evals::Recording, nil] nil when there is no plan folder
+      def self.at(dir) = File.directory?(File.join(dir, "plan")) ? new(dir:) : nil
+
+      # @return [String] the plan folder
+      def plan = File.join(dir, "plan")
+
+      # @return [String, nil] the repository, when one was recorded
+      def repo
+        path = File.join(dir, "repo")
+        File.directory?(path) ? path : nil
+      end
+
+      # @return [OpenLanes::Evals::Run]
+      def run = Run.load(File.join(dir, "run.json"))
+
+      # @return [Symbol] :pass or :fail
+      def kind = File.basename(dir).to_sym
+    end
+  end
+end

data/lib/openlanes/evals/run.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "json"
+
+module OpenLanes
+  module Evals
+    # What one run spent, and where it left the folder.
+    #
+    # A live run measures all three. A recording carries them in `run.json`,
+    # because a directory on disk cannot say how long it took to write.
+    #
+    # @!attribute [r] seconds
+    #   @return [Float, nil] wall clock
+    # @!attribute [r] tokens
+    #   @return [Integer, nil] sent plus generated
+    # @!attribute [r] state
+    #   @return [Symbol, nil] the status key the folder ended in, when its
+    #     name cannot say so itself
+    Run = Data.define(:seconds, :tokens, :state) do
+      def initialize(seconds: nil, tokens: nil, state: nil)
+        super(seconds: seconds&.to_f, tokens: tokens&.to_i, state: state&.to_sym)
+      end
+
+      # @param path [String] a `run.json`
+      # @return [OpenLanes::Evals::Run] empty when the file is absent
+      def self.load(path)
+        return new unless File.file?(path)
+
+        data = JSON.parse(File.read(path))
+        new(seconds: data["seconds"], tokens: data["tokens"], state: data["state"])
+      end
+    end
+  end
+end

data/lib/openlanes/evals/score.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module OpenLanes
+  module Evals
+    # Every verdict for one case against one folder.
+    #
+    # @!attribute [r] kase
+    #   @return [OpenLanes::Evals::Case] at the depth it was scored
+    # @!attribute [r] verdicts
+    #   @return [Array<OpenLanes::Evals::Verdict>]
+    Score = Data.define(:kase, :verdicts) do
+      # A case with no checks has proved nothing, so it does not pass.
+      #
+      # @return [Boolean]
+      def passed? = !verdicts.empty? && verdicts.all?(&:ok?)
+
+      # @return [Array<OpenLanes::Evals::Verdict>]
+      def failures = verdicts.reject(&:ok?)
+
+      # @return [String] `7/8`
+      def tally = "#{verdicts.count(&:ok?)}/#{verdicts.size}"
+    end
+  end
+end

data/lib/openlanes/evals/scorer.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module OpenLanes
+  module Evals
+    # Applies a case's checks to a folder.
+    #
+    # The scorer does not care whether the folder was recorded months ago or
+    # written a second ago by a live run; both reach it as a path plus a
+    # {Run}, which is what lets the recordings test the scorer that the live
+    # runs depend on.
+    class Scorer
+      # @param kase [OpenLanes::Evals::Case]
+      # @param registry [OpenLanes::Agents::Registry]
+      def initialize(kase, registry: Agents.registry)
+        @kase = kase
+        @agent = registry.find(kase.agent) or raise Invalid, "#{kase.path}: names no agent called #{kase.agent}"
+      end
+
+      # @param folder [String] the plan folder
+      # @param run [OpenLanes::Evals::Run, nil]
+      # @param repo [String, nil] the repository afterwards
+      # @return [OpenLanes::Evals::Score]
+      def call(folder:, run: nil, repo: nil)
+        run ||= Run.new
+        evidence = Evidence.new(folder: File.expand_path(folder),
+          state:   state_of(folder, run),
+          agent:   @agent,
+          run:,
+          changes: repo && Changes.between(@kase.fixture.repo, repo))
+        Score.new(kase: @kase, verdicts: @kase.checks.map { |check| check.call(evidence) })
+      end
+
+      private
+
+      # A folder named like a plan says where it is; a recording's `plan/`
+      # cannot, and leans on what `run.json` says instead.
+      #
+      # @return [Symbol, nil]
+      def state_of(folder, run) = Plans.feature(File.expand_path(folder))&.status&.key || run.state
+    end
+  end
+end

data/lib/openlanes/evals/verdict.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module OpenLanes
+  module Evals
+    # One check's answer.
+    #
+    # @!attribute [r] name
+    #   @return [String] the check, as a reader would name it
+    # @!attribute [r] ok
+    #   @return [Boolean]
+    # @!attribute [r] detail
+    #   @return [String] what was found, pass or fail
+    Verdict = Data.define(:name, :ok, :detail) do
+      # @return [Boolean]
+      def ok? = ok
+    end
+  end
+end

data/lib/openlanes/evals.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module OpenLanes
+  # Per-agent evals: did the agent finish the task, and did it work at the
+  # depth it was asked for?
+  #
+  # A case (`evals/cases/<agent>/<id>.yml`) describes a fixture plan, the depth
+  # requested and the deterministic checks that hold an agent to both. Offline,
+  # the checks score recorded plan folders, a passing and a failing one per
+  # case, so the scorers are tested by the same corpus they score. Live, the
+  # agent runs in a throwaway repository under a token cap, and the folder it
+  # leaves behind is scored the same way.
+  module Evals
+    # Where the shipped cases live: beside the gem, so `openlanes eval` finds
+    # them from any project root.
+    DEFAULT_DIR = File.expand_path("../../evals/cases", __dir__)
+
+    # A case file that cannot be scored as written.
+    class Invalid < OpenLanes::Error; end
+
+    module_function
+
+    # @param dir [String] a directory of `<agent>/<id>.yml` files
+    # @param registry [OpenLanes::Agents::Registry] the agents a case may name
+    # @return [Array<OpenLanes::Evals::Case>] ordered by agent, then id
+    # @raise [OpenLanes::Evals::Invalid] when any case fails validation
+    def cases(dir = DEFAULT_DIR, registry: Agents.registry) = Case.all(dir, registry:)
+
+    # @param kase [OpenLanes::Evals::Case]
+    # @param folder [String] the plan folder as the agent left it
+    # @param run [OpenLanes::Evals::Run, nil] what the run spent, and where it ended
+    # @param repo [String, nil] the repository as the agent left it, for the
+    #   changed-path checks
+    # @param registry [OpenLanes::Agents::Registry]
+    # @return [OpenLanes::Evals::Score]
+    def score(kase, folder:, run: nil, repo: nil, registry: Agents.registry)
+      Scorer.new(kase, registry:).call(folder:, run:, repo:)
+    end
+
+    # @param kase [OpenLanes::Evals::Case]
+    # @param kind [Symbol] :pass or :fail
+    # @param registry [OpenLanes::Agents::Registry]
+    # @return [OpenLanes::Evals::Score, nil] nil when the case ships no such recording
+    def score_recording(kase, kind = :pass, registry: Agents.registry)
+      recording = kase.recording(kind) or return nil
+
+      score(kase, folder: recording.plan, run: recording.run, repo: recording.repo, registry:)
+    end
+
+    # Runs the real agent. Nothing calls this unless `--live` was typed.
+    #
+    # @param kase [OpenLanes::Evals::Case]
+    # @return [OpenLanes::Evals::Live::Outcome]
+    def run_live(kase, **) = Live.new(kase, **).call
+  end
+end

data/lib/openlanes/execution/checks.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module OpenLanes
+  module Execution
+    # The requirement every agent that changes code carries whether or not the
+    # spec says it: after the changes, every test suite and linter passes.
+    #
+    # A spec author writes the things that are particular to a plan. That the
+    # suite stays green is particular to none of them, so it was written in
+    # none of them, and a builder that never ran the linter shipped work the
+    # reviewer then had to reject for it. Deriving the command from the
+    # checkout, rather than asking each spec to name it, keeps it one fact in
+    # one place: the project's own justfile or Gemfile.
+    module Checks
+      # Agents whose work is code, or the judging of code. Research, planning
+      # and unblocking write documents, and running a suite for a document is
+      # minutes spent proving nothing.
+      PHASES = %w[build frontend review].freeze
+
+      # The recipe a project names its everything-gate with.
+      RECIPE = "check-all"
+
+      JUSTFILES = %w[justfile Justfile .justfile].freeze
+
+      # A recipe header: optional `@`, the name, optional parameters, a colon
+      # that is not the start of `:=` (an assignment).
+      RECIPE_LINE = /^@?#{Regexp.escape(RECIPE)}(?:\s[^:\n]*)?:(?!=)/
+
+      RUBY = ["bundle exec rspec", "bundle exec rubocop"].freeze
+
+      class << self
+        # @param agent [OpenLanes::Agents::Agent]
+        # @return [Boolean] whether the agent carries the requirement
+        def apply_to?(agent) = PHASES.include?(agent.phase.to_s)
+
+        # @param root [String] the checkout the agent works in
+        # @return [Array<String>] the commands that must pass, empty when the
+        #   project names none this can recognise
+        def commands(root)
+          return ["just #{RECIPE}"] if recipe?(root)
+          return RUBY if File.file?(File.join(root, "Gemfile"))
+
+          []
+        end
+
+        # @param root [String]
+        # @return [String] the requirement, one line
+        def requirement(root)
+          commands = commands(root)
+          if commands.empty?
+            "After your changes, every test suite and linter the project defines passes. " \
+              "Find them in its README, AGENTS.md or CLAUDE.md, run them all, and fix what they report."
+          else
+            "After your changes, all tests and linters pass: #{commands.map { "`#{it}`" }.join(" and ")} " \
+              "#{commands.size == 1 ? "exits" : "each exit"} 0. Run #{commands.size == 1 ? "it" : "them"} " \
+              "before you sign Completed, and fix what they report rather than reporting it."
+          end
+        end
+
+        private
+
+        # @param root [String]
+        # @return [Boolean]
+        def recipe?(root)
+          JUSTFILES.any? do |name|
+            path = File.join(root, name)
+            File.file?(path) && File.read(path, encoding: "UTF-8").match?(RECIPE_LINE)
+          end
+        end
+      end
+    end
+  end
+end