openlanes 3.0.0

data/lib/openlanes/engine/tally.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+
+module OpenLanes
+  module Engine
+    # What a `run` cost, once it is over.
+    #
+    # The loop reports movement — which plan went from ⭐️ to 🟡, which agent
+    # failed and why — and says nothing about the bill. A round of five agents
+    # against a large repository spends several million tokens and the better
+    # part of an hour, and until this existed the only record of that was a
+    # directory of trace files nobody reads.
+    #
+    # {#render} returns a string and prints nothing, so the caller decides where
+    # it goes, the same way {Reporter} works.
+    class Tally
+      # One agent's share of the run, added up across every round it ran in.
+      #
+      # @!attribute [r] name
+      #   @return [String]
+      # @!attribute [r] invocations
+      #   @return [Integer] times this agent was started
+      # @!attribute [r] subagents
+      #   @return [Integer] sub-agents it spawned, across all of them
+      # @!attribute [r] up
+      #   @return [Integer] tokens sent, sub-agent spend included
+      # @!attribute [r] down
+      #   @return [Integer] tokens generated
+      # @!attribute [r] seconds
+      #   @return [Float] how long it was running, added up
+      Row = Data.define(:name, :invocations, :subagents, :up, :down, :seconds)
+
+      # Cells per column, so the header, the rows and the rule cannot drift.
+      WIDTHS = { name: 22, invocations: 6, subagents: 11, up: 9, down: 9, seconds: 8 }.freeze
+
+      # @param attempts [Array<OpenLanes::Engine::Runner::Attempt>]
+      # @param seconds [Float] wall clock for the whole loop, which is not the
+      #   sum of the agents' own times: several of them run at once
+      # @param rounds [Integer, nil] accepted so an older caller still works, and
+      #   otherwise unused: the loop has no rounds of its own any more, each
+      #   agent counts its own per plan, so the summary counts invocations
+      def initialize(attempts:, seconds:, rounds: nil)
+        @attempts = attempts.reject { |a| a.ordinal == "?" }
+        @seconds = seconds
+        @rounds = rounds
+      end
+
+      # @return [Array<OpenLanes::Engine::Runner::Attempt>]
+      attr_reader :attempts
+
+      # @return [Float] wall clock
+      attr_reader :seconds
+
+      # @return [Integer, nil]
+      attr_reader :rounds
+
+      # Plans an agent was actually started against, counted once each however
+      # many rounds they took.
+      #
+      # @return [Integer]
+      def plans = attempts.map(&:ordinal).uniq.size
+
+      # @return [Integer] tokens sent, everything included
+      def up = attempts.sum(&:up)
+
+      # @return [Integer] tokens generated
+      def down = attempts.sum(&:down)
+
+      # @return [Integer] sub-agents spawned by every agent in the run
+      def subagents = attempts.sum(&:subagents)
+
+      # Sub-agent spend that arrived as one number. `claude` reports what a
+      # sub-agent cost without splitting it between the two directions, so this
+      # much of {#up} is really up and down together.
+      #
+      # @return [Integer]
+      def delegated = attempts.sum(&:delegated)
+
+      # @return [Float] seconds of agent time, added up across all of them
+      def busy = attempts.sum(&:seconds)
+
+      # How many agents were running at any given moment, averaged over the
+      # run: agent-seconds divided by wall-clock seconds. Below the `-j` ceiling
+      # by however much the round spent waiting for its slowest member.
+      #
+      # @return [Float]
+      def concurrency = seconds.positive? ? busy / seconds : 0.0
+
+      # @return [Array<OpenLanes::Engine::Tally::Row>] one per agent, dearest first
+      def by_agent
+        attempts.group_by(&:agent).map { |name, list|
+          Row.new(name:,
+            invocations: list.size,
+            subagents: list.sum(&:subagents),
+            up: list.sum(&:up),
+            down: list.sum(&:down),
+            seconds: list.sum(&:seconds))
+        }.sort_by { |row| -row.up }
+      end
+
+      # @return [String] the table and the summary line under it
+      def render
+        return summary if attempts.empty?
+
+        ([header, rule] + by_agent.map { |row| line(row) } + [rule, totals, "", summary]).join("\n")
+      end
+
+      # @return [String] the one-line version, for a box
+      def summary
+        [
+          "#{plans} plan#{"s" unless plans == 1} addressed",
+          "#{attempts.size} invocation#{"s" unless attempts.size == 1}",
+          duration(seconds),
+          "#{format("%.1f", concurrency)} agents at a time",
+          "↑ #{UI.abbreviate(up)} ↓ #{UI.abbreviate(down)}"
+        ].join(" · ") + delegation
+      end
+
+      private
+
+      # @return [String] the caveat, where there is one to make
+      def delegation
+        return "" if delegated.zero?
+
+        "\n#{subagents} sub-agent#{"s" unless subagents == 1} reported #{UI.abbreviate(delegated)} " \
+          "tokens without splitting them, counted above as ↑."
+      end
+
+      # @return [String]
+      def header
+        cell("Agent", :name) + %i[invocations subagents up down seconds]
+                               .zip(["runs", "sub-agents", "↑", "↓", "time"])
+                               .map { |key, text| right(text, key) }.join
+      end
+
+      # @return [String]
+      def rule = WIDTHS.values.map { |width| ("─" * (width - 1)) + " " }.join
+
+      # @param row [OpenLanes::Engine::Tally::Row]
+      # @return [String]
+      def line(row)
+        cell(row.name, :name) + right(row.invocations.to_s, :invocations) +
+          right(row.subagents.to_s, :subagents) + right(UI.abbreviate(row.up), :up) +
+          right(UI.abbreviate(row.down), :down) + right(duration(row.seconds), :seconds)
+      end
+
+      # @return [String]
+      def totals
+        cell("", :name) + right(attempts.size.to_s, :invocations) + right(subagents.to_s, :subagents) +
+          right(UI.abbreviate(up), :up) + right(UI.abbreviate(down), :down) + right(duration(busy), :seconds)
+      end
+
+      # @param text [String]
+      # @param key [Symbol]
+      # @return [String] padded to its column, left-justified
+      def cell(text, key) = UI.fit(text, WIDTHS.fetch(key))
+
+      # @param text [String]
+      # @param key [Symbol]
+      # @return [String] padded to its column, right-justified
+      def right(text, key)
+        width = WIDTHS.fetch(key) - 1
+        text = UI.fit(text, width).rstrip
+        UI.fit("", width - UI.display_width(text)) + text + " "
+      end
+
+      # @param seconds [Numeric]
+      # @return [String] e.g. "42s", "12m40s", "1h04m"
+      def duration(seconds)
+        total = seconds.to_i
+        return "#{total}s" if total < 60
+        return "#{total / 60}m#{format("%02ds", total % 60)}" if total < 3600
+
+        "#{total / 3600}h#{format("%02dm", (total % 3600) / 60)}"
+      end
+    end
+  end
+end

data/lib/openlanes/engine.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module OpenLanes
+  # The run loop: which agent works which plan, in what order, until a round
+  # changes nothing. It decides; {Execution} carries the decision out.
+  module Engine
+    module_function
+
+    # @return [OpenLanes::Engine::Runner]
+    def runner(**) = Runner.new(**)
+
+    # @param tree [OpenLanes::Plans::Tree]
+    # @return [OpenLanes::Engine::StateFile]
+    def state_file(tree) = StateFile.new(path: StateFile.for(tree))
+
+    # @return [OpenLanes::Engine::Tally]
+    def tally(**) = Tally.new(**)
+  end
+end

data/lib/openlanes/evals/case.rb

@@ -0,0 +1,191 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module OpenLanes
+  module Evals
+    # One eval: a fixture plan, the agent given it, the depth asked for, and
+    # what the folder must look like afterwards.
+    #
+    # Validation refuses rather than ignores. A misspelt expectation would
+    # otherwise be a check that silently never runs, and an eval that passes
+    # because it checked nothing is worse than no eval at all.
+    #
+    # @!attribute [r] id
+    #   @return [String] the file's basename, unique within its agent
+    # @!attribute [r] agent
+    #   @return [String] an `agents/*.md` name
+    # @!attribute [r] depth
+    #   @return [Symbol] one of {OpenLanes::Plans::Spec::DEPTHS}
+    # @!attribute [r] description
+    #   @return [String]
+    # @!attribute [r] fixture
+    #   @return [OpenLanes::Evals::Case::Fixture]
+    # @!attribute [r] expect
+    #   @return [Hash{String => Object}] as written, string keys
+    # @!attribute [r] path
+    #   @return [String] the YAML file
+    Case = Data.define(:id, :agent, :depth, :description, :fixture, :expect, :path) do
+      # The same case held to another depth's bounds.
+      #
+      # @param depth [Symbol, String, nil] nil keeps the case's own
+      # @return [OpenLanes::Evals::Case]
+      def at(depth) = depth.nil? ? self : with(depth: Case.depth!(depth, path))
+
+      # The bounds for {#depth}, or for the one given. A depth the case says
+      # nothing about has no bounds, so only the outcome checks apply.
+      #
+      # @param level [Symbol]
+      # @return [Hash{String => Object}]
+      def bounds(level = depth) = expect.fetch("depth", {}).fetch(level.to_s, {})
+
+      # @return [String] where this case keeps its recordings
+      def dir = path.delete_suffix(File.extname(path))
+
+      # @param kind [Symbol, String] :pass or :fail
+      # @return [OpenLanes::Evals::Recording, nil]
+      def recording(kind) = Recording.at(File.join(dir, kind.to_s))
+
+      # @return [Array<OpenLanes::Evals::Check>] every check, outcome first
+      def checks = Check.for(self)
+
+      # @return [String] `agent/id`, how the CLI names a case
+      def to_s = "#{agent}/#{id}"
+    end
+
+    class Case
+      # What the plan folder, and the repository around it, start as.
+      #
+      # @!attribute [r] state
+      #   @return [Symbol] a status key
+      # @!attribute [r] slug
+      #   @return [String]
+      # @!attribute [r] files
+      #   @return [Hash{String => String}] plan-folder file to content
+      # @!attribute [r] repo
+      #   @return [Hash{String => String}] repository path to content
+      Fixture = Data.define(:state, :slug, :files, :repo)
+
+      TOP = %w[id agent depth description fixture expect].freeze
+      FIXTURE = %w[state slug files repo].freeze
+      EXPECT = %w[state signed files_exist files_absent sections contains excludes changed_paths untouched depth].freeze
+      BOUNDS = %w[words count changed_files changed_lines max_seconds max_tokens].freeze
+
+      # What each bound may carry. `words` and `count` take one mapping or a
+      # list of them, so a case can bound two sections of one file.
+      BOUND_KEYS = {
+        "words"         => %w[file section min max],
+        "count"         => %w[file section pattern min max],
+        "changed_files" => %w[min max],
+        "changed_lines" => %w[min max]
+      }.freeze
+
+      class << self
+        # @param dir [String]
+        # @param registry [OpenLanes::Agents::Registry]
+        # @return [Array<OpenLanes::Evals::Case>]
+        def all(dir, registry:)
+          Dir.glob(File.join(File.expand_path(dir), "*", "*.yml")).map { |path| load(path, registry:) }
+        end
+
+        # @param path [String]
+        # @param registry [OpenLanes::Agents::Registry]
+        # @return [OpenLanes::Evals::Case]
+        # @raise [OpenLanes::Evals::Invalid]
+        def load(path, registry:)
+          data = YAML.safe_load_file(path)
+          refuse(path, "is not a mapping") unless data.is_a?(Hash)
+
+          from(data, path:, registry:)
+        rescue Psych::Exception => e
+          refuse(path, "is not valid YAML (#{e.message.lines.first.strip})")
+        end
+
+        # @param data [Hash]
+        # @param path [String]
+        # @param registry [OpenLanes::Agents::Registry]
+        # @return [OpenLanes::Evals::Case]
+        def from(data, path:, registry:)
+          unknown(path, data, TOP, "")
+          id = data["id"].to_s
+          refuse(path, "id #{id.inspect} must match the file name") unless id == File.basename(path, ".yml")
+          agent = data["agent"].to_s
+          refuse(path, "names no agent called #{agent.inspect}") unless registry.find(agent)
+          refuse(path, "sits under #{File.basename(File.dirname(path))}/ but names #{agent}") unless File.basename(File.dirname(path)) == agent
+
+          new(id:,
+            agent:,
+            depth:       depth!(data["depth"], path),
+            description: data["description"].to_s.strip,
+            fixture:     fixture(data["fixture"], path),
+            expect:      expectations(data["expect"], path),
+            path:)
+        end
+
+        # @param value [Object]
+        # @param path [String]
+        # @return [Symbol]
+        def depth!(value, path)
+          depth = value.to_s.strip.downcase.to_sym
+          return depth if Plans::Spec::DEPTHS.include?(depth)
+
+          refuse(path, "depth #{value.inspect} is not one of #{Plans::Spec::DEPTHS.join(", ")}")
+        end
+
+        private
+
+        # @return [OpenLanes::Evals::Case::Fixture]
+        def fixture(data, path)
+          refuse(path, "has no fixture") unless data.is_a?(Hash)
+          unknown(path, data, FIXTURE, "fixture.")
+          state = Plans.status(data["state"].to_s) or refuse(path, "fixture.state #{data["state"].inspect} is not a status")
+          files = files(data.fetch("files", {}), path, "fixture.files")
+          refuse(path, "fixture.files has no spec.md") unless files.key?(Plans::Spec::FILENAME)
+
+          Fixture.new(state:   state.key,
+            slug:    data.fetch("slug", File.basename(path, ".yml")).to_s,
+            files:,
+            repo:    files(data.fetch("repo", {}), path, "fixture.repo"))
+        end
+
+        # @return [Hash{String => String}]
+        def files(data, path, where)
+          refuse(path, "#{where} must map file names to contents") unless data.is_a?(Hash)
+          data.to_h { |name, body| [name.to_s, body.to_s] }
+        end
+
+        # @return [Hash]
+        def expectations(data, path)
+          refuse(path, "has no expect") unless data.is_a?(Hash)
+          unknown(path, data, EXPECT, "expect.")
+          if data.key?("state") && !Plans.status(data["state"].to_s)
+            refuse(path, "expect.state #{data["state"].inspect} is not a status")
+          end
+          depth = data.fetch("depth", {})
+          refuse(path, "expect.depth must map fast, medium or deep to bounds") unless depth.is_a?(Hash)
+          depth.each do |level, bounds|
+            depth!(level, path)
+            refuse(path, "expect.depth.#{level} must be a mapping") unless bounds.is_a?(Hash)
+            unknown(path, bounds, BOUNDS, "expect.depth.#{level}.")
+            bounds.slice(*BOUND_KEYS.keys).each do |name, value|
+              Array(value.is_a?(Hash) ? [value] : value).each do |bound|
+                refuse(path, "expect.depth.#{level}.#{name} must be a mapping") unless bound.is_a?(Hash)
+                unknown(path, bound, BOUND_KEYS.fetch(name), "expect.depth.#{level}.#{name}.")
+              end
+            end
+          end
+          data
+        end
+
+        # @return [void]
+        def unknown(path, data, allowed, prefix)
+          extra = data.keys.map(&:to_s) - allowed
+          refuse(path, "#{extra.map { |k| "#{prefix}#{k}" }.join(", ")}: unknown, expected one of #{allowed.join(", ")}") unless extra.empty?
+        end
+
+        # @return [void] never returns
+        def refuse(path, message) = raise(Invalid, "#{path}: #{message}")
+      end
+    end
+  end
+end

data/lib/openlanes/evals/changes.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module OpenLanes
+  module Evals
+    # What an agent changed in the repository: which paths, and how many lines.
+    #
+    # Computed by comparing the fixture's files with a directory, not by asking
+    # git, so a recording on disk and a live checkout are measured by the same
+    # code. Lines are counted as a multiset difference, added plus removed,
+    # which is what a reviewer reads as the size of a diff without needing one.
+    #
+    # @!attribute [r] paths
+    #   @return [Array<String>] relative, sorted
+    # @!attribute [r] lines
+    #   @return [Integer] lines added plus lines removed
+    Changes = Data.define(:paths, :lines) do
+      # @param glob [String]
+      # @return [Array<String>] changed paths matching it
+      def matching(glob) = paths.select { |p| Changes.match?(glob, p) }
+    end
+
+    class Changes
+      # Never part of the diff: git's own bookkeeping and the plans, which the
+      # plan-folder checks already cover.
+      IGNORED = [".git", OpenLanes::PLANS_DIR].freeze
+
+      FNMATCH = File::FNM_PATHNAME | File::FNM_EXTGLOB | File::FNM_DOTMATCH
+
+      class << self
+        # @param before [Hash{String => String}] the fixture's repository
+        # @param dir [String] the repository afterwards
+        # @return [OpenLanes::Evals::Changes]
+        def between(before, dir)
+          after = snapshot(dir)
+          paths = (before.keys | after.keys).reject { |p| before[p] == after[p] }.sort
+          lines = paths.sum { |p| line_delta(before[p].to_s, after[p].to_s) }
+          new(paths:, lines:)
+        end
+
+        # @param glob [String]
+        # @param path [String]
+        # @return [Boolean]
+        def match?(glob, path) = File.fnmatch?(glob, path, FNMATCH)
+
+        private
+
+        # @return [Hash{String => String}]
+        def snapshot(dir)
+          root = File.expand_path(dir)
+          Dir.glob("**/*", File::FNM_DOTMATCH, base: root).each_with_object({}) do |rel, out|
+            next if IGNORED.include?(rel.split("/").first)
+
+            full = File.join(root, rel)
+            out[rel] = File.read(full, encoding: "UTF-8") if File.file?(full)
+          end
+        end
+
+        # @return [Integer]
+        def line_delta(old, new)
+          a = old.lines.tally
+          b = new.lines.tally
+          (a.keys | b.keys).sum { |line| (a.fetch(line, 0) - b.fetch(line, 0)).abs }
+        end
+      end
+    end
+  end
+end

data/lib/openlanes/evals/check.rb

@@ -0,0 +1,224 @@
+# frozen_string_literal: true
+
+module OpenLanes
+  module Evals
+    # One deterministic question about a finished folder.
+    #
+    # Two families, matching the two things D8 asks of every agent:
+    #
+    #   outcome — state, signed, files, sections, contents, changed paths:
+    #             did it finish the task?
+    #   depth   — words, counts, diff size, seconds, tokens, bounded per
+    #             depth: did it work at the level it was asked for? Too
+    #             shallow for deep fails, and so does overdone for fast.
+    #
+    # @!attribute [r] kind
+    #   @return [Symbol] which judge answers it
+    # @!attribute [r] name
+    #   @return [String] how a report names it
+    # @!attribute [r] args
+    #   @return [Hash{Symbol => Object}] what the judge is given
+    Check = Data.define(:kind, :name, :args) do
+      # @param evidence [OpenLanes::Evals::Evidence]
+      # @return [OpenLanes::Evals::Verdict]
+      def call(evidence)
+        ok, detail = Check::Judges.public_send(kind, evidence, **args)
+        Verdict.new(name:, ok:, detail:)
+      end
+    end
+
+    class Check
+      class << self
+        # @param kase [OpenLanes::Evals::Case]
+        # @return [Array<OpenLanes::Evals::Check>]
+        def for(kase) = outcome(kase.expect) + depth(kase.bounds, kase.depth)
+
+        private
+
+        # @return [Array<OpenLanes::Evals::Check>]
+        def outcome(expect)
+          list = []
+          list << new(kind: :state, name: "state #{expect["state"]}", args: { expected: expect["state"].to_sym }) if expect["state"]
+          if expect["signed"]
+            note = expect["signed"] == true ? nil : expect["signed"].to_s
+            list << new(kind: :signed, name: "signed Completed#{" (#{note})" if note}", args: { note: })
+          end
+          Array(expect["files_exist"]).each { |f| list << new(kind: :file_exists, name: "has #{f}", args: { file: f }) }
+          Array(expect["files_absent"]).each { |f| list << new(kind: :file_absent, name: "no #{f}", args: { file: f }) }
+          expect.fetch("sections", {}).each do |file, headings|
+            list << new(kind: :sections, name: "sections in #{file}", args: { file:, headings: Array(headings) })
+          end
+          %w[contains excludes].each do |kind|
+            expect.fetch(kind, {}).each do |file, texts|
+              Array(texts).each { |text| list << new(kind: kind.to_sym, name: "#{file} #{kind} #{text}", args: { file:, text: }) }
+            end
+          end
+          Array(expect["changed_paths"]).each { |g| list << new(kind: :changed, name: "changed #{g}", args: { glob: g }) }
+          Array(expect["untouched"]).each { |g| list << new(kind: :untouched, name: "untouched #{g}", args: { glob: g }) }
+          list
+        end
+
+        # @return [Array<OpenLanes::Evals::Check>]
+        def depth(bounds, level)
+          list = []
+          %w[words count].each do |kind|
+            Array(bounds[kind].is_a?(Hash) ? [bounds[kind]] : bounds[kind]).each do |b|
+              where = b["section"] ? "#{b["file"]}##{b["section"]}" : b["file"]
+              what = kind == "count" ? "count /#{b["pattern"]}/ in #{where}" : "words in #{where}"
+              args = { file: b["file"], section: b["section"], min: b["min"], max: b["max"] }
+              args[:pattern] = b["pattern"] if kind == "count"
+              list << new(kind: kind.to_sym, name: "#{what} #{range(b)} @#{level}", args:)
+            end
+          end
+          %w[changed_files changed_lines].each do |kind|
+            next unless (b = bounds[kind])
+
+            list << new(kind: kind.to_sym, name: "#{kind.tr("_", " ")} #{range(b)} @#{level}", args: { min: b["min"], max: b["max"] })
+          end
+          list << new(kind: :seconds, name: "seconds ≤ #{bounds["max_seconds"]} @#{level}", args: { max: bounds["max_seconds"] }) if bounds["max_seconds"]
+          list << new(kind: :tokens, name: "tokens ≤ #{bounds["max_tokens"]} @#{level}", args: { max: bounds["max_tokens"] }) if bounds["max_tokens"]
+          list
+        end
+
+        # @return [String] `10..40`, `≥ 10` or `≤ 40`
+        def range(bound)
+          min, max = bound.values_at("min", "max")
+          return "#{min}..#{max}" if min && max
+
+          min ? "≥ #{min}" : "≤ #{max}"
+        end
+      end
+
+      # The answers. Each takes the evidence and its check's arguments and
+      # returns `[ok, detail]`; {Check#call} names the verdict.
+      module Judges
+        module_function
+
+        # Where the folder ended, and whether that state's own invariant holds
+        # of what is in it. A recording that claims 🔎 without a Research
+        # chapter fails here exactly as a rename would be refused.
+        #
+        # @return [Array(Boolean, String)]
+        def state(evidence, expected:)
+          actual = evidence.state or return [false, "the run recorded no final state"]
+          return [false, "ended #{actual}, expected #{expected}"] unless actual == expected
+
+          status = Plans.status(expected)
+          violation = status.violation(evidence.subject_as(status))
+          violation ? [false, violation] : [true, status.to_s]
+        end
+
+        # @return [Array(Boolean, String)]
+        def signed(evidence, note:)
+          agent = evidence.agent
+          entry = Plans::Ledger.last_for(Plans::Ledger.read(evidence.folder, agent.ledger), agent.name)
+          return [false, "#{agent.name} signed none of #{agent.ledger.join(", ")}"] unless entry
+          return [false, "last entry says #{entry.status}"] unless entry.completed?
+          if note && !entry.note.to_s.downcase.include?(note.downcase)
+            return [false, "signed Completed (#{entry.note || "no note"}), expected #{note}"]
+          end
+
+          [true, "#{entry.file}:#{entry.line}"]
+        end
+
+        # @return [Array(Boolean, String)]
+        def file_exists(evidence, file:)
+          File.exist?(File.join(evidence.folder, file)) ? [true, "present"] : [false, "#{file} is missing"]
+        end
+
+        # @return [Array(Boolean, String)]
+        def file_absent(evidence, file:)
+          File.exist?(File.join(evidence.folder, file)) ? [false, "#{file} is still there"] : [true, "absent"]
+        end
+
+        # @return [Array(Boolean, String)]
+        def sections(evidence, file:, headings:)
+          body = evidence.text(file) or return [false, "#{file} is missing"]
+          missing = headings.reject { |h| body.lines.any? { |l| l.match?(Evidence.heading(h)) } }
+          missing.empty? ? [true, "all #{headings.size} present"] : [false, "missing #{missing.map { |h| "## #{h}" }.join(", ")}"]
+        end
+
+        # @return [Array(Boolean, String)]
+        def contains(evidence, file:, text:)
+          body = evidence.read(file) or return [false, "#{file} is missing"]
+          found?(body, text) ? [true, "found"] : [false, "#{file} does not mention #{text}"]
+        end
+
+        # @return [Array(Boolean, String)]
+        def excludes(evidence, file:, text:)
+          body = evidence.read(file) or return [true, "#{file} is absent"]
+          found?(body, text) ? [false, "#{file} still says #{text}"] : [true, "not found"]
+        end
+
+        # @return [Array(Boolean, String)]
+        def changed(evidence, glob:)
+          changes = evidence.changes or return [false, "no repository was recorded"]
+          hits = changes.matching(glob)
+          hits.empty? ? [false, "nothing matching #{glob} changed"] : [true, hits.join(", ")]
+        end
+
+        # @return [Array(Boolean, String)]
+        def untouched(evidence, glob:)
+          changes = evidence.changes or return [false, "no repository was recorded"]
+          hits = changes.matching(glob)
+          hits.empty? ? [true, "untouched"] : [false, "changed #{hits.join(", ")}"]
+        end
+
+        # @return [Array(Boolean, String)]
+        def words(evidence, file:, section: nil, min: nil, max: nil)
+          body = evidence.text(file, section) or return [false, missing(file, section)]
+          within(body.scan(/[[:alnum:]][[:alnum:]'’_-]*/).size, min, max, "words")
+        end
+
+        # @return [Array(Boolean, String)]
+        def count(evidence, file:, pattern:, section: nil, min: nil, max: nil)
+          body = evidence.text(file, section) or return [false, missing(file, section)]
+          within(body.scan(Regexp.new(pattern)).size, min, max, "matches")
+        end
+
+        # @return [Array(Boolean, String)]
+        def changed_files(evidence, min: nil, max: nil)
+          changes = evidence.changes or return [false, "no repository was recorded"]
+          within(changes.paths.size, min, max, "files changed")
+        end
+
+        # @return [Array(Boolean, String)]
+        def changed_lines(evidence, min: nil, max: nil)
+          changes = evidence.changes or return [false, "no repository was recorded"]
+          within(changes.lines, min, max, "lines changed")
+        end
+
+        # @return [Array(Boolean, String)]
+        def seconds(evidence, max:)
+          spent = evidence.run.seconds or return [false, "not measured"]
+          within(spent.round, nil, max, "seconds")
+        end
+
+        # @return [Array(Boolean, String)]
+        def tokens(evidence, max:)
+          spent = evidence.run.tokens or return [false, "not measured"]
+          within(spent, nil, max, "tokens")
+        end
+
+        # @return [Array(Boolean, String)]
+        def within(value, min, max, unit)
+          return [false, "#{value} #{unit}, fewer than #{min}"] if min && value < min
+          return [false, "#{value} #{unit}, more than #{max}"] if max && value > max
+
+          [true, "#{value} #{unit}"]
+        end
+
+        # `/.../` is a pattern; anything else is a literal.
+        #
+        # @return [Boolean]
+        def found?(body, text)
+          text = text.to_s
+          text.length > 2 && text.start_with?("/") && text.end_with?("/") ? body.match?(Regexp.new(text[1..-2])) : body.include?(text)
+        end
+
+        # @return [String]
+        def missing(file, section) = section ? "#{file} has no ## #{section}" : "#{file} is missing"
+      end
+    end
+  end
+end