insika 0.0.1 → 0.2.0

data/lib/insika/evals/judge.rb

@@ -0,0 +1,225 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Insika
+  module Evals
+    # The LLM-judge. Scores a golden's `rubric` against the
+    # actual assistant reply — the subjective layer on top of the deterministic
+    # asserts. Pure over an injected `ask` callable (prompt -> raw model text), so it's
+    # unit-testable without an LLM; the real ask (RubyLLM on the utility_model, temp 0)
+    # is built by the CLI.
+    #
+    # Conservative by construction: an unparseable judge reply scores 0 (fails) rather
+    # than silently passing.
+    #
+    # A PANEL, not a single voice. `quorum: N` samples ONE model N
+    # times, which measures that model's variance and little else — at temperature 0 it
+    # mostly returns the same answer, including the same blind spot. Two DIFFERENT
+    # models disagreeing about a rubric is the signal worth having, so `asks:` takes one
+    # callable per model: each judge is scored independently (its own samples, its own
+    # median, its own pass/fail against the case's `min_score`), then
+    #   • `aggregate`     combines the scores into the one the report and the baseline
+    #                     read (:median | :mean | :min — :min is the strict panel),
+    #   • `min_agreement` decides the verdict: the FRACTION of judges that must pass on
+    #                     their own. 0.5 = a majority; 1.0 = unanimous.
+    # `quorum` still applies, per judge, so a panel can also be sampled.
+    class Judge
+      # `judges` carries the per-model scores, so a split panel is visible in the report
+      # instead of hiding inside an average.
+      Verdict = Struct.new(:score, :pass, :reason, :judges, keyword_init: true)
+
+      DEFAULT_MIN_SCORE = 0.7
+      AGGREGATES = %i[median mean min].freeze
+
+      # The store's operating policy, stated to the judge in the same words the
+      # deterministic layer checks. Empty when the store has no opinion — then the
+      # rubric alone decides, and inventing a default here would be inventing an
+      # opinion for someone else's store.
+      POLICY_INSTRUCTIONS = {
+        "ask_once" => "This store allows AT MOST ONE question per reply. Two questions in " \
+                      "one message is a failure even if the content is otherwise good.",
+        "investigate_first" => "This store wants the objective established BEFORE acting: on a " \
+                               "vague request the assistant should ask (once or twice, not a " \
+                               "form), not search immediately.",
+        "act_fast" => "This store wants the assistant to ACT on the first plausible reading and " \
+                      "refine after — asking something it could have answered by searching is a " \
+                      "failure."
+      }.freeze
+
+      # ask:  ->(prompt) { "<raw model text>" } — one judge (kept: the common case).
+      # asks: [callable, …] — a panel, one entry per model.
+      def initialize(ask: nil, asks: nil, quorum: 1, aggregate: :median, min_agreement: 0.5)
+        @asks = Array(asks || ask).compact
+        raise ArgumentError, "a judge needs at least one `ask`" if @asks.empty?
+
+        @quorum = [quorum.to_i, 1].max
+        @aggregate = aggregate.to_s.to_sym
+        raise ArgumentError, "unknown aggregate: #{aggregate}" unless AGGREGATES.include?(@aggregate)
+
+        @min_agreement = min_agreement.to_f.clamp(0.0, 1.0)
+      end
+
+      # Golden + the LAST TurnResult -> Verdict, or nil when there's nothing to judge
+      # (no rubric). `min_score` comes from the golden (default 0.7).
+      def score(golden:, result:)
+        rubric = golden.rubric.to_s.strip
+        return nil if rubric.empty?
+
+        prompt = build_prompt(rubric, golden.user_turns, result.output_text.to_s, golden.policy)
+        min = golden.min_score || DEFAULT_MIN_SCORE
+        panel = @asks.map { |ask| judge_once(ask, prompt, min) }
+
+        agreed = panel.count { |j| j[:pass] }
+        Verdict.new(score: combine(panel.map { |j| j[:score] }).round(3),
+                    pass: (agreed.to_f / panel.length) >= @min_agreement,
+                    reason: panel.map { |j| j[:reason] }.reject(&:empty?).first.to_s,
+                    judges: panel.map { |j| j[:score] })
+      end
+
+      private
+
+      # One model's verdict: its own samples, its own median, its own pass/fail.
+      def judge_once(ask, prompt, min)
+        samples = Array.new(@quorum) { parse(ask.call(prompt).to_s) }
+        med = median(samples.map { |s| s[:score] })
+        { score: med, pass: med >= min,
+          reason: samples.map { |s| s[:reason] }.compact.reject(&:empty?).first.to_s }
+      end
+
+      def combine(scores)
+        case @aggregate
+        when :mean then scores.empty? ? 0.0 : scores.sum / scores.length.to_f
+        when :min then scores.min || 0.0
+        else median(scores)
+        end
+      end
+
+      # The `policy` is the one thing a rubric cannot carry alone: how
+      # much this store wants the agent to ask before acting is a per-store decision,
+      # and a judge that is not TOLD it will guess — half the time wrongly. The
+      # deterministic half is `Assertions.policy_checks`; this is the other half.
+      def build_prompt(rubric, user_turns, reply, policy = nil)
+        <<~PROMPT
+          You are a strict QA judge for a customer-service AI assistant. Judge the
+          ASSISTANT REPLY against the RUBRIC — nothing else.
+
+          RUBRIC:
+          #{rubric}
+          #{policy_clause(policy)}
+          CONVERSATION (user turns, in order):
+          #{user_turns.map { |t| "- #{t}" }.join("\n")}
+
+          ASSISTANT REPLY:
+          #{reply}
+
+          Score from 0.0 (fails the rubric) to 1.0 (fully meets it). Respond with ONLY a
+          JSON object, no prose:
+          {"score": <0..1>, "reason": "<one short sentence>"}
+        PROMPT
+      end
+
+      def policy_clause(policy)
+        instruction = POLICY_INSTRUCTIONS[policy.to_s]
+        return "" unless instruction
+
+        "\nSTORE POLICY (weigh this as part of the rubric):\n#{instruction}\n"
+      end
+
+      # Extracts the first {...} block and parses it. Any failure (no JSON, bad JSON,
+      # non-numeric score) -> score 0.0 with a diagnostic reason, so a broken judge
+      # never masquerades as a pass. Score is clamped to [0,1].
+      def parse(raw)
+        block = raw[/\{.*\}/m]
+        raise JSON::ParserError, "no JSON object" unless block
+
+        obj = JSON.parse(block)
+        score = Float(obj["score"])
+        { score: score.clamp(0.0, 1.0), reason: obj["reason"].to_s }
+      rescue JSON::ParserError, ArgumentError, TypeError
+        { score: 0.0, reason: "unparseable judge output: #{raw.to_s[0, 120].inspect}" }
+      end
+
+      def median(nums)
+        sorted = nums.compact.sort
+        return 0.0 if sorted.empty?
+
+        mid = sorted.length / 2
+        sorted.length.odd? ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2.0
+      end
+    end
+
+    # Builds the configured judge panel from `settings["evals"]`.
+    #
+    # It lives HERE and not in `evals/run.rb` because the CLI is no longer the only
+    # caller: the refinement gate scores a candidate with the SAME judges the operator
+    # configured, and is explicit that a second copy of the judge would be the
+    # worst possible outcome — the gate would then be grading against a rubric nobody
+    # tuned. One builder, two callers.
+    module JudgePanel
+      module_function
+
+      # settings: the `evals` hash (judges/quorum/aggregate/min_agreement).
+      # overrides: the CLI's flags, which win over the stored config.
+      # -> [Judge, [model names]] | nil when nobody is configured to ask. NIL AND NOT
+      # a no-op judge: a rubric'd case with no judge reads as `judge_pending`, which is
+      # visible, where a judge that always passes would be silent.
+      # `llm`: the graph's own RubyLLM context; nil = the global
+      # constant. Only the deployment root and the CLI build judges today, and both
+      # are one graph per process — the seam keeps the default honest for the day
+      # an embedded graph gates a candidate on its own credentials.
+      def build(settings, overrides: {}, chat_factory: nil, llm: nil)
+        settings = Coercion.deep_stringify(settings || {})
+        overrides = overrides.transform_keys(&:to_s)
+        models = resolve_models(settings, overrides)
+        return nil if models.empty?
+
+        factory = chat_factory || ->(model, provider) { ruby_llm_ask(model, provider, llm: llm) }
+        judge = Judge.new(
+          asks: models.map { |m| factory.call(m["model"], m["provider"]) },
+          quorum: overrides["quorum"] || settings["quorum"] || 1,
+          aggregate: overrides["aggregate"] || settings["aggregate"] || "median",
+          min_agreement: overrides["min_agreement"] || settings["min_agreement"] || 0.5
+        )
+        [judge, models.map { |m| m["model"] }]
+      end
+
+      # Sugar for the callers that only want the judge (the gate).
+      def judge(settings, **kw) = build(settings, **kw)&.first
+
+      # The SAME configured models, asked a different question. One
+      # builder because "who judges here" is one operator decision: a pairwise panel
+      # configured apart from the rubric panel would let a run be graded by judges
+      # nobody chose. -> [Pairwise, [model names]] | nil when nobody is configured.
+      def pairwise(settings, overrides: {}, chat_factory: nil, llm: nil)
+        settings = Coercion.deep_stringify(settings || {})
+        models = resolve_models(settings, overrides.transform_keys(&:to_s))
+        return nil if models.empty?
+
+        factory = chat_factory || ->(model, provider) { ruby_llm_ask(model, provider, llm: llm) }
+        [Pairwise.new(asks: models.map { |m| factory.call(m["model"], m["provider"]) }),
+         models.map { |m| m["model"] }]
+      end
+
+      def resolve_models(settings, overrides)
+        models = if Coercion.present?(overrides["judge_model"])
+                   [{ "model" => overrides["judge_model"], "provider" => overrides["judge_provider"] }]
+                 else
+                   Array(settings["judges"])
+                 end
+        models.map { |m| Coercion.deep_stringify(m) }.reject { |m| m["model"].to_s.strip.empty? }
+      end
+
+      # The default way to reach a model: RubyLLM, temperature 0, required lazily so
+      # nothing here loads a provider gem until a judge is actually configured.
+      def ruby_llm_ask(model, provider, llm: nil)
+        require "ruby_llm"
+        llm ||= RubyLLM
+        lambda do |prompt|
+          llm.chat(model: model, provider: provider, assume_model_exists: true)
+             .with_temperature(0).ask(prompt).content
+        end
+      end
+    end
+  end
+end

data/lib/insika/evals/pairwise.rb

@@ -0,0 +1,178 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Insika
+  module Evals
+    # PAIRWISE AGAINST THE INCUMBENT — the number that answers "can we
+    # replace it". An absolute 0.72 says a reply cleared a bar we invented; it says
+    # nothing about whether the system already answering 403,231 chats would have done
+    # better with the same customer.
+    #
+    # Same opening, two transcripts, one question: which one served the customer
+    # better? Three outcomes (better / comparable / worse), plus two the panel can
+    # produce and must not hide — `split` when the judges disagree and `unknown` when
+    # none of them answered in a readable way.
+    #
+    # Three honesty rules, each of them the difference between a number people should
+    # trust and one they should not:
+    #
+    # 1. **Anonymous.** The judge sees "A" and "B" and is never told which one is
+    #    Insika. Told, it would have an opinion about the new system rather than about
+    #    the conversations.
+    # 2. **Both orders, every judge.** Position bias is THE known failure of pairwise
+    #    LLM judging, and it is the same species as the kill criterion ("`better`
+    #    tracks length, or politeness"). So each judge is asked twice with the
+    #    transcripts swapped, and a verdict that FLIPS with presentation order is
+    #    reported as `comparable` with `order_dependent: true` — a preference that
+    #    depends on which one was printed first is not a preference.
+    # 3. **A split panel stays split.** Averaging "better" and "worse" into
+    #    "comparable" invents agreement that nobody expressed.
+    #
+    # Cost: 2 provider calls per judge per case. This is why it never runs as
+    # part of the gate and is opt-in on the CLI.
+    class Pairwise
+      BETTER = "better"
+      COMPARABLE = "comparable"
+      WORSE = "worse"
+      SPLIT = "split"
+      UNKNOWN = "unknown"
+
+      # `vs` names WHO the reference half actually is: `agent` (model against model)
+      # or `human-assisted` (a person typed part of it,'s `origin: operator`).
+      # It rides on the verdict rather than beside it so a report cannot print the
+      # outcome without the label.
+      Verdict = Struct.new(:outcome, :reason, :vs, :judges, :order_dependent, keyword_init: true) do
+        def human_assisted? = vs == "human-assisted"
+        def decided? = [BETTER, COMPARABLE, WORSE].include?(outcome)
+      end
+
+      # asks: [->(prompt) { "<raw model text>" }] — the SAME panel the rubric judge
+      # uses (JudgePanel builds both), because "the judges the operator configured" is
+      # one decision, not two.
+      def initialize(asks:)
+        @asks = Array(asks).compact
+        raise ArgumentError, "a pairwise comparison needs at least one `ask`" if @asks.empty?
+      end
+
+      # golden + the run's [TurnResult] -> Verdict, or nil when the case carries no
+      # reference (most of them: a pair is curated by a human, like the case itself).
+      def compare(golden:, turns:)
+        return nil unless golden.reference?
+
+        ours = Pairwise.transcript(golden.user_turns, turns)
+        theirs = Pairwise.reference_transcript(golden.reference_messages)
+        return nil if ours.strip.empty?
+
+        panel = @asks.map { |ask| judge_once(ask, ours, theirs) }
+        combine(panel, vs: golden.human_assisted? ? "human-assisted" : "agent")
+      end
+
+      # The replayed conversation as the judge reads it: the user turns we sent,
+      # interleaved with the answers the deployment published. `turns` may be shorter
+      # than `user_turns` (an errored turn aborts the replay) — zip on what ran.
+      def self.transcript(user_turns, turns)
+        Array(turns).each_with_index.map do |t, i|
+          ["customer: #{user_turns[i]}", "assistant: #{t.output_text.to_s.strip}"]
+        end.flatten.join("\n")
+      end
+
+      # The incumbent's half. Human turns are NOT flagged to the judge: what it grades
+      # is the conversation as the customer received it, and telling it "a person wrote
+      # this one" is an invitation to grade the author instead. The fact is carried to
+      # the READER as `vs: human-assisted` instead, which is where it changes a decision.
+      def self.reference_transcript(messages)
+        Array(messages).map do |m|
+          speaker = m["role"].to_s == "user" ? "customer" : "assistant"
+          "#{speaker}: #{m['text'].to_s.strip}"
+        end.join("\n")
+      end
+
+      private
+
+      # One model, asked twice with the sides swapped. -> { outcome:, reason:,
+      # order_dependent: }
+      def judge_once(ask, ours, theirs)
+        first = outcome_of(parse(ask.call(prompt(ours, theirs))), ours_is: "A")
+        second = outcome_of(parse(ask.call(prompt(theirs, ours))), ours_is: "B")
+
+        return { outcome: UNKNOWN, reason: [first[:reason], second[:reason]].compact.first.to_s, order_dependent: false } \
+          if first[:outcome] == UNKNOWN && second[:outcome] == UNKNOWN
+
+        reason = [first[:reason], second[:reason]].reject { |r| r.to_s.empty? }.first.to_s
+        agreed = [first[:outcome], second[:outcome]].reject { |o| o == UNKNOWN }.uniq
+        return { outcome: agreed.first, reason: reason, order_dependent: false } if agreed.length == 1
+
+        { outcome: COMPARABLE, reason: reason, order_dependent: true }
+      end
+
+      # A strict majority decides; anything less is `split`. Unknown judges are left
+      # OUT of the tally (they expressed nothing) but stay visible in `judges`.
+      def combine(panel, vs:)
+        voted = panel.reject { |j| j[:outcome] == UNKNOWN }
+        winner, group = voted.group_by { |j| j[:outcome] }.max_by { |_, js| js.length }
+        outcome = if voted.empty? then UNKNOWN
+                  elsif group.length * 2 > voted.length then winner
+                  else SPLIT
+                  end
+        # The reason comes from the judges that CARRIED the verdict, so a split or an
+        # unknown is explained by whoever produced it rather than by an outvoted one.
+        spoke = outcome == winner ? group : panel
+
+        Verdict.new(outcome: outcome, vs: vs,
+                    reason: spoke.map { |j| j[:reason].to_s }.reject(&:empty?).first.to_s,
+                    judges: panel.map { |j| j[:outcome] },
+                    order_dependent: panel.any? { |j| j[:order_dependent] })
+      end
+
+      # "A won" means Insika won only when Insika WAS A. The mapping is the whole
+      # point of asking twice.
+      def outcome_of(parsed, ours_is:)
+        winner = parsed[:winner]
+        outcome = case winner
+                  when "tie" then COMPARABLE
+                  when nil then UNKNOWN
+                  else winner == ours_is ? BETTER : WORSE
+                  end
+        { outcome: outcome, reason: parsed[:reason] }
+      end
+
+      def prompt(a, b)
+        <<~PROMPT
+          You are comparing two customer-service conversations that began with the SAME
+          customer message. Decide which one SERVED THE CUSTOMER BETTER: did the customer
+          get what they came for, without detours, wrong information or invented facts?
+
+          Ignore length, tone, politeness, emoji and formatting UNLESS they changed what
+          the customer actually got. A short answer that solves the problem beats a long
+          one that does not.
+
+          CONVERSATION A:
+          #{a}
+
+          CONVERSATION B:
+          #{b}
+
+          Respond with ONLY a JSON object, no prose:
+          {"winner": "A" | "B" | "tie", "reason": "<one short sentence>"}
+        PROMPT
+      end
+
+      # An unreadable reply is UNKNOWN, never a preference: scoring it as a tie would
+      # quietly count a broken judge as evidence that the two systems are equivalent.
+      def parse(raw)
+        block = raw.to_s[/\{.*\}/m]
+        raise JSON::ParserError, "no JSON object" unless block
+
+        obj = JSON.parse(block)
+        winner = obj["winner"].to_s.strip.upcase
+        winner = "tie" if %w[TIE DRAW EQUAL COMPARABLE].include?(winner)
+        raise JSON::ParserError, "unknown winner" unless %w[A B tie].include?(winner)
+
+        { winner: winner, reason: obj["reason"].to_s }
+      rescue JSON::ParserError, TypeError
+        { winner: nil, reason: "unparseable pairwise output: #{raw.to_s[0, 120].inspect}" }
+      end
+    end
+  end
+end

data/lib/insika/evals/report.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Insika
+  module Evals
+    # Renders a run's [CaseResult] as a machine-readable JSON blob (for the baseline
+    # gating in) and a human-readable markdown summary.
+    # Pure over the results — takes a clock value in, never reads it (so callers stay
+    # deterministic/testable).
+    module Report
+      module_function
+
+      # -> Hash ready for JSON. `at` is an ISO-8601 string stamped by the caller.
+      def to_h(results, at:)
+        passed = results.count(&:pass?)
+        skipped = results.count(&:skipped?)
+        {
+          "at" => at,
+          "total" => results.size,
+          "passed" => passed,
+          # A skipped case is neither: counting it as failed is the lie this outcome
+          # exists to stop, and counting it as passed is worse.
+          "failed" => results.size - passed - skipped,
+          "skipped" => skipped,
+          "judge_pending" => results.count(&:judge_pending?),
+          # Absent (not an empty tally) when nothing was compared — a run with no
+          # pairwise is the normal one, and a zeroed block reads like every case tied.
+          "pairwise" => pairwise_summary(results),
+          "cases" => results.map do |r|
+            {
+              "id" => r.id, "agent" => r.agent, "pass" => r.pass?,
+              "skipped" => r.skipped,
+              "judge_pending" => r.judge_pending?, "error" => r.error,
+              "checks" => r.checks.map { |c| { "name" => c.name, "pass" => c.pass, "detail" => c.detail } },
+              "judge" => (r.judge && { "score" => r.judge.score, "pass" => r.judge.pass, "reason" => r.judge.reason }),
+              "pairwise" => (r.pairwise && { "outcome" => r.pairwise.outcome, "vs" => r.pairwise.vs,
+                                             "reason" => r.pairwise.reason, "judges" => r.pairwise.judges,
+                                             "order_dependent" => r.pairwise.order_dependent })
+            }
+          end
+        }.compact
+      end
+
+      # -> counts by outcome, or nil when no case carried a reference. `human_assisted`
+      # is counted separately and always printed: a "better" against a conversation a
+      # PERSON typed is a different claim from one against the incumbent's model, and
+      # the two must never be summed into one number somebody quotes.
+      def pairwise_summary(results)
+        compared = results.filter_map(&:pairwise)
+        return nil if compared.empty?
+
+        counts = compared.group_by(&:outcome).transform_values(&:size)
+        { "compared" => compared.size,
+          "human_assisted" => compared.count(&:human_assisted?),
+          "order_dependent" => compared.count(&:order_dependent),
+          "outcomes" => counts }
+      end
+
+      def to_json(results, at:)
+        JSON.pretty_generate(to_h(results, at: at))
+      end
+
+      PAIRWISE_MARK = { "better" => "🟢", "comparable" => "🟡", "worse" => "🔴",
+                        "split" => "⚖️", "unknown" => "❔" }.freeze
+
+      # Never without `vs:` — see `pairwise_summary`.
+      def pairwise_line(v)
+        flip = v.order_dependent ? " (order-dependent)" : ""
+        "#{PAIRWISE_MARK.fetch(v.outcome, '·')} vs incumbent (#{v.vs}): #{v.outcome}#{flip} — #{v.reason}"
+      end
+
+      def pairwise_block(summary)
+        counts = summary["outcomes"].map { |k, n| "#{n} #{k}" }.join(" · ")
+        lines = ["", "**vs incumbent** (#{summary['compared']} compared): #{counts}"]
+        if summary["human_assisted"].positive?
+          lines << "- #{summary['human_assisted']} against a HUMAN-ASSISTED transcript (a person typed " \
+                   "part of the reference — not a model-vs-model result)"
+        end
+        if summary["order_dependent"].positive?
+          lines << "- #{summary['order_dependent']} flipped when the transcripts were swapped, " \
+                   "and were reported as comparable"
+        end
+        lines
+      end
+
+      # Human summary. One line per case; failing checks nested underneath.
+      def to_markdown(results, at:)
+        h = to_h(results, at: at)
+        lines = ["# Eval report — #{at}", "",
+                 "**#{h['passed']}/#{h['total'] - h['skipped']} passed** · #{h['failed']} failed" \
+                 "#{" · #{h['skipped']} skipped" if h['skipped'].positive?}" \
+                 "#{" · #{h['judge_pending']} awaiting judge" if h['judge_pending'].positive?}", ""]
+        results.each do |r|
+          if r.skipped?
+            # WITH the reason, always: "12 skipped" alone is indistinguishable from a
+            # suite that quietly stopped testing anything.
+            lines << "- ⏭️ `#{r.id}` (#{r.agent}) — skipped: #{r.skipped}"
+            next
+          end
+
+          lines << "- #{r.pass? ? '✅' : '❌'} `#{r.id}` (#{r.agent})#{'  ⏳ judge pending' if r.judge_pending?}"
+          r.failures.each { |c| lines << "    - ❌ #{c.name}: #{c.detail}" }
+          if r.judge
+            v = r.judge
+            lines << "    - #{v.pass ? '✅' : '❌'} judge: #{v.score} — #{v.reason}"
+          end
+          lines << "    - #{pairwise_line(r.pairwise)}" if r.pairwise
+        end
+        lines.concat(pairwise_block(h["pairwise"])) if h["pairwise"]
+        "#{lines.join("\n")}\n"
+      end
+    end
+  end
+end

data/lib/insika/evals/runner.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+require_relative "golden"
+require_relative "assertions"
+
+module Insika
+  module Evals
+    # Replays golden cases through a Transport and evaluates each. Pure over the
+    # Transport (injected) — the fake makes the orchestration unit-testable offline;
+    # the HttpTransport makes a real run.
+    #
+    # Multi-turn: a case's turns replay IN ORDER under one conversation id
+    # (`eval-<id>`); earlier turns build context, the assertion runs on the LAST
+    # turn's result. A turn that errors aborts the rest of that conversation (there's
+    # nothing to continue from) and fails the case.
+    class Runner
+      # `tokens` is what the whole case cost, summed over its turns, or nil when no
+      # turn reported usage. `cached` is how much of that was served from the prompt
+      # cache, carried separately because it is the number that explains a total.
+      # Only the refinement gate reads them (records a run's cost); the
+      # report and the exit code are untouched.
+      RunCase = Struct.new(:result, :timings, :tokens, :cached, keyword_init: true)
+
+      # judge: an Evals::Judge (optional). When set, a case with a rubric whose turn
+      # ran cleanly gets a subjective verdict attached on top of the deterministic pass.
+      #
+      # capabilities: what the DEPLOYMENT has, per agent — anything answering
+      # `#for(agent_id)` with { "tools" =>, "capabilities" => } or nil. Used to skip a
+      # case the deployment cannot satisfy, BEFORE spending a turn on
+      # it. nil (or an unknown agent) = no resolution, and then a case with `requires`
+      # RUNS and says so in the report: "could not rule it out" is not a reason to
+      # stop testing something, and a suite that shrinks in silence is the failure
+      # this feature exists to avoid.
+      #
+      # pairwise: an Evals::Pairwise (optional). Only cases carrying a
+      # `reference:` are compared, and the verdict never touches pass/fail — it is the
+      # answer to "can we replace it", reported beside the suite's own verdict.
+      def initialize(transport:, judge: nil, conv_map: {}, capabilities: nil, pairwise: nil)
+        @transport = transport
+        @judge = judge
+        @conv_map = conv_map || {}
+        @capabilities = capabilities
+        @pairwise = pairwise
+      end
+
+      # [Golden] -> [RunCase]. Each RunCase carries the CaseResult (for the report) +
+      # per-turn timings (for `--mode perf`).
+      def run(goldens)
+        goldens.map { |g| run_case(g) }
+      end
+
+      def run_case(golden)
+        skip = skip_reason(golden)
+        return RunCase.new(result: Assertions.skip(golden, skip), timings: []) if skip
+
+        # A backend that resolves state from a pre-existing conversation (e.g. a
+        # consumer needing a real Chat UUID as X-Chat-Id) supplies it via conv_map; otherwise the
+        # synthetic "eval-<id>" keeps the adapter's own multi-turn continuation.
+        conv = @conv_map[golden.id] || "eval-#{golden.id}"
+        turns = []
+        timings = []
+        spent = []
+        cached = []
+        golden.user_turns.each do |message|
+          outcome = @transport.turn(agent: golden.agent, conv: conv, message: message)
+          timings << { ttfb: outcome.ttfb, total: outcome.total }
+          spent << billed_tokens(outcome.usage)
+          cached << cached_tokens(outcome.usage)
+          turns << outcome.result
+          break if outcome.result.error
+        end
+        last = turns.last
+        # Tool/content assertions read the last turn (unchanged); the policy checks
+        # read every turn — "one question per reply" is a rule about each of them.
+        result = Assertions.evaluate(golden, last, turns: turns)
+        # Subjective layer: only when a judge is configured, the case has a rubric, and
+        # the turn ran cleanly (nothing to judge on an errored turn).
+        result.judge = @judge.score(golden: golden, result: last) if @judge && result.rubric && result.error.nil?
+        # Against the incumbent. Same rule as the judge: nothing to
+        # compare on a turn that errored — half a conversation would lose the
+        # comparison for a reason that has nothing to do with the agent.
+        result.pairwise = @pairwise.compare(golden: golden, turns: turns) if @pairwise && result.error.nil?
+        RunCase.new(result: result, timings: timings, tokens: sum_tokens(spent),
+                    cached: sum_tokens(cached))
+      end
+
+      private
+
+      # nil when NO turn reported usage; otherwise the sum of the ones that did. A
+      # partially-metered case is reported as what was actually measured, low rather
+      # than absent — a budget under-counting is a smaller lie than a budget that
+      # throws the number away because one leg was silent.
+      def sum_tokens(spent)
+        counted = spent.compact
+        counted.empty? ? nil : counted.sum
+      end
+
+      # What the turn actually SENT, cache included. The engine's `total_tokens` is
+      # input + output and DELIBERATELY excludes the cached prefix (`Executor#usage_of`
+      # reports `cached_tokens` alongside it), so reading it as the cost of a turn
+      # under-reads a cached identity by an order of magnitude.
+      #
+      # Measured on the pilot: one real turn reports `total_tokens: 88` with
+      # `cached_tokens: 26624`. A refinement budget built on the first number would
+      # have let a run send ~300× what its ceiling said. Cached tokens are cheaper
+      # than fresh ones; they are not free, and a ceiling has to see them.
+      def billed_tokens(usage)
+        return nil unless usage.is_a?(Hash)
+
+        total = read(usage, "total_tokens")
+        return nil if total.nil?
+
+        total + cached_tokens(usage).to_i
+      end
+
+      def cached_tokens(usage)
+        return nil unless usage.is_a?(Hash)
+
+        values = [read(usage, "cached_tokens"), read(usage, "cache_creation_tokens")].compact
+        values.empty? ? nil : values.sum
+      end
+
+      def read(usage, key)
+        value = usage[key] || usage[key.to_sym]
+        value&.to_i
+      end
+
+      # -> the reason to skip, or nil to run. A case with no `requires` always runs
+      # (and never even asks), which keeps the whole existing corpus untouched.
+      def skip_reason(golden)
+        return nil unless golden.requirements?
+
+        available = @capabilities&.for(golden.agent)
+        return nil if available.nil? # unresolved: run it, and the report says so
+
+        unmet = Assertions.unmet_requirements(golden, available)
+        unmet.empty? ? nil : unmet.join("; ")
+      end
+    end
+  end
+end