insika 0.0.1 → 0.1.0

data/lib/insika/evals/baseline.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Insika
+  module Evals
+    # Fase C gating (RFC-0008 §3.4). A baseline is the accepted state of the golden
+    # set — `{ cases: { id => { pass, score } } }`. A gated run compares against it and
+    # blocks only on a REGRESSION, so known-failing cases don't wedge the gate while a
+    # real drop (a passing case that now fails, or a judge score that fell past the
+    # tolerance) does. That's the pre-merge gate for prompt/tool/model changes.
+    module Baseline
+      Regression = Struct.new(:id, :kind, :detail, keyword_init: true)
+
+      module_function
+
+      # [CaseResult] -> baseline hash. `at` is stamped by the caller (kept out of here
+      # so the module stays deterministic/testable).
+      # A SKIPPED case is left out entirely (RFC-0014 §3.2): writing it as `pass:
+      # false` would accept "this deployment cannot run it" as the accepted state,
+      # and the case would never block anywhere again.
+      def snapshot(results, at:)
+        {
+          "at" => at,
+          "cases" => results.reject(&:skipped?).each_with_object({}) do |r, h|
+            h[r.id] = { "pass" => r.pass?, "score" => (r.judge&.score) }
+          end
+        }
+      end
+
+      def load(path) = JSON.parse(File.read(path))
+
+      def write(path, results, at:)
+        File.write(path, JSON.pretty_generate(snapshot(results, at: at)))
+      end
+
+      # Compares a run against a loaded baseline. -> [Regression]. Only cases present
+      # in BOTH are compared: a new case (no baseline entry) never blocks the gate (it
+      # shows in the report as ❌ but is not a "regression"); document this in README.
+      #   • pass→fail   : baseline pass, now failing (hard regression).
+      #   • pass→skipped: baseline pass, now unrunnable HERE. RFC-0014 §3.2 says the
+      #                   gate never blocks ON a skip, and it does not: a case that was
+      #                   already skipped or unknown stays silent. But a case that used
+      #                   to run on this deployment and no longer can means the agent
+      #                   lost a tool or a declaration — a suite shrinking in silence is
+      #                   exactly what this outcome was added to prevent. Re-baseline if
+      #                   the shrink is intended.
+      #   • score-drop  : baseline score - current judge score > tolerance (quality
+      #                   drift, even if the case still technically passes).
+      def compare(results, baseline, tolerance:)
+        base = baseline["cases"] || {}
+        results.filter_map do |r|
+          b = base[r.id]
+          next unless b
+
+          if b["pass"] && r.skipped?
+            Regression.new(id: r.id, kind: "pass→skipped",
+                           detail: "was passing, now unrunnable here: #{r.skipped}")
+          elsif b["pass"] && !r.pass?
+            Regression.new(id: r.id, kind: "pass→fail", detail: "was passing, now failing")
+          elsif b["score"] && r.judge && (b["score"] - r.judge.score) > tolerance
+            Regression.new(id: r.id, kind: "score-drop",
+                           detail: "judge #{b['score']} -> #{r.judge.score} (> #{tolerance})")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/insika/evals/golden.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+# Evals — the quality harness (RFC-0008). It lives in `lib/` so the engine itself can
+# call it (the refinement gate of RFC-0013 needs to score a candidate agent, and a
+# second copy of the judge would be the worst possible outcome), but it stays a
+# CLIENT: it reaches a running deployment over HTTP through `HttpTransport` and never
+# reads a store directly. `evals/run.rb` is a thin CLI over this module.
+module Insika
+  module Evals
+    # A curated behavior case, loaded from a data file (evals/golden/<agent>/*.yml).
+    # Data, not code — same spirit as tools-as-data. See evals/README.md for the format.
+    Golden = Struct.new(:id, :agent, :turns, :expect, :requires, :reference, :source, keyword_init: true) do
+      # The user messages to replay, in order.
+      def user_turns = turns.map { |t| t["user"] }
+
+      # Tool refs the case expects; a trailing "?" marks OPTIONAL (never fails).
+      # -> [{ name:, optional: }]
+      def tools_called
+        Array(expect["tools_called"]).map do |ref|
+          s = ref.to_s
+          optional = s.end_with?("?")
+          { name: optional ? s[0..-2] : s, optional: optional }
+        end
+      end
+
+      # Names of the negative assertions to run (e.g. "pii_leak", "tool_error").
+      def must_not = Array(expect["must_not"]).map(&:to_s)
+
+      # How much the agent should ask before acting (RFC-0014 §3.3). nil = the store
+      # has no opinion and only the rubric decides.
+      def policy = GoldenLoader.presence(expect["policy"])
+
+      # What the DEPLOYMENT must have for this case to mean anything (RFC-0014 §3.2).
+      # Empty = runs everywhere.
+      def required_tools = Array(requires["tools"]).map(&:to_s)
+      def required_capabilities = Array(requires["capabilities"]).map(&:to_s)
+      def requirements? = !(required_tools + required_capabilities).empty?
+
+      # THE INCUMBENT'S CONVERSATION for the same opening (RFC-0014 §3.4) — the other
+      # half of a pairwise comparison. Data in the case, not a store read: the eval is
+      # a client, and a pair that lives in one reviewable file cannot go stale against
+      # a database nobody looked at.
+      def reference_messages = Array(reference["messages"])
+      def reference_source = GoldenLoader.presence(reference["source"])
+      def reference? = !reference_messages.empty?
+
+      # Did a PERSON type part of the reference half? After a handoff the operator's
+      # words are stored as `role: assistant` (P23a), and comparing a model to a human
+      # and calling it a win is a lie in both directions — so the pair is LABELLED and
+      # the report never prints the outcome without it.
+      def human_assisted?
+        reference_messages.any? { |m| MessageOrigin.origin_of(m) == MessageOrigin::OPERATOR }
+      end
+
+      # LLM-judge rubric + threshold (consumed in Fase B — deferred here).
+      def rubric = expect["rubric"]
+      def min_score = expect["min_score"]
+    end
+
+    # Loads + validates golden files. Fails LOUD on a malformed case — a silently
+    # dropped golden is a hole in the safety net.
+    module GoldenLoader
+      class InvalidGolden < StandardError; end
+
+      module_function
+
+      # Loads every *.yml/*.yaml under `dir` (recursive), sorted by path for a stable
+      # run order. -> [Golden].
+      def load_dir(dir)
+        Dir.glob(File.join(dir, "**", "*.{yml,yaml}")).sort.map { |f| load_file(f) }
+      end
+
+      def load_file(path)
+        raw = YAML.safe_load(File.read(path), permitted_classes: [], aliases: false) || {}
+        build(raw, source: path)
+      rescue Psych::SyntaxError => e
+        raise InvalidGolden, "#{path}: invalid YAML — #{e.message}"
+      end
+
+      # hash (string keys) -> validated Golden. `source` is only for error messages.
+      def build(raw, source: "(inline)")
+        raise InvalidGolden, "#{source}: golden must be a mapping" unless raw.is_a?(Hash)
+
+        id = presence(raw["id"]) || (raise InvalidGolden, "#{source}: 'id' is required")
+        agent = presence(raw["agent"]) || (raise InvalidGolden, "#{source}: 'agent' is required (case '#{id}')")
+        turns = normalize_turns(raw["turns"], id: id, source: source)
+        expect = raw["expect"] || {}
+        raise InvalidGolden, "#{source}: 'expect' must be a mapping (case '#{id}')" unless expect.is_a?(Hash)
+
+        validate_policy!(expect["policy"], id: id, source: source)
+        requires = raw["requires"] || {}
+        unless requires.is_a?(Hash)
+          raise InvalidGolden, "#{source}: 'requires' must be a mapping (case '#{id}')"
+        end
+
+        reference = normalize_reference(raw["reference"], id: id, source: source)
+
+        Golden.new(id: id, agent: agent, turns: turns, expect: expect,
+                   requires: requires, reference: reference, source: source)
+      end
+
+      # reference: { "source" => String?, "messages" => [{ "role" =>, "text" =>,
+      # "origin" => }] }. Absent -> {}, and the case simply has nothing to compare
+      # against. Malformed is REFUSED: a reference that half-loads would produce a
+      # pairwise verdict about a transcript nobody wrote.
+      def normalize_reference(raw, id:, source:)
+        return {} if raw.nil?
+        raise InvalidGolden, "#{source}: 'reference' must be a mapping (case '#{id}')" unless raw.is_a?(Hash)
+
+        messages = raw["messages"]
+        unless messages.is_a?(Array) && !messages.empty?
+          raise InvalidGolden, "#{source}: reference needs a non-empty 'messages' array (case '#{id}')"
+        end
+
+        { "source" => presence(raw["source"]),
+          "messages" => messages.each_with_index.map { |m, i| reference_message(m, i, id: id, source: source) } }.compact
+      end
+
+      def reference_message(raw, index, id:, source:)
+        where = "#{source}: reference.messages[#{index}] (case '#{id}')"
+        raise InvalidGolden, "#{where} must be a mapping" unless raw.is_a?(Hash)
+
+        role = presence(raw["role"])
+        raise InvalidGolden, "#{where} needs a 'role' of user or assistant" unless %w[user assistant].include?(role)
+
+        text = presence(raw["text"]) || (raise InvalidGolden, "#{where} needs a non-empty 'text'")
+        # The SAME closed vocabulary the engine stamps (P23a). A typo'd marker would
+        # read as "absent" downstream, which is how a human turn gets scored as the
+        # incumbent's model.
+        origin = begin
+          MessageOrigin.parse!(raw["origin"])
+        rescue Insika::ValidationError => e
+          raise InvalidGolden, "#{where}: #{e.message}"
+        end
+        { "role" => role, "text" => text }.merge(origin ? { "origin" => origin } : {})
+      end
+
+      # A typo'd policy must not silently mean "no policy" — the case would go on
+      # passing while the rule it was written for stopped being checked. The
+      # `Assertions` constant is resolved at CALL time (this file loads first, and
+      # assertions.rb touches `Safety::Detectors` at load time).
+      def validate_policy!(value, id:, source:)
+        name = presence(value)
+        return if name.nil? || Assertions::POLICIES.key?(name)
+
+        raise InvalidGolden, "#{source}: unknown policy #{name.inspect} (case '#{id}') — " \
+                             "known: #{Assertions::POLICIES.keys.join(', ')}"
+      end
+
+      # turns: a non-empty array of { "user" => String }. Rejects anything else so a
+      # typo (e.g. `users:`) surfaces at load time, not as an empty replay.
+      def normalize_turns(turns, id:, source:)
+        unless turns.is_a?(Array) && !turns.empty?
+          raise InvalidGolden, "#{source}: 'turns' must be a non-empty array (case '#{id}')"
+        end
+
+        turns.each_with_index.map do |t, i|
+          user = t.is_a?(Hash) ? presence(t["user"]) : nil
+          user || (raise InvalidGolden, "#{source}: turns[#{i}] needs a non-empty 'user' (case '#{id}')")
+          { "user" => user }
+        end
+      end
+
+      def presence(v)
+        s = v.to_s.strip
+        s.empty? ? nil : s
+      end
+    end
+  end
+end

data/lib/insika/evals/judge.rb

@@ -0,0 +1,225 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Insika
+  module Evals
+    # The LLM-judge (RFC-0008 §3.3, Fase B). Scores a golden's `rubric` against the
+    # actual assistant reply — the subjective layer on top of the deterministic
+    # asserts. Pure over an injected `ask` callable (prompt -> raw model text), so it's
+    # unit-testable without an LLM; the real ask (RubyLLM on the utility_model, temp 0)
+    # is built by the CLI.
+    #
+    # Conservative by construction: an unparseable judge reply scores 0 (fails) rather
+    # than silently passing.
+    #
+    # A PANEL, not a single voice (RFC-0013 §3.9). `quorum: N` samples ONE model N
+    # times, which measures that model's variance and little else — at temperature 0 it
+    # mostly returns the same answer, including the same blind spot. Two DIFFERENT
+    # models disagreeing about a rubric is the signal worth having, so `asks:` takes one
+    # callable per model: each judge is scored independently (its own samples, its own
+    # median, its own pass/fail against the case's `min_score`), then
+    #   • `aggregate`     combines the scores into the one the report and the baseline
+    #                     read (:median | :mean | :min — :min is the strict panel),
+    #   • `min_agreement` decides the verdict: the FRACTION of judges that must pass on
+    #                     their own. 0.5 = a majority; 1.0 = unanimous.
+    # `quorum` still applies, per judge, so a panel can also be sampled.
+    class Judge
+      # `judges` carries the per-model scores, so a split panel is visible in the report
+      # instead of hiding inside an average.
+      Verdict = Struct.new(:score, :pass, :reason, :judges, keyword_init: true)
+
+      DEFAULT_MIN_SCORE = 0.7
+      AGGREGATES = %i[median mean min].freeze
+
+      # The store's operating policy, stated to the judge in the same words the
+      # deterministic layer checks. Empty when the store has no opinion — then the
+      # rubric alone decides, and inventing a default here would be inventing an
+      # opinion for someone else's store.
+      POLICY_INSTRUCTIONS = {
+        "ask_once" => "This store allows AT MOST ONE question per reply. Two questions in " \
+                      "one message is a failure even if the content is otherwise good.",
+        "investigate_first" => "This store wants the objective established BEFORE acting: on a " \
+                               "vague request the assistant should ask (once or twice, not a " \
+                               "form), not search immediately.",
+        "act_fast" => "This store wants the assistant to ACT on the first plausible reading and " \
+                      "refine after — asking something it could have answered by searching is a " \
+                      "failure."
+      }.freeze
+
+      # ask:  ->(prompt) { "<raw model text>" } — one judge (kept: the common case).
+      # asks: [callable, …] — a panel, one entry per model.
+      def initialize(ask: nil, asks: nil, quorum: 1, aggregate: :median, min_agreement: 0.5)
+        @asks = Array(asks || ask).compact
+        raise ArgumentError, "a judge needs at least one `ask`" if @asks.empty?
+
+        @quorum = [quorum.to_i, 1].max
+        @aggregate = aggregate.to_s.to_sym
+        raise ArgumentError, "unknown aggregate: #{aggregate}" unless AGGREGATES.include?(@aggregate)
+
+        @min_agreement = min_agreement.to_f.clamp(0.0, 1.0)
+      end
+
+      # Golden + the LAST TurnResult -> Verdict, or nil when there's nothing to judge
+      # (no rubric). `min_score` comes from the golden (default 0.7).
+      def score(golden:, result:)
+        rubric = golden.rubric.to_s.strip
+        return nil if rubric.empty?
+
+        prompt = build_prompt(rubric, golden.user_turns, result.output_text.to_s, golden.policy)
+        min = golden.min_score || DEFAULT_MIN_SCORE
+        panel = @asks.map { |ask| judge_once(ask, prompt, min) }
+
+        agreed = panel.count { |j| j[:pass] }
+        Verdict.new(score: combine(panel.map { |j| j[:score] }).round(3),
+                    pass: (agreed.to_f / panel.length) >= @min_agreement,
+                    reason: panel.map { |j| j[:reason] }.reject(&:empty?).first.to_s,
+                    judges: panel.map { |j| j[:score] })
+      end
+
+      private
+
+      # One model's verdict: its own samples, its own median, its own pass/fail.
+      def judge_once(ask, prompt, min)
+        samples = Array.new(@quorum) { parse(ask.call(prompt).to_s) }
+        med = median(samples.map { |s| s[:score] })
+        { score: med, pass: med >= min,
+          reason: samples.map { |s| s[:reason] }.compact.reject(&:empty?).first.to_s }
+      end
+
+      def combine(scores)
+        case @aggregate
+        when :mean then scores.empty? ? 0.0 : scores.sum / scores.length.to_f
+        when :min then scores.min || 0.0
+        else median(scores)
+        end
+      end
+
+      # The `policy` is the one thing a rubric cannot carry alone (RFC-0014 §3.3): how
+      # much this store wants the agent to ask before acting is a per-store decision,
+      # and a judge that is not TOLD it will guess — half the time wrongly. The
+      # deterministic half is `Assertions.policy_checks`; this is the other half.
+      def build_prompt(rubric, user_turns, reply, policy = nil)
+        <<~PROMPT
+          You are a strict QA judge for a customer-service AI assistant. Judge the
+          ASSISTANT REPLY against the RUBRIC — nothing else.
+
+          RUBRIC:
+          #{rubric}
+          #{policy_clause(policy)}
+          CONVERSATION (user turns, in order):
+          #{user_turns.map { |t| "- #{t}" }.join("\n")}
+
+          ASSISTANT REPLY:
+          #{reply}
+
+          Score from 0.0 (fails the rubric) to 1.0 (fully meets it). Respond with ONLY a
+          JSON object, no prose:
+          {"score": <0..1>, "reason": "<one short sentence>"}
+        PROMPT
+      end
+
+      def policy_clause(policy)
+        instruction = POLICY_INSTRUCTIONS[policy.to_s]
+        return "" unless instruction
+
+        "\nSTORE POLICY (weigh this as part of the rubric):\n#{instruction}\n"
+      end
+
+      # Extracts the first {...} block and parses it. Any failure (no JSON, bad JSON,
+      # non-numeric score) -> score 0.0 with a diagnostic reason, so a broken judge
+      # never masquerades as a pass. Score is clamped to [0,1].
+      def parse(raw)
+        block = raw[/\{.*\}/m]
+        raise JSON::ParserError, "no JSON object" unless block
+
+        obj = JSON.parse(block)
+        score = Float(obj["score"])
+        { score: score.clamp(0.0, 1.0), reason: obj["reason"].to_s }
+      rescue JSON::ParserError, ArgumentError, TypeError
+        { score: 0.0, reason: "unparseable judge output: #{raw.to_s[0, 120].inspect}" }
+      end
+
+      def median(nums)
+        sorted = nums.compact.sort
+        return 0.0 if sorted.empty?
+
+        mid = sorted.length / 2
+        sorted.length.odd? ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2.0
+      end
+    end
+
+    # Builds the configured judge panel from `settings["evals"]` (RFC-0013 §3.9).
+    #
+    # It lives HERE and not in `evals/run.rb` because the CLI is no longer the only
+    # caller: the refinement gate scores a candidate with the SAME judges the operator
+    # configured, and §3.7 is explicit that a second copy of the judge would be the
+    # worst possible outcome — the gate would then be grading against a rubric nobody
+    # tuned. One builder, two callers.
+    module JudgePanel
+      module_function
+
+      # settings: the `evals` hash (judges/quorum/aggregate/min_agreement).
+      # overrides: the CLI's flags, which win over the stored config.
+      # -> [Judge, [model names]] | nil when nobody is configured to ask. NIL AND NOT
+      # a no-op judge: a rubric'd case with no judge reads as `judge_pending`, which is
+      # visible, where a judge that always passes would be silent.
+      # `llm` (RFC-0017 A2): the graph's own RubyLLM context; nil = the global
+      # constant. Only the deployment root and the CLI build judges today, and both
+      # are one graph per process — the seam keeps the default honest for the day
+      # an embedded graph gates a candidate on its own credentials.
+      def build(settings, overrides: {}, chat_factory: nil, llm: nil)
+        settings = Coercion.deep_stringify(settings || {})
+        overrides = overrides.transform_keys(&:to_s)
+        models = resolve_models(settings, overrides)
+        return nil if models.empty?
+
+        factory = chat_factory || ->(model, provider) { ruby_llm_ask(model, provider, llm: llm) }
+        judge = Judge.new(
+          asks: models.map { |m| factory.call(m["model"], m["provider"]) },
+          quorum: overrides["quorum"] || settings["quorum"] || 1,
+          aggregate: overrides["aggregate"] || settings["aggregate"] || "median",
+          min_agreement: overrides["min_agreement"] || settings["min_agreement"] || 0.5
+        )
+        [judge, models.map { |m| m["model"] }]
+      end
+
+      # Sugar for the callers that only want the judge (the gate).
+      def judge(settings, **kw) = build(settings, **kw)&.first
+
+      # The SAME configured models, asked a different question (RFC-0014 §3.4). One
+      # builder because "who judges here" is one operator decision: a pairwise panel
+      # configured apart from the rubric panel would let a run be graded by judges
+      # nobody chose. -> [Pairwise, [model names]] | nil when nobody is configured.
+      def pairwise(settings, overrides: {}, chat_factory: nil, llm: nil)
+        settings = Coercion.deep_stringify(settings || {})
+        models = resolve_models(settings, overrides.transform_keys(&:to_s))
+        return nil if models.empty?
+
+        factory = chat_factory || ->(model, provider) { ruby_llm_ask(model, provider, llm: llm) }
+        [Pairwise.new(asks: models.map { |m| factory.call(m["model"], m["provider"]) }),
+         models.map { |m| m["model"] }]
+      end
+
+      def resolve_models(settings, overrides)
+        models = if Coercion.present?(overrides["judge_model"])
+                   [{ "model" => overrides["judge_model"], "provider" => overrides["judge_provider"] }]
+                 else
+                   Array(settings["judges"])
+                 end
+        models.map { |m| Coercion.deep_stringify(m) }.reject { |m| m["model"].to_s.strip.empty? }
+      end
+
+      # The default way to reach a model: RubyLLM, temperature 0, required lazily so
+      # nothing here loads a provider gem until a judge is actually configured.
+      def ruby_llm_ask(model, provider, llm: nil)
+        require "ruby_llm"
+        llm ||= RubyLLM
+        lambda do |prompt|
+          llm.chat(model: model, provider: provider, assume_model_exists: true)
+             .with_temperature(0).ask(prompt).content
+        end
+      end
+    end
+  end
+end

data/lib/insika/evals/pairwise.rb

@@ -0,0 +1,178 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Insika
+  module Evals
+    # PAIRWISE AGAINST THE INCUMBENT (RFC-0014 §3.4) — the number that answers "can we
+    # replace it". An absolute 0.72 says a reply cleared a bar we invented; it says
+    # nothing about whether the system already answering 403,231 chats would have done
+    # better with the same customer.
+    #
+    # Same opening, two transcripts, one question: which one served the customer
+    # better? Three outcomes (better / comparable / worse), plus two the panel can
+    # produce and must not hide — `split` when the judges disagree and `unknown` when
+    # none of them answered in a readable way.
+    #
+    # Three honesty rules, each of them the difference between a number people should
+    # trust and one they should not:
+    #
+    # 1. **Anonymous.** The judge sees "A" and "B" and is never told which one is
+    #    Insika. Told, it would have an opinion about the new system rather than about
+    #    the conversations.
+    # 2. **Both orders, every judge.** Position bias is THE known failure of pairwise
+    #    LLM judging, and it is the same species as the kill criterion ("`better`
+    #    tracks length, or politeness"). So each judge is asked twice with the
+    #    transcripts swapped, and a verdict that FLIPS with presentation order is
+    #    reported as `comparable` with `order_dependent: true` — a preference that
+    #    depends on which one was printed first is not a preference.
+    # 3. **A split panel stays split.** Averaging "better" and "worse" into
+    #    "comparable" invents agreement that nobody expressed.
+    #
+    # Cost: 2 provider calls per judge per case (§5). This is why it never runs as
+    # part of the gate and is opt-in on the CLI.
+    class Pairwise
+      BETTER = "better"
+      COMPARABLE = "comparable"
+      WORSE = "worse"
+      SPLIT = "split"
+      UNKNOWN = "unknown"
+
+      # `vs` names WHO the reference half actually is: `agent` (model against model)
+      # or `human-assisted` (a person typed part of it, P23a's `origin: operator`).
+      # It rides on the verdict rather than beside it so a report cannot print the
+      # outcome without the label.
+      Verdict = Struct.new(:outcome, :reason, :vs, :judges, :order_dependent, keyword_init: true) do
+        def human_assisted? = vs == "human-assisted"
+        def decided? = [BETTER, COMPARABLE, WORSE].include?(outcome)
+      end
+
+      # asks: [->(prompt) { "<raw model text>" }] — the SAME panel the rubric judge
+      # uses (JudgePanel builds both), because "the judges the operator configured" is
+      # one decision, not two.
+      def initialize(asks:)
+        @asks = Array(asks).compact
+        raise ArgumentError, "a pairwise comparison needs at least one `ask`" if @asks.empty?
+      end
+
+      # golden + the run's [TurnResult] -> Verdict, or nil when the case carries no
+      # reference (most of them: a pair is curated by a human, like the case itself).
+      def compare(golden:, turns:)
+        return nil unless golden.reference?
+
+        ours = Pairwise.transcript(golden.user_turns, turns)
+        theirs = Pairwise.reference_transcript(golden.reference_messages)
+        return nil if ours.strip.empty?
+
+        panel = @asks.map { |ask| judge_once(ask, ours, theirs) }
+        combine(panel, vs: golden.human_assisted? ? "human-assisted" : "agent")
+      end
+
+      # The replayed conversation as the judge reads it: the user turns we sent,
+      # interleaved with the answers the deployment published. `turns` may be shorter
+      # than `user_turns` (an errored turn aborts the replay) — zip on what ran.
+      def self.transcript(user_turns, turns)
+        Array(turns).each_with_index.map do |t, i|
+          ["customer: #{user_turns[i]}", "assistant: #{t.output_text.to_s.strip}"]
+        end.flatten.join("\n")
+      end
+
+      # The incumbent's half. Human turns are NOT flagged to the judge: what it grades
+      # is the conversation as the customer received it, and telling it "a person wrote
+      # this one" is an invitation to grade the author instead. The fact is carried to
+      # the READER as `vs: human-assisted` instead, which is where it changes a decision.
+      def self.reference_transcript(messages)
+        Array(messages).map do |m|
+          speaker = m["role"].to_s == "user" ? "customer" : "assistant"
+          "#{speaker}: #{m['text'].to_s.strip}"
+        end.join("\n")
+      end
+
+      private
+
+      # One model, asked twice with the sides swapped. -> { outcome:, reason:,
+      # order_dependent: }
+      def judge_once(ask, ours, theirs)
+        first = outcome_of(parse(ask.call(prompt(ours, theirs))), ours_is: "A")
+        second = outcome_of(parse(ask.call(prompt(theirs, ours))), ours_is: "B")
+
+        return { outcome: UNKNOWN, reason: [first[:reason], second[:reason]].compact.first.to_s, order_dependent: false } \
+          if first[:outcome] == UNKNOWN && second[:outcome] == UNKNOWN
+
+        reason = [first[:reason], second[:reason]].reject { |r| r.to_s.empty? }.first.to_s
+        agreed = [first[:outcome], second[:outcome]].reject { |o| o == UNKNOWN }.uniq
+        return { outcome: agreed.first, reason: reason, order_dependent: false } if agreed.length == 1
+
+        { outcome: COMPARABLE, reason: reason, order_dependent: true }
+      end
+
+      # A strict majority decides; anything less is `split`. Unknown judges are left
+      # OUT of the tally (they expressed nothing) but stay visible in `judges`.
+      def combine(panel, vs:)
+        voted = panel.reject { |j| j[:outcome] == UNKNOWN }
+        winner, group = voted.group_by { |j| j[:outcome] }.max_by { |_, js| js.length }
+        outcome = if voted.empty? then UNKNOWN
+                  elsif group.length * 2 > voted.length then winner
+                  else SPLIT
+                  end
+        # The reason comes from the judges that CARRIED the verdict, so a split or an
+        # unknown is explained by whoever produced it rather than by an outvoted one.
+        spoke = outcome == winner ? group : panel
+
+        Verdict.new(outcome: outcome, vs: vs,
+                    reason: spoke.map { |j| j[:reason].to_s }.reject(&:empty?).first.to_s,
+                    judges: panel.map { |j| j[:outcome] },
+                    order_dependent: panel.any? { |j| j[:order_dependent] })
+      end
+
+      # "A won" means Insika won only when Insika WAS A. The mapping is the whole
+      # point of asking twice.
+      def outcome_of(parsed, ours_is:)
+        winner = parsed[:winner]
+        outcome = case winner
+                  when "tie" then COMPARABLE
+                  when nil then UNKNOWN
+                  else winner == ours_is ? BETTER : WORSE
+                  end
+        { outcome: outcome, reason: parsed[:reason] }
+      end
+
+      def prompt(a, b)
+        <<~PROMPT
+          You are comparing two customer-service conversations that began with the SAME
+          customer message. Decide which one SERVED THE CUSTOMER BETTER: did the customer
+          get what they came for, without detours, wrong information or invented facts?
+
+          Ignore length, tone, politeness, emoji and formatting UNLESS they changed what
+          the customer actually got. A short answer that solves the problem beats a long
+          one that does not.
+
+          CONVERSATION A:
+          #{a}
+
+          CONVERSATION B:
+          #{b}
+
+          Respond with ONLY a JSON object, no prose:
+          {"winner": "A" | "B" | "tie", "reason": "<one short sentence>"}
+        PROMPT
+      end
+
+      # An unreadable reply is UNKNOWN, never a preference: scoring it as a tie would
+      # quietly count a broken judge as evidence that the two systems are equivalent.
+      def parse(raw)
+        block = raw.to_s[/\{.*\}/m]
+        raise JSON::ParserError, "no JSON object" unless block
+
+        obj = JSON.parse(block)
+        winner = obj["winner"].to_s.strip.upcase
+        winner = "tie" if %w[TIE DRAW EQUAL COMPARABLE].include?(winner)
+        raise JSON::ParserError, "unknown winner" unless %w[A B tie].include?(winner)
+
+        { winner: winner, reason: obj["reason"].to_s }
+      rescue JSON::ParserError, TypeError
+        { winner: nil, reason: "unparseable pairwise output: #{raw.to_s[0, 120].inspect}" }
+      end
+    end
+  end
+end