insika 0.0.1 → 0.1.0

data/lib/insika/refinement/evidence_collector.rb

@@ -0,0 +1,371 @@
+# frozen_string_literal: true
+
+require "json"
+require "time"
+
+module Insika
+  module Refinement
+    # RFC-0013 phase A. Reads a window of an agent's real traffic and emits RANKED
+    # FINDINGS — "here is what broke, how often, and in which conversations". No
+    # model runs here and nothing is written to the agent: this is the evidence half
+    # of the loop, and it is deliberately useful on its own.
+    #
+    # It reads ONLY durable data the engine already records:
+    #   TaskStore       — the turn, its Command (which carries the agent) and the
+    #                     executions (a failed turn keeps its error)
+    #   SessionStore    — the transcript (repetition, canned safe replies)
+    #   ToolTraceStore  — per-session tool calls with ok/args/result (already masked
+    #                     and clipped by the store itself)
+    #
+    # Two signals of RFC-0013 §3.3 are NOT computed here, and that is a finding about
+    # the engine rather than about an agent: guardrail decisions and edge-limit hits
+    # are emitted as EVENTS and never persisted, so the only durable footprint they
+    # leave is the canned safe reply in the transcript — which is exactly what the
+    # `safe_reply` finding matches. Attributing one to a specific rule needs a write
+    # path that does not exist yet.
+    #
+    # The agent of a turn comes from the Task's Command payload, not from the
+    # Session: a Session does not stamp which agent produced it.
+    class EvidenceCollector
+      include Coercion
+
+      DEFAULT_WINDOW      = 200   # distinct sessions, most recent first
+      DEFAULT_MAX_FINDINGS = 20
+      MAX_PROVENANCE      = 5     # session ids kept per finding
+      SNIPPET_CHARS       = 160
+      REPETITION_JACCARD  = 0.6
+      REPETITION_MIN_WORDS = 3    # "oi"/"sim" repeated is not a defect
+
+      # LEGACY FALLBACK, for transcripts written before messages carried an origin.
+      #
+      # A message the engine wrote itself — an injected context fragment, a delegation
+      # result delivered as a new turn — is persisted with `role: user` like any
+      # other, because it is what the model saw. Counting those as the customer
+      # repeating themselves turned the first production run into 219 false positives,
+      # every one of them the engine reading its own `<cacau_cep_obrigatorio>` back.
+      #
+      # `MessageOrigin` is the structural answer and is preferred whenever a message
+      # carries it. This regex stays for everything written before that field existed
+      # (the pilot's database is full of it) and for consumers that have not started
+      # declaring `origin` — it is a guess, and it only ever runs on messages that
+      # made no claim about themselves.
+      INJECTED_FRAGMENT_RE = /\A<[a-z][a-z0-9_:.-]*>/i
+
+      # Weight of a finding kind when ranking (count × severity).
+      SEVERITY = {
+        tool_error: 3, task_failed: 3, safe_reply: 2, repetition: 2, tool_unused: 1
+      }.freeze
+
+      # One defect, aggregated over the window. `key` is the stable identity used to
+      # dedupe/group (and, later, for a proposal to say which finding it addresses);
+      # `sessions` is provenance — ids only, capped, never content.
+      Finding = Data.define(:kind, :key, :title, :count, :severity, :sessions, :detail)
+
+      # What a run looked at, alongside what it found. `excluded` is reported rather
+      # than swallowed: a window that quietly dropped half the traffic reads like a
+      # clean deployment.
+      Report = Data.define(:agent_id, :window, :findings, :sessions_seen, :turns_seen,
+                           :excluded)
+
+      def initialize(task_store:, session_store:, tool_trace_store:, profiles:,
+                     settings_store: nil)
+        @task_store = task_store
+        @session_store = session_store
+        @tool_trace_store = tool_trace_store
+        @profiles = ProfileSource.coerce(profiles)
+        @settings_store = settings_store
+      end
+
+      # -> Report. `since` (ISO8601) wins over `last_sessions` when both are given:
+      # an incremental run ("what happened since the last one") is the common case.
+      #
+      # `exclude_sessions` drops sessions whose id starts with any of the given
+      # prefixes. It defaults to NOTHING — a report must not decide on its own what
+      # counts as real traffic — but a deployment that replays load tests or debug
+      # conversations into the same store needs it: on the pilot, `loadtest-` sessions
+      # outnumbered real ones and drowned every genuine finding.
+      def collect(agent_id:, last_sessions: DEFAULT_WINDOW, since: nil,
+                  max_findings: DEFAULT_MAX_FINDINGS, exclude_sessions: [])
+        agent = agent_id.to_s
+        profile = @profiles[agent] ||
+                  (raise Insika::NotFoundError, "agent '#{agent}' not configured")
+
+        tasks, excluded = window_tasks(agent, last_sessions: last_sessions, since: since,
+                                              exclude_sessions: Array(exclude_sessions))
+        session_ids = tasks.filter_map { |t| presence(t.session_id) }.uniq
+        traces = session_ids.to_h { |sid| [sid, @tool_trace_store.for_session(sid)] }
+
+        findings = [
+          *tool_error_findings(traces),
+          *task_failed_findings(tasks),
+          *repetition_findings(session_ids),
+          *safe_reply_findings(session_ids, profile),
+          # An EMPTY window says nothing about a tool being unused — it says the agent
+          # did not run. Without this guard every incremental run over quiet traffic
+          # would report the whole tool list as "never called".
+          *(tasks.empty? ? [] : tool_unused_findings(profile, traces))
+        ]
+
+        Report.new(
+          agent_id: agent,
+          window: since ? { "since" => since.to_s } : { "last_sessions" => last_sessions },
+          findings: rank(findings).first(max_findings),
+          sessions_seen: session_ids.size, turns_seen: tasks.size, excluded: excluded
+        )
+      end
+
+      private
+
+      # The agent's turns, most recent first, and how many were excluded.
+      # -> [[Task], Integer]. O(n) over every task: the key is a UUID so `list` cannot
+      # order by time and there is no index by agent — same trade-off
+      # TaskStore#with_status already takes (one node, local SQLite).
+      def window_tasks(agent, last_sessions:, since:, exclude_sessions:)
+        # The id breaks the tie: `created_at` has second precision, so several turns
+        # can share it and `sort_by` alone would order them arbitrarily — two runs over
+        # the same data must produce the same window.
+        all = @task_store.each_id
+                         .filter_map { |id| @task_store.find(id) }
+                         .select { |t| agent_of(t) == agent }
+                         .sort_by { |t| [t.created_at.to_s, t.id] }.reverse
+
+        kept = exclude_sessions.empty? ? all : all.reject { |t| excluded?(t, exclude_sessions) }
+        excluded = all.size - kept.size
+
+        return [kept.select { |t| t.created_at.to_s >= since.to_s }, excluded] if presence(since)
+
+        [take_until_sessions(kept, last_sessions), excluded]
+      end
+
+      def excluded?(task, prefixes)
+        sid = task.session_id.to_s
+        prefixes.any? { |p| sid.start_with?(p.to_s) }
+      end
+
+      # Walks newest-first and stops once `limit` DISTINCT sessions have been seen —
+      # so the window is "the last N conversations", not "the last N turns".
+      def take_until_sessions(tasks, limit)
+        seen = {}
+        tasks.take_while do |task|
+          sid = presence(task.session_id)
+          seen[sid] = true if sid
+          seen.size <= limit
+        end
+      end
+
+      def agent_of(task) = task.command.is_a?(Hash) ? task.command.dig("payload", "agent").to_s : ""
+
+      # --- findings ---------------------------------------------------------------
+
+      # A tool call that came back an error. Grouped by tool + a normalized error
+      # signature, so 40 instances of the same broken argument are ONE finding with
+      # count 40 instead of 40 rows nobody reads.
+      def tool_error_findings(traces)
+        group(traces_entries(traces).reject { |_sid, e| e["ok"] }) do |_sid, entry|
+          tool = entry["tool"].to_s
+          [:tool_error, "tool_error:#{tool}:#{error_signature(entry['result'])}",
+           "#{tool} failed: #{error_signature(entry['result'])}", nil]
+        end
+      end
+
+      # A turn that died. The error is whatever the Executor recorded on the
+      # execution, grouped by its normalized message.
+      def task_failed_findings(tasks)
+        failed = tasks.flat_map do |task|
+          task.executions.filter_map do |ex|
+            next if ex.error.nil?
+
+            [presence(task.session_id), ex.error]
+          end
+        end
+        group(failed) do |_sid, error|
+          sig = signature(error_text(error))
+          [:task_failed, "task_failed:#{sig}", "turn failed: #{sig}", nil]
+        end
+      end
+
+      # The customer said the same thing AGAIN AFTER THE AGENT ANSWERED — the outside view
+      # of an instruction the agent is not following. Heuristic on purpose (token overlap,
+      # no model call); the snippet is PII-redacted.
+      #
+      # "After the agent answered" is the load-bearing half, and RFC-0015 is what forced
+      # it to be said out loud. Two customer messages in a row is now ORDINARY: `collect`
+      # merges the fragments a person types into one turn and `steer` appends one into a
+      # run in flight, so a turn legitimately holds two of them. Someone still typing is
+      # not someone repeating themselves — and a steered message cannot be told apart in
+      # the transcript, because it correctly declares no origin (§7). The structure is the
+      # only honest signal: a reply has to sit between the two.
+      def repetition_findings(session_ids)
+        hits = session_ids.flat_map do |sid|
+          repeated_after_a_reply(messages(sid)).map { |text| [sid, text] }
+        end
+        group(hits) do |_sid, text|
+          [:repetition, "repetition", "customer repeated themselves", snippet(text)]
+        end
+      end
+
+      # A canned safe reply reached the customer: a guardrail block or an edge limit.
+      # Those decisions are events, not records (see the class comment), so the reply
+      # text IS the evidence.
+      #
+      # This is the one finding that reads the engine's OWN replies, so it looks at
+      # every assistant-role message rather than the agent's (`assistant_messages`
+      # deliberately excludes them). Two ways to recognise one, and the first is now
+      # exact: a reply the engine wrote SAYS so (`origin: engine`). The canned-text
+      # match stays for transcripts written before that — and for a text match the
+      # strings are emitted verbatim, so it is an exact compare, never a prefix.
+      def safe_reply_findings(session_ids, profile)
+        canned = canned_replies(profile)
+
+        hits = session_ids.flat_map do |sid|
+          messages(sid)
+            .select { |m| m["role"].to_s == "assistant" }
+            .select { |m| MessageOrigin.origin_of(m) == MessageOrigin::ENGINE || canned.include?(m["content"].to_s.strip) }
+            .map { |m| [sid, m["content"].to_s] }
+        end
+        group(hits) do |_sid, text|
+          [:safe_reply, "safe_reply", "a canned safe reply was served instead of an answer",
+           snippet(text)]
+        end
+      end
+
+      # A tool the agent is allowed to use and never used in the whole window —
+      # either the prompt does not mention it or it answers from memory instead.
+      # Skipped when `tools_allow` is nil (= every tool; nothing to compare against).
+      def tool_unused_findings(profile, traces)
+        allowed = profile.tools_allow
+        return [] if allowed.nil?
+
+        used = traces_entries(traces).map { |_sid, e| e["tool"].to_s }.uniq
+        (Array(allowed).map(&:to_s) - used).sort.map do |tool|
+          Finding.new(kind: :tool_unused, key: "tool_unused:#{tool}",
+                      title: "#{tool} was never called in this window",
+                      count: 1, severity: SEVERITY[:tool_unused], sessions: [], detail: nil)
+        end
+      end
+
+      # --- helpers ----------------------------------------------------------------
+
+      def traces_entries(traces)
+        traces.flat_map { |sid, entries| Array(entries).map { |e| [sid, e] } }
+      end
+
+      # [[session_id, item], …] -> [Finding] aggregated by the key the block returns.
+      # The block gets (session_id, item) and returns [kind, key, title, detail].
+      def group(pairs)
+        pairs.each_with_object({}) do |(sid, item), acc|
+          kind, key, title, detail = yield(sid, item)
+          f = acc[key] ||= { kind: kind, key: key, title: title, detail: detail, sessions: [], count: 0 }
+          f[:count] += 1
+          f[:sessions] << sid if sid && !f[:sessions].include?(sid)
+        end.values.map do |f|
+          Finding.new(kind: f[:kind], key: f[:key], title: f[:title], count: f[:count],
+                      severity: SEVERITY.fetch(f[:kind], 1),
+                      sessions: f[:sessions].first(MAX_PROVENANCE), detail: f[:detail])
+        end
+      end
+
+      # count × severity, then a stable tiebreak so two runs over the same window
+      # produce the same report.
+      def rank(findings)
+        findings.sort_by { |f| [-(f.count * f.severity), f.kind.to_s, f.key] }
+      end
+
+      def messages(session_id) = Array(@session_store.find(session_id)&.messages)
+
+      # What the AGENT actually replied — not a guardrail's canned safe reply, and not
+      # a human operator's typing in an imported transcript. Both are `role: assistant`
+      # and neither is the model, so scoring them as the agent's work is wrong in both
+      # directions: it flags text no model produced, and it credits the model with a
+      # human's rescue.
+      def assistant_messages(session_id)
+        messages(session_id).select { |m| MessageOrigin.agent?(m) }.map { |m| m["content"].to_s }
+      end
+
+      # Walks a session in order and returns the customer texts that repeat something the
+      # customer had ALREADY said and the agent had already answered. Two consecutive
+      # customer messages with nothing between them are one person typing (see
+      # #repetition_findings), so they never pair.
+      def repeated_after_a_reply(list)
+        previous = nil
+        answered = false
+
+        list.each_with_object([]) do |message, hits|
+          next answered = true if MessageOrigin.agent?(message)
+          next unless customer_text?(message)
+
+          text = message["content"].to_s
+          hits << text if previous && answered && similar?(previous, text)
+          previous = text
+          answered = false
+        end
+      end
+
+      # What the CUSTOMER actually said. A message that DECLARES its origin is taken at its
+      # word; one that declares nothing falls back to the tag heuristic, which is all a
+      # pre-origin transcript offers.
+      def customer_text?(message)
+        MessageOrigin.customer?(message) &&
+          !INJECTED_FRAGMENT_RE.match?(message["content"].to_s.lstrip)
+      end
+
+      def similar?(first, second)
+        a = words(first)
+        b = words(second)
+        return false if a.size < REPETITION_MIN_WORDS || b.size < REPETITION_MIN_WORDS
+
+        union = (a | b).size
+        union.positive? && ((a & b).size.to_f / union) >= REPETITION_JACCARD
+      end
+
+      def words(text) = text.to_s.downcase.scan(/[[:alnum:]]+/).uniq
+
+      # Every reply the deployment may emit INSTEAD of a real answer: the RFC-0009
+      # defaults, the agent's own overrides, and the edge limiter's reply (per-agent
+      # first, then the platform setting).
+      def canned_replies(profile)
+        agent_overrides = (profile.guardrails || {})["responses"]
+        [
+          *Insika::Safety::SafeResponses::DEFAULTS.values,
+          *Array(agent_overrides.is_a?(Hash) ? agent_overrides.values : nil),
+          (profile.limits || {})[:limit_response],
+          @settings_store&.get&.dig("edge", "limit_response")
+        ].filter_map { |t| presence(t)&.strip }.uniq
+      end
+
+      # ToolTraceStore clips `result` to a String (JSON text when it was structured),
+      # so the error has to be dug back out of it.
+      def error_signature(result)
+        parsed = begin
+          JSON.parse(result.to_s)
+        rescue StandardError
+          nil
+        end
+        signature(parsed.is_a?(Hash) ? error_text(parsed) : result)
+      end
+
+      def error_text(error)
+        return error["error"] || error["message"] || error.to_s if error.is_a?(Hash)
+
+        error.to_s
+      end
+
+      # Groups variants of the same failure: collapse whitespace, blank out numbers
+      # and ids (a "product 4711 not found" is the same defect as "product 4712"),
+      # then cap the length.
+      def signature(text)
+        s = text.to_s.gsub(/\s+/, " ").strip
+        s = s.gsub(/\b[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}\b/i, "<id>").gsub(/\d+/, "<n>")
+        s = "(no message)" if s.empty?
+        s.length > 80 ? "#{s[0, 80]}…" : s
+      end
+
+      # Redacted, capped excerpt — a report is read by an operator, so it may carry
+      # customer words, but never a CPF, a phone number or a token.
+      def snippet(text)
+        redacted, = Insika::Safety::Detectors.redact(text.to_s.gsub(/\s+/, " ").strip)
+        redacted.length > SNIPPET_CHARS ? "#{redacted[0, SNIPPET_CHARS]}…" : redacted
+      end
+    end
+  end
+end

data/lib/insika/refinement/gate.rb

@@ -0,0 +1,234 @@
+# frozen_string_literal: true
+
+module Insika
+  module Refinement
+    # Scores a candidate by RUNNING it (RFC-0013 §3.5). Not by asking a model whether
+    # the edit looks good — that measures nothing, and D3 says so in one line.
+    #
+    #   1. clone the agent into a throwaway id (`<agent>-cand-<run8>`)
+    #   2. copy its instruction files, apply the candidate's edits to the COPY
+    #   3. replay the agent's golden set against the clone over the ordinary public
+    #      surface (`POST /v1/responses`) — real turns, real tools, real guardrails
+    #   4. compare to the accepted baseline; ANY regression disqualifies
+    #   5. delete the clone, keep the report
+    #
+    # Step 3 is what makes this expensive and what makes it worth anything. The gate
+    # is the entire safety story of refinement: everything upstream can be wrong —
+    # a hallucinated rationale, a model that misread the evidence — and the worst
+    # outcome is still a candidate that fails to improve a score and never lands.
+    #
+    # The clone is deleted in an `ensure`, including when the replay raises. A
+    # leftover `-cand-` agent is servable at `/v1/responses` by anyone who knows the
+    # id, so leaking one is a real (if obscure) exposure, not just clutter.
+    class Gate
+      # The verdict for one candidate. `passed` is the only field the caller acts on;
+      # the rest is what an operator reads to decide whether the loop earns its keep.
+      # `tokens` is what the replay SENT, cache included, as the deployment reported
+      # it — nil when no turn carried usage. `cached` is how much of that came from
+      # the prompt cache, kept separate because it is what explains one candidate
+      # costing 8× another over the same cases. `tokens` is what the panel's budget
+      # (§3.9) spends and what the operator reads on the run: a gate is the expensive
+      # half of refinement and a loop whose cost is invisible is one nobody can decide
+      # to keep.
+      Report = Data.define(:candidate_id, :passed, :reason, :cases, :passed_cases,
+                           :baseline_cases, :regressions, :report, :tokens, :cached) do
+        def to_h
+          { "candidate_id" => candidate_id, "passed" => passed, "reason" => reason,
+            "cases" => cases, "passed_cases" => passed_cases, "baseline_cases" => baseline_cases,
+            "regressions" => regressions, "report" => report, "tokens" => tokens,
+            "cached" => cached }
+        end
+      end
+
+      DEFAULT_TOLERANCE = 0.05
+
+      # transport_factory: -> an Evals transport for the replay. A LAMBDA and not a
+      # transport, because the gate is constructed at boot and the deployment's own
+      # URL/token are what it has to talk to; passing the built object would freeze a
+      # credential the operator can rotate.
+      # capabilities_factory: -> an `Evals::HttpCapabilities` for the clone, or nil.
+      # Without it a case whose `requires` the agent cannot satisfy RUNS and fails
+      # (RFC-0014 §3.2 says it must skip) — and then the gate and `evals/run.rb`, the
+      # two callers of the one evaluator, disagree about what the corpus even
+      # measures. §3.7 exists to prevent exactly that.
+      def initialize(profiles:, agent_files:, goldens:, baselines:, transport_factory:,
+                     capabilities_factory: nil, judge_factory: nil, tolerance: DEFAULT_TOLERANCE)
+        @profiles = profiles
+        @agent_files = agent_files
+        @goldens = goldens
+        @baselines = baselines
+        @transport_factory = transport_factory
+        @capabilities_factory = capabilities_factory
+        @judge_factory = judge_factory
+        @tolerance = tolerance
+      end
+
+      # -> Report. Never raises for an ordinary refusal (no cases, no baseline, a
+      # replay that blew up): those are verdicts, and a run that recorded WHY it could
+      # not gate is more useful than an exception in a log.
+      def score(agent_id:, candidate:, run_id:, tolerance: nil)
+        cases = @goldens.for_agent(agent_id)
+        return refusal(candidate, "the agent has no golden cases — nothing to gate against (RFC-0013 D4)") if cases.empty?
+
+        baseline = @baselines.get(agent_id)
+        # Without an accepted state, `Baseline.compare` compares nothing and reports
+        # zero regressions — a green light meaning "we did not look". Refusing is the
+        # only honest reading, and the fix is one command.
+        if baseline.nil?
+          return refusal(candidate, "no recorded baseline for '#{agent_id}' — " \
+                                    "run `insika evals:baseline import` or record one before gating")
+        end
+
+        # And an ALL-RED baseline is the same hole with a record in front of it.
+        # `compare` only reports a regression against a case the baseline had
+        # PASSING, so a baseline where nothing passes cannot produce one: every
+        # candidate sails through, including a harmful one.
+        #
+        # Found by running this against a real agent: a replay that 401'd recorded a
+        # baseline of two failures, and from then on the gate accepted everything —
+        # including an edit written to be harmful. "Known-failing cases do not wedge
+        # the gate" is the right rule for the pre-merge check (a red case is work in
+        # progress, not a blocker); here it degrades into "nothing can ever fail",
+        # and a gate that cannot fail is not a gate.
+        if passing_cases(baseline).zero?
+          return refusal(candidate, "the recorded baseline for '#{agent_id}' has no PASSING case " \
+                                    "(#{baseline_size(baseline)} recorded, all failing) — nothing could " \
+                                    "regress, so every candidate would pass. Fix the agent or the cases, " \
+                                    "then re-record the baseline from a green run")
+        end
+
+        # And a baseline JUDGED by a rubric, replayed with no judge, is the third
+        # shape of the same hole — the one this gate actually shipped with.
+        #
+        # `CaseResult#pass?` reads a missing judge verdict as a pass (a rubric'd case
+        # is `judge_pending?`, which nothing consults), so a replay with no judge
+        # scores every rubric case as passing. Compared against a baseline recorded
+        # WITH a judge, that is not a weaker measurement, it is an inverted one:
+        # every candidate reads as an improvement.
+        #
+        # Measured, not reasoned: gating the real pilot agent with `settings["evals"]`
+        # unset reported **6/6, no regression** against a baseline the same corpus had
+        # just scored **2/6** — `produto-sem-cep` was judged 0.0 and "passed". Both
+        # candidates on the panel cleared. That is §3.7's failure exactly: the CLI and
+        # the gate, the two callers of the one evaluator, disagreeing about what the
+        # corpus measures.
+        judge = @judge_factory&.call
+        if judge.nil? && judged?(baseline)
+          return refusal(candidate, "the recorded baseline for '#{agent_id}' carries judge scores but no " \
+                                    "judge is configured — a rubric'd case with no verdict counts as a " \
+                                    "PASS, so every candidate would beat it. Configure the judge panel " \
+                                    "(Studio → Settings → Evals, or `settings[\"evals\"][\"judges\"]`) or " \
+                                    "re-record the baseline without one")
+        end
+
+        clone_id = clone_id_for(agent_id, run_id)
+        begin
+          build_clone(agent_id, clone_id, candidate)
+          ran = replay(cases, clone_id, judge)
+          verdict(candidate, ran, baseline, tolerance || @tolerance)
+        rescue StandardError => e
+          refusal(candidate, "gate failed to run: #{e.class}: #{e.message}")
+        ensure
+          destroy_clone(clone_id)
+        end
+      end
+
+      # `<agent>-cand-<run8>`: recognizable at a glance in the Studio's agent list and
+      # in a provider bill, and scoped to the run so two gates cannot collide.
+      def clone_id_for(agent_id, run_id) = "#{agent_id}-cand-#{run_id.to_s.delete('-')[0, 8]}"
+
+      # How many accepted cases could actually regress. This is the gate's real
+      # strength, and it is worth being able to say out loud.
+      def passing_cases(baseline)
+        (baseline["cases"] || {}).count { |_id, entry| entry.is_a?(Hash) && entry["pass"] }
+      end
+
+      # Was this baseline recorded with a judge? A single scored case is enough: it
+      # proves the accepted state was measured by a rubric the replay has to match.
+      # A baseline with no scores at all was recorded blind too, so both sides are
+      # equally deterministic and the comparison, while weak, is not inverted.
+      def judged?(baseline)
+        (baseline["cases"] || {}).any? { |_id, entry| entry.is_a?(Hash) && !entry["score"].nil? }
+      end
+
+      private
+
+      def baseline_size(baseline) = (baseline["cases"] || {}).size
+
+      # Same profile, same tools, same guardrails — only the id and the instruction
+      # files differ. Copying the profile rather than editing the real one is what
+      # makes this safe to run against production: the live agent is never touched,
+      # not even for a moment.
+      def build_clone(agent_id, clone_id, candidate)
+        profile = @profiles.fetch(agent_id) ||
+                  (raise Insika::NotFoundError, "agent '#{agent_id}' not configured")
+        @profiles.put(profile.with(id: clone_id))
+
+        contents = current_files(agent_id)
+        edited = candidate.apply(contents)
+        contents.merge(edited).each { |name, body| @agent_files.write(clone_id, name, body) }
+      end
+
+      # Every file the agent has, not only the ones the candidate touches: the clone
+      # has to be the same agent for the replay to mean anything.
+      def current_files(agent_id)
+        @agent_files.list(agent_id).each_with_object({}) do |name, acc|
+          acc[name] = @agent_files.read(agent_id, name).to_s
+        end
+      end
+
+      # The goldens name the REAL agent; the replay has to address the clone. The
+      # case is otherwise untouched — same turns, same rubric, same assertions — so
+      # what is measured is the edit and nothing else.
+      #
+      # The RunCases are kept whole (not `.map(&:result)`) because the token counts
+      # ride on them, and the budget is only honest if it sees what the replay spent.
+      def replay(cases, clone_id, judge)
+        retargeted = cases.map { |g| g.class.new(**g.to_h.merge(agent: clone_id)) }
+        runner = Insika::Evals::Runner.new(transport: @transport_factory.call, judge: judge,
+                                           capabilities: @capabilities_factory&.call)
+        runner.run(retargeted)
+      end
+
+      def verdict(candidate, ran, baseline, tolerance)
+        results = ran.map(&:result)
+        regressions = Insika::Evals::Baseline.compare(results, baseline, tolerance: tolerance)
+        passed = results.count(&:pass?)
+        graded = results.reject(&:skipped?).size
+        spent = ran.filter_map(&:tokens)
+        cached = ran.filter_map(&:cached)
+
+        Report.new(
+          candidate_id: candidate.id, passed: regressions.empty?,
+          reason: regressions.empty? ? nil : regression_reason(regressions),
+          cases: graded, passed_cases: passed,
+          baseline_cases: (baseline["cases"] || {}).size,
+          regressions: regressions.map { |r| { "id" => r.id, "kind" => r.kind, "detail" => r.detail } },
+          report: Insika::Evals::Report.to_h(results, at: Time.now.utc.iso8601),
+          tokens: spent.empty? ? nil : spent.sum,
+          cached: cached.empty? ? nil : cached.sum
+        )
+      end
+
+      def regression_reason(regressions)
+        "#{regressions.size} regression(s): " +
+          regressions.first(3).map { |r| "#{r.id} (#{r.kind})" }.join(", ")
+      end
+
+      def refusal(candidate, reason)
+        Report.new(candidate_id: candidate.id, passed: false, reason: reason,
+                   cases: 0, passed_cases: 0, baseline_cases: 0, regressions: [], report: nil,
+                   tokens: nil, cached: nil)
+      end
+
+      # Both halves, both tolerant of a missing one: this runs in an `ensure` after a
+      # failure that may have happened before either was created.
+      def destroy_clone(clone_id)
+        @agent_files.list(clone_id).each { |name| @agent_files.delete(clone_id, name) }
+        @profiles.delete(clone_id)
+      rescue StandardError => e
+        warn "[refinement] could not delete the gate clone '#{clone_id}': #{e.class}: #{e.message}"
+      end
+    end
+  end
+end