insika 0.0.1 → 0.2.0

data/lib/insika/refinement/panel.rb

@@ -0,0 +1,222 @@
+# frozen_string_literal: true
+
+module Insika
+  module Refinement
+    # What a refinement run is allowed to SPEND.
+    #
+    # A panel of 3 proposers over a 7-case golden set is 3 model calls plus 21
+    # replayed conversations, each a real turn with real tools. That is the honest
+    # objection to the whole feature, and this is the answer to it: a ceiling the
+    # operator sets, checked before each expensive step and never in the middle of
+    # one. Nothing is aborted mid-flight — a half-scored candidate is worse than an
+    # unscored one, because it looks like a verdict.
+    #
+    # **Unmetered legs are counted, not guessed.** A provider that reports no token
+    # usage makes a leg invisible to the ceiling; recording it as 0 would let a budget
+    # sit at "0 spent" forever while real money went out, so those legs are tallied
+    # separately and shown to the operator. The structural bounds — the fan-out cap on
+    # the panel, `max_edits`, the gate's own refusals — are what bound a run whose
+    # provider says nothing.
+    class Budget
+      def initialize(tokens: nil)
+        limit = tokens.to_i
+        @limit = limit.positive? ? limit : nil
+        @spent = 0
+        @cached = 0
+        @unmetered = 0
+      end
+
+      def limited? = !@limit.nil?
+
+      # `tokens` is what the leg SENT, prompt cache included — see
+      # `Evals::Runner#billed_tokens` for why the engine's `total_tokens` alone is not
+      # that number. `cached` is the part of it that was cached, carried so the record
+      # can explain a total rather than just state one. nil/0 tokens = the leg
+      # happened and reported nothing.
+      def spend(tokens, cached: nil)
+        count = tokens.to_i
+        if count.positive?
+          @spent += count
+          @cached += cached.to_i
+        else
+          @unmetered += 1
+        end
+        self
+      end
+
+      def exhausted? = limited? && @spent >= @limit
+
+      def remaining = limited? ? [@limit - @spent, 0].max : nil
+
+      def to_h = { "tokens" => @limit, "spent" => @spent, "cached" => @cached,
+                   "unmetered" => @unmetered }
+    end
+
+    # The proposer PANEL: N models write N independent
+    # candidates, the gate scores each one by replaying the golden set, and the best
+    # SURVIVOR becomes the proposal a human is asked about.
+    #
+    # Independent, not consensus-seeking. Two models agreeing on wording is weak
+    # evidence and a golden case passing is strong evidence, so convergence only
+    # ever breaks a tie between candidates the gate already ranked equal.
+    #
+    # A panel of one is unchanged, which is why there is no second code path:
+    # `refinement.proposer` (a single ref) resolves to a one-element panel.
+    class Panel
+      # One member of the panel: the candidate, WHO wrote it (more than one model when
+      # they converged on the identical edit set), and how it scored.
+      Entry = Data.define(:candidate, :proposers, :report) do
+        def converged = proposers.size
+        def passed? = report&.passed == true
+
+        def to_h = { "candidate" => candidate.to_h, "proposers" => proposers,
+                     "gate" => report&.to_h }
+      end
+
+      Result = Data.define(:entries, :winner, :budget, :failed) do
+        def winner_report = winner&.report
+      end
+
+      # gate:      a Refinement::Gate (anything answering #score).
+      # proposers: [Refinement::Proposer], already resolved by ProposerFactory.panel.
+      # budget:    a Budget. The default is unlimited — a deployment that configured
+      #            none gets behaviour, which had no ceiling either.
+      def initialize(gate:, proposers: [], budget: Budget.new, fan_out: nil)
+        @gate = gate
+        @proposers = Array(proposers)
+        @budget = budget
+        @fan_out = fan_out || Insika::SubagentGraph.fan_out_cap
+      end
+
+      # Proposes (unless `raw` is given), builds, gates and ranks. Yields the built
+      # entries BEFORE any of them is scored, so the caller can record the panel and
+      # move the run to :gating — the gate is the slow part and a run that says
+      # nothing until it finishes looks hung.
+      #
+      # -> Result. Raises ValidationError when there is nothing gateable at all,
+      # because that is an operator-facing refusal ("every edit was dropped, here is
+      # why"), not a verdict about the agent.
+      def run(agent_id:, run_id:, findings:, files:, allowlist:, contents:, limits: {},
+              raw: nil, tolerance: nil)
+        proposals, failed = raw ? [[raw], []] : propose(agent_id, findings, files, limits)
+        # A candidate that ARRIVED (Studio form, API client) cost this run nothing —
+        # counting it as an unmetered leg would make the cost record read as if a model
+        # had been asked and stayed quiet.
+        entries = build(proposals, allowlist, contents, limits, metered: raw.nil?)
+        yield entries if block_given?
+
+        scored = entries.map { |entry| score(entry, agent_id, run_id, tolerance) }
+        Result.new(entries: scored, winner: self.class.rank(scored), budget: @budget, failed: failed)
+      end
+
+      # -> the best SURVIVOR, or nil when none passed.
+      #
+      # Highest graded score first; ties broken by the fewest edits (a smaller diff is
+      # a smaller bet), then by how many proposers converged on it. `min_by`
+      # over a negated tuple keeps the comparison in one place and stays stable, so
+      # two genuinely indistinguishable candidates resolve to the first proposer the
+      # operator listed rather than to whichever fiber finished first.
+      def self.rank(entries)
+        entries.select(&:passed?)
+               .min_by { |e| [-e.report.passed_cases, e.candidate.edits.size, -e.converged] }
+      end
+
+      # The refusal to RECORD when nothing survived: the entry that got furthest,
+      # because "1 regression on `quotes`" tells an operator something and "budget
+      # exhausted" tells them only that the run stopped.
+      def self.best_refusal(entries)
+        entries.find { |e| e.report&.cases.to_i.positive? } ||
+          entries.find { |e| e.report } || entries.first
+      end
+
+      private
+
+      # All proposers at once, bounded by the fan-out cap (says the
+      # panel reuses it). Each is one blocking HTTP call to a provider, so the
+      # wall-clock is the slowest model rather than their sum.
+      #
+      # A proposer that answers prose, times out or 500s takes ITSELF out of the panel
+      # and nothing else: the whole point of asking several models is that one of them
+      # being useless is a survivable event. All of them failing is not, and raises.
+      def propose(agent_id, findings, files, limits)
+        raise Insika::ValidationError, "no proposer is configured" if @proposers.empty?
+
+        require_relative "../tools/concurrency"
+        blocks = @proposers.map do |proposer|
+          lambda do
+            proposer.propose(agent_id: agent_id, findings: findings, files: files, limits: limits)
+          rescue StandardError => e
+            { "failed" => "#{proposer.model}: #{e.class}: #{e.message}" }
+          end
+        end
+
+        outcomes = Insika::Tools::Concurrency.gather(blocks, max: @fan_out)
+        proposals, failures = outcomes.partition { |o| o["failed"].nil? }
+        failed = failures.map { |o| o["failed"] }
+        if proposals.empty?
+          raise Insika::ValidationError,
+                "every proposer failed — #{failed.join('; ')}"
+        end
+
+        [proposals, failed]
+      end
+
+      # Raw candidates -> Entries, DEDUPED by their edit set. Two models that wrote
+      # the identical edit are one candidate with two proposers: gating it twice would
+      # spend a whole golden replay to learn the same number, and the fact that they
+      # agreed is worth more as a tie-break than as a second row.
+      def build(proposals, allowlist, contents, limits, metered: true)
+        entries = {}
+        dropped = []
+
+        proposals.each do |proposal|
+          candidate = CandidateBuilder.build(proposal, allowlist: allowlist,
+                                                       contents: contents, limits: limits)
+          if metered
+            raw = proposal.is_a?(Hash) ? proposal : {}
+            @budget.spend(raw["tokens"], cached: raw["cached"])
+          end
+          if candidate.empty?
+            dropped.concat(candidate.dropped)
+            next
+          end
+
+          signature = candidate.edits.map(&:to_h)
+          if (seen = entries[signature])
+            entries[signature] = seen.with(proposers: seen.proposers + [candidate.proposer])
+          else
+            entries[signature] = Entry.new(candidate: candidate, proposers: [candidate.proposer],
+                                           report: nil)
+          end
+        end
+
+        return entries.values unless entries.empty?
+
+        # Every edit of every proposal fell off. Say which and why: an operator who
+        # gets "invalid candidate" back learns nothing, and a stale `before` is the
+        # common one.
+        raise Insika::ValidationError,
+              "every edit was dropped — #{dropped.map { |d| "#{d.file}: #{d.reason}" }.join('; ')}"
+      end
+
+      # One gate run, unless the budget is already spent. An unscored candidate is
+      # recorded with a refusal that names the ceiling — never dropped in silence,
+      # which would read as "the panel only produced one idea".
+      def score(entry, agent_id, run_id, tolerance)
+        if @budget.exhausted?
+          return entry.with(report: Gate::Report.new(
+            candidate_id: entry.candidate.id, passed: false,
+            reason: "not gated — the run's token budget (#{@budget.to_h['tokens']}) was spent",
+            cases: 0, passed_cases: 0, baseline_cases: 0, regressions: [], report: nil,
+            tokens: nil, cached: nil
+          ))
+        end
+
+        report = @gate.score(agent_id: agent_id, candidate: entry.candidate,
+                             run_id: run_id, tolerance: tolerance)
+        @budget.spend(report.tokens, cached: report.cached)
+        entry.with(report: report)
+      end
+    end
+  end
+end

data/lib/insika/refinement/proposer.rb

@@ -0,0 +1,262 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Insika
+  module Refinement
+    # Writes a CANDIDATE from a run's findings — the
+    # one place in refinement where a model is asked for anything.
+    #
+    # It is deliberately the WEAKEST link and it is built that way: everything this
+    # class produces is data that the CandidateBuilder bounds (allowlist, size,
+    # growth, a `before` that must still match) and the Gate then scores by REPLAYING
+    # the golden set. A hallucinated rationale, a misread finding, an invented anchor
+    # — the worst outcome of each is a candidate that gets dropped or fails to improve
+    # a score, and never reaches a customer. Nothing here is trusted; it is measured.
+    #
+    # Pure over an injected `ask` (prompt -> raw model text), like `Evals::Judge` and
+    # `Safety::Factory`: unit-testable without a provider, and the real ask is one
+    # lambda built by `ProposerFactory`.
+    #
+    # ## What it is shown
+    #
+    # The findings (already redacted at collection — a snippet went through the same
+    # output filter a customer-facing turn does) and the CURRENT CONTENT of the
+    # allowlisted files, verbatim and unmasked. That is not a leak: those files are
+    # the agent's own instructions, which are sent to a model on every single turn.
+    # Masking them here would buy nothing and would break anchoring — a `before`
+    # copied from a masked view never matches the real file, so every edit near a
+    # secret would drop as stale.
+    #
+    # Files OUTSIDE the allowlist are not shown at all. A model that can read them
+    # proposes edits to them, which drop, which spends the operator's attention on
+    # rejects.
+    class Proposer
+      # A model that answers with prose instead of JSON produces NOTHING, loudly.
+      # Silently returning an empty candidate would read as "the traffic is fine".
+      class Unusable < Insika::ValidationError; end
+
+      MAX_FINDINGS = 10
+
+      # The model ref, so a panel can name WHICH proposer failed without guessing.
+      attr_reader :model
+
+      # ask:   ->(prompt) { "<raw model text>" }, or something answering `#content`
+      #        plus `#input_tokens`/`#output_tokens` (a RubyLLM message). The second
+      #        shape is what lets the panel's budget count what a proposal cost; a
+      #        plain String stays valid and simply reports no cost, which is what
+      #        every existing caller and every fake does.
+      # model: what to record as the candidate's `proposer` — the ref an operator
+      #        reads on the review card and in `:refinement_proposed`.
+      def initialize(ask:, model: "unknown")
+        @ask = ask
+        @model = model.to_s
+      end
+
+      # -> a RAW candidate hash (string keys) for `CandidateBuilder.build`. It is not
+      # a Candidate: this class does not get to decide what is in bounds.
+      #
+      # findings: the run's findings as stored (string keys).
+      # files:    name => current content, allowlist only.
+      def propose(agent_id:, findings:, files:, limits: {})
+        raise Unusable, "there is nothing to propose from — the run found no findings" if Array(findings).empty?
+        raise Unusable, "no writable file has any content to anchor an edit in" if files.empty?
+
+        answer = @ask.call(build_prompt(agent_id, Array(findings).first(MAX_FINDINGS), files, limits))
+        parsed = parse(text_of(answer))
+        parsed["proposer"] = @model
+        parsed["tokens"] = tokens_of(answer)
+        parsed["cached"] = cached_of(answer)
+        parsed
+      end
+
+      private
+
+      def text_of(answer) = (answer.respond_to?(:content) ? answer.content : answer).to_s
+
+      # nil when the provider said nothing — never 0. `Budget` distinguishes the two
+      # and an operator reading "0 tokens" for a real model call would be reading a
+      # lie the record cannot correct. The cached prefix is INCLUDED, for the same
+      # reason `Evals::Runner#billed_tokens` includes it: a ceiling that cannot see
+      # what the cache served is not a ceiling on what was sent.
+      def tokens_of(answer)
+        return nil unless answer.respond_to?(:input_tokens) && answer.respond_to?(:output_tokens)
+
+        total = answer.input_tokens.to_i + answer.output_tokens.to_i + cached_of(answer).to_i
+        total.positive? ? total : nil
+      end
+
+      def cached_of(answer)
+        return nil unless answer.respond_to?(:cached_tokens)
+
+        cached = answer.cached_tokens.to_i
+        cached.positive? ? cached : nil
+      end
+
+      # JSON or nothing. ````json` fences are the common wrapper and stripping them is
+      # not leniency — the payload inside is still parsed strictly, so a model that
+      # improvises a schema fails here instead of producing half a candidate.
+      def parse(raw)
+        body = raw.strip.gsub(/\A```(?:json)?\s*|\s*```\z/, "")
+        first = body.index("{")
+        last = body.rindex("}")
+        raise Unusable, "the proposer answered with no JSON object" if first.nil? || last.nil? || last < first
+
+        parsed = JSON.parse(body[first..last])
+        raise Unusable, "the proposer's JSON is not an object" unless parsed.is_a?(Hash)
+        raise Unusable, "the proposer's JSON carries no `edits`" unless parsed["edits"].is_a?(Array)
+
+        parsed
+      rescue JSON::ParserError => e
+        raise Unusable, "the proposer's answer is not valid JSON: #{e.message}"
+      end
+
+      def build_prompt(agent_id, findings, files, limits)
+        bounds = DEFAULT_LIMITS.merge(Coercion.deep_stringify(limits.is_a?(Hash) ? limits : {}))
+        <<~PROMPT
+          You are improving the written instructions of a production customer-service agent
+          called "#{agent_id}". You are given (1) what actually broke in its recent real
+          traffic and (2) the current text of the only files you may edit.
+
+          Propose the SMALLEST set of anchored edits that would plausibly fix the findings.
+          Your proposal will be scored by replaying the agent's test cases with your edits
+          applied — plausible prose that changes no behaviour is worth nothing here.
+
+          ## What broke
+
+          #{render_findings(findings)}
+
+          ## Files you may edit (current content, verbatim)
+
+          #{render_files(files)}
+
+          ## Rules
+
+          - Answer with a single JSON object and NOTHING else. No prose, no fences.
+          - At most #{bounds['max_edits']} edits. Each `after` at most #{bounds['max_bytes']} bytes.
+          - `op` is "replace" or "append". Never rewrite a whole file.
+          - For "replace": `before` MUST be copied character-for-character from the file
+            above and must appear there EXACTLY ONCE. If you cannot find a unique anchor,
+            use "append" instead. An edit whose `before` does not match is discarded.
+          - For "append": `before` is ignored; the text is added at the end of the file.
+          - `file` must be one of the file names listed above.
+          - `addresses` lists the findings the edit is meant to fix, by their `kind` and
+            subject (e.g. "tool_error:shipping_quote").
+          - Do not propose an edit you cannot justify from a finding above. Fewer, better
+            edits beat filling the quota.
+          - You cannot remove a tool, change a guardrail or a model with these edits —
+            those are configuration, not text. Do not write instructions that pretend to.
+          - Some findings are INFRASTRUCTURE, not behaviour: a tool that failed on the
+            network, on TLS, on a refused connection, on a timeout, on a blocked
+            destination or on an HTTP status. The assistant does not choose URLs,
+            schemes, hosts or credentials and cannot fix any of that by being told to.
+            You may say what to DO when a tool fails; never say how to call it correctly.
+
+          ## Answer with exactly this shape
+
+          {"rationale": "one or two sentences on the cause you are addressing",
+           "edits": [{"file": "TOOLS.md", "op": "replace", "anchor": "## shipping_quote",
+                      "before": "<text copied from the file>",
+                      "after": "<the replacement>",
+                      "addresses": ["tool_error:shipping_quote"]}]}
+        PROMPT
+      end
+
+      # Counts included: "this happened 24 times" is the difference between a defect
+      # worth a prompt edit and a one-off the operator should ignore.
+      def render_findings(findings)
+        findings.map do |f|
+          f = Coercion.deep_stringify(f.respond_to?(:to_h) ? f.to_h : f)
+          line = "- #{f['kind']} (×#{f['count']}): #{f['title']}"
+          line += "\n  #{f['detail']}" if Coercion.present?(f["detail"])
+          line
+        end.join("\n")
+      end
+
+      def render_files(files)
+        files.map { |name, body| "### #{name}\n\n```\n#{body}\n```" }.join("\n\n")
+      end
+    end
+
+    # Resolves WHICH model(s) write the candidate, and builds the ask.
+    #
+    #   refinement.proposers  on the agent  (a PANEL)
+    #   -> refinement.proposer  ("deepseek/deepseek-v4-flash" | "deepseek-v4-flash")
+    #   -> the platform utility_model
+    #   -> nothing, and the caller refuses. There is no default model here on purpose:
+    #      guessing one spends an operator's provider budget without being asked.
+    module ProposerFactory
+      module_function
+
+      # config: the agent's `refinement` hash. -> Proposer | nil (the FIRST of the
+      # panel — single-proposer entry point, kept because a deployment that
+      # never configured a panel is a panel of one).
+      def build(config, utility_model: nil, ask_factory: nil, llm: nil)
+        panel(config, utility_model: utility_model, ask_factory: ask_factory, llm: llm).first
+      end
+
+      # -> [Proposer], in configured order, DEDUPED by model ref and capped at the
+      # fan-out (says the panel reuses it). Two entries naming the same
+      # model are one proposer: asking the same model twice at temperature 0 measures
+      # its variance, which is exactly what rejected for the judges.
+      # `llm`: the graph's own RubyLLM context; nil = the global
+      # constant. Today only the deployment root builds a panel, and a deployment
+      # is one graph per process — the seam exists so an embedded graph that ever
+      # gains the refinement commands proposes on its own credentials.
+      def panel(config, utility_model: nil, ask_factory: nil, max: nil, llm: nil)
+        refs = refs_for(Coercion.deep_stringify(config || {}), utility_model)
+        cap = max || Insika::SubagentGraph.fan_out_cap
+        factory = ask_factory || ->(model, provider) { ruby_llm_ask(model, provider, llm: llm) }
+        refs.first(cap).map do |ref|
+          provider, model = split_ref(ref)
+          Proposer.new(ask: factory.call(model, provider), model: ref)
+        end
+      end
+
+      # `proposers` accepts either syntax — a bare ref ("deepseek/deepseek-v4-flash") or
+      # the RFC's `{ "model" =>, "provider"? => }` — because the two already coexist in
+      # this config (`proposer` is a bare ref, `judges` are hashes) and refusing one of
+      # them would only teach operators which page they were reading.
+      def refs_for(config, utility_model)
+        listed = Array(config["proposers"]).filter_map { |entry| normalize_ref(entry) }
+        return listed.uniq unless listed.empty?
+
+        [Coercion.presence(config["proposer"]) || Coercion.presence(utility_model)].compact
+      end
+
+      def normalize_ref(entry)
+        return Coercion.presence(entry) unless entry.is_a?(Hash)
+
+        e = Coercion.deep_stringify(entry)
+        model = Coercion.presence(e["model"])
+        return nil if model.nil?
+
+        (provider = Coercion.presence(e["provider"])) ? "#{provider}/#{model}" : model
+      end
+
+      # "provider/model" -> [provider, model]; "model" -> [nil, model]. Same reading
+      # `Safety::Factory` uses for the moderator — one syntax for "which model", not
+      # one per feature.
+      def split_ref(ref)
+        prov, name = ref.to_s.split("/", 2)
+        name ? [prov, name] : [nil, prov]
+      end
+
+      # Temperature 0: a proposal an operator rejects should be re-proposable, and a
+      # gate result should be attributable to the edit rather than to a sampling seed.
+      # `ruby_llm` is required lazily so nothing loads a provider gem until a proposer
+      # is actually configured.
+      #
+      # Returns the MESSAGE, not `.content`: the token counts ride on it and the
+      # budget is what spends them. `Proposer` reads either shape.
+      def ruby_llm_ask(model, provider, llm: nil)
+        require "ruby_llm"
+        llm ||= RubyLLM
+        lambda do |prompt|
+          llm.chat(model: model, provider: provider, assume_model_exists: true)
+             .with_temperature(0).ask(prompt)
+        end
+      end
+    end
+  end
+end