insika 0.8.0 → 1.0.1

data/lib/insika/evals/judge.rb

@@ -37,8 +37,13 @@ module Insika
       # rubric alone decides, and inventing a default here would be inventing an
       # opinion for someone else's store.
       POLICY_INSTRUCTIONS = {
-        "ask_once" => "This store allows AT MOST ONE question per reply. Two questions in " \
-                      "one message is a failure even if the content is otherwise good.",
+        # %<max>s is the store's number, not ours. Stated to the judge in the same
+        # words and with the same threshold the deterministic layer uses — a judge
+        # told "at most one" while the counter allows two grades a different rule.
+        "ask_once" => "This store allows AT MOST %<max>s question(s) per reply. One more " \
+                      "than that in a single message is a failure even if the content is " \
+                      "otherwise good. A greeting formula that asks nothing the assistant " \
+                      "needs (\"tudo bem?\", \"how are you?\") is courtesy, not a question.",
         "investigate_first" => "This store wants the objective established BEFORE acting: on a " \
                                "vague request the assistant should ask (once or twice, not a " \
                                "form), not search immediately.",
@@ -66,7 +71,8 @@ module Insika
         rubric = golden.rubric.to_s.strip
         return nil if rubric.empty?
 
-        prompt = build_prompt(rubric, golden.user_turns, result.output_text.to_s, golden.policy)
+        prompt = build_prompt(rubric, golden.user_turns, result.output_text.to_s, golden.policy,
+                              golden.respond_to?(:policy_options) ? golden.policy_options : {})
         score_prompt(prompt, golden.min_score || DEFAULT_MIN_SCORE)
       end
 
@@ -75,11 +81,12 @@ module Insika
       # [{ role: "user"|"assistant", text: }] in order. -> Verdict, or nil without
       # a rubric. Same panel, same rules as `score`: the rubric that scores a
       # scripted replay scores a generated one.
-      def score_conversation(rubric:, transcript:, policy: nil, min_score: DEFAULT_MIN_SCORE)
+      def score_conversation(rubric:, transcript:, policy: nil, policy_options: {},
+                             min_score: DEFAULT_MIN_SCORE)
         rubric = rubric.to_s.strip
         return nil if rubric.empty?
 
-        prompt = conversation_prompt(rubric, transcript, policy)
+        prompt = conversation_prompt(rubric, transcript, policy, policy_options)
         score_prompt(prompt, min_score)
       end
 
@@ -102,7 +109,7 @@ module Insika
       # user turn and every assistant reply interleaved. A rubric about the whole
       # exchange ("does it discover the objective before recommending?") is
       # unanswerable from the last reply alone.
-      def conversation_prompt(rubric, transcript, policy)
+      def conversation_prompt(rubric, transcript, policy, policy_options = {})
         lines = Array(transcript).map do |m|
           role = m[:role].to_s == "user" ? "customer" : "assistant"
           "#{role}: #{m[:text].to_s.strip}"
@@ -114,7 +121,7 @@ module Insika
 
           RUBRIC:
           #{rubric}
-          #{policy_clause(policy)}
+          #{policy_clause(policy, policy_options)}
           CONVERSATION (in order):
           #{lines}
 
@@ -144,14 +151,14 @@ module Insika
       # much this store wants the agent to ask before acting is a per-store decision,
       # and a judge that is not TOLD it will guess — half the time wrongly. The
       # deterministic half is `Assertions.policy_checks`; this is the other half.
-      def build_prompt(rubric, user_turns, reply, policy = nil)
+      def build_prompt(rubric, user_turns, reply, policy = nil, policy_options = {})
         <<~PROMPT
           You are a strict QA judge for a customer-service AI assistant. Judge the
           ASSISTANT REPLY against the RUBRIC — nothing else.
 
           RUBRIC:
           #{rubric}
-          #{policy_clause(policy)}
+          #{policy_clause(policy, policy_options)}
           CONVERSATION (user turns, in order):
           #{user_turns.map { |t| "- #{t}" }.join("\n")}
 
@@ -164,10 +171,12 @@ module Insika
         PROMPT
       end
 
-      def policy_clause(policy)
+      def policy_clause(policy, policy_options = {})
         instruction = POLICY_INSTRUCTIONS[policy.to_s]
         return "" unless instruction
 
+        instruction = format(instruction, max: (policy_options || {}).fetch("max", 1)) if
+          instruction.include?("%<max>s")
         "\nSTORE POLICY (weigh this as part of the rubric):\n#{instruction}\n"
       end
 

data/lib/insika/evals/moderator_compare.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Insika
+  module Evals
+    # Runs each moderator over the same labeled messages and counts how they
+    # differ from the label. Pure: the moderators are injected.
+    class ModeratorCompare
+      BLOCKS = %w[refuse escalate].freeze
+
+      def initialize(moderators:)
+        @moderators = moderators
+      end
+
+      def run(cases)
+        @moderators.to_h { |name, mod| [name, score(mod, cases)] }
+      end
+
+      private
+
+      def score(mod, cases)
+        r = { "correct" => 0, "total" => cases.size, "false_blocks" => 0, "missed_blocks" => 0,
+              "unavailable" => 0, "misses" => [], "verdicts" => {} }
+        times = cases.map do |c|
+          t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          v = mod.classify(c["message"])
+          tally(r, c, v)
+          (Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0) * 1000
+        end.sort
+        r.merge("p50_ms" => pct(times, 0.5), "p95_ms" => pct(times, 0.95))
+      end
+
+      def tally(r, c, v)
+        r["verdicts"][c["id"]] = { "action" => v.action.to_s, "category" => v.category.to_s, "reason" => v.reason.to_s }
+        r["unavailable"] += 1 if v.unavailable?
+        want_block = BLOCKS.include?(c["expected"])
+        ok = want_block ? v.block? : !v.block? && !v.unavailable?
+        return r["correct"] += 1 if ok
+
+        r["misses"] << c["id"]
+        r["false_blocks"] += 1 if !want_block && v.block?
+        r["missed_blocks"] += 1 if want_block && !v.block?
+      end
+
+      def pct(sorted, q) = sorted.empty? ? 0 : sorted[((sorted.size - 1) * q).round].round(1)
+    end
+  end
+end

data/lib/insika/evals/report.rb

@@ -12,7 +12,15 @@ module Insika
       module_function
 
       # -> Hash ready for JSON. `at` is an ISO-8601 string stamped by the caller.
-      def to_h(results, at:)
+      #
+      # `perf` is what each case COST, keyed by case id: `{"ms" =>, "turns" =>,
+      # "tokens" =>, "cached" =>}`. Optional, because the answer to "did it pass" must
+      # not depend on anyone having measured the clock — but a cross-harness table
+      # without it is a table about correctness only, and a merchant asks how long the
+      # customer waited before asking anything else. Wall clock is measured by the
+      # RUNNER around the turn, never self-reported: it is the one column every
+      # entrant can be held to.
+      def to_h(results, at:, perf: nil)
         passed = results.count(&:pass?)
         skipped = results.count(&:skipped?)
         {
@@ -27,17 +35,22 @@ module Insika
           # Absent (not an empty tally) when nothing was compared — a run with no
           # pairwise is the normal one, and a zeroed block reads like every case tied.
           "pairwise" => pairwise_summary(results),
+          "perf" => perf_summary(perf),
           "cases" => results.map do |r|
             {
               "id" => r.id, "agent" => r.agent, "pass" => r.pass?,
               "skipped" => r.skipped,
               "judge_pending" => r.judge_pending?, "error" => r.error,
-              "checks" => r.checks.map { |c| { "name" => c.name, "pass" => c.pass, "detail" => c.detail } },
+              "checks" => r.checks.map do |c|
+                { "name" => c.name, "pass" => c.pass, "detail" => c.detail,
+                  "skipped" => (true if c.skipped?) }.compact
+              end,
               "judge" => (r.judge && { "score" => r.judge.score, "pass" => r.judge.pass, "reason" => r.judge.reason }),
               "pairwise" => (r.pairwise && { "outcome" => r.pairwise.outcome, "vs" => r.pairwise.vs,
                                              "reason" => r.pairwise.reason, "judges" => r.pairwise.judges,
-                                             "order_dependent" => r.pairwise.order_dependent })
-            }
+                                             "order_dependent" => r.pairwise.order_dependent }),
+              "perf" => perf&.[](r.id)
+            }.compact
           end
         }.compact
       end
@@ -57,8 +70,36 @@ module Insika
           "outcomes" => counts }
       end
 
-      def to_json(results, at:)
-        JSON.pretty_generate(to_h(results, at: at))
+      # nil when nothing was timed — an absent block, never a zeroed one: "nobody
+      # measured" and "it took no time" are different facts, and a table that confuses
+      # them publishes a harness as instant.
+      def perf_summary(perf)
+        measured = Array(perf&.values).select { |v| v["ms"] }
+        return nil if measured.empty?
+
+        case_ms = measured.map { |v| v["ms"].to_f }.sort
+        turn_ms = measured.flat_map { |v| Array(v["turn_ms"]) }.compact.sort
+        tokens = measured.filter_map { |v| v["tokens"] }
+        { "cases_measured" => measured.size,
+          "case_ms_p50" => percentile(case_ms, 50), "case_ms_p95" => percentile(case_ms, 95),
+          "turn_ms_p50" => percentile(turn_ms, 50), "turn_ms_p95" => percentile(turn_ms, 95),
+          # Absent rather than zero when no provider reported usage — most rival
+          # harnesses will not, and a zero there reads as free.
+          "tokens_total" => (tokens.sum unless tokens.empty?),
+          "tokens_measured" => tokens.size }.compact
+      end
+
+      def percentile(sorted, p)
+        return nil if sorted.empty?
+
+        r = (p / 100.0) * (sorted.length - 1)
+        lo = sorted[r.floor]
+        hi = sorted[r.ceil]
+        (lo + ((hi - lo) * (r - r.floor))).round
+      end
+
+      def to_json(results, at:, perf: nil)
+        JSON.pretty_generate(to_h(results, at: at, perf: perf))
       end
 
       PAIRWISE_MARK = { "better" => "🟢", "comparable" => "🟡", "worse" => "🔴",
@@ -85,8 +126,8 @@ module Insika
       end
 
       # Human summary. One line per case; failing checks nested underneath.
-      def to_markdown(results, at:)
-        h = to_h(results, at: at)
+      def to_markdown(results, at:, perf: nil)
+        h = to_h(results, at: at, perf: perf)
         lines = ["# Eval report — #{at}", "",
                  "**#{h['passed']}/#{h['total'] - h['skipped']} passed** · #{h['failed']} failed" \
                  "#{" · #{h['skipped']} skipped" if h['skipped'].positive?}" \
@@ -99,14 +140,25 @@ module Insika
             next
           end
 
-          lines << "- #{r.pass? ? '✅' : '❌'} `#{r.id}` (#{r.agent})#{'  ⏳ judge pending' if r.judge_pending?}"
+          took = perf&.dig(r.id, "ms")
+          lines << "- #{r.pass? ? '✅' : '❌'} `#{r.id}` (#{r.agent})" \
+                   "#{" · #{(took / 1000.0).round(1)}s" if took}" \
+                   "#{'  ⏳ judge pending' if r.judge_pending?}"
           r.failures.each { |c| lines << "    - ❌ #{c.name}: #{c.detail}" }
+          # A grader the transport could not feed. Printed even on a passing case:
+          # "passed" over checks nobody could run is the claim this line prevents.
+          r.skipped_checks.each { |c| lines << "    - ⏭️ #{c.name}: #{c.detail}" }
           if r.judge
             v = r.judge
             lines << "    - #{v.pass ? '✅' : '❌'} judge: #{v.score} — #{v.reason}"
           end
           lines << "    - #{pairwise_line(r.pairwise)}" if r.pairwise
         end
+        if (summary = h["perf"])
+          lines << "" << "**time** (#{summary['cases_measured']} case(s)): " \
+                   "case p50/p95 #{summary['case_ms_p50']} / #{summary['case_ms_p95']} ms · " \
+                   "turn p50/p95 #{summary['turn_ms_p50']} / #{summary['turn_ms_p95']} ms"
+        end
         lines.concat(pairwise_block(h["pairwise"])) if h["pairwise"]
         "#{lines.join("\n")}\n"
       end

data/lib/insika/evals/runner.rb

@@ -19,7 +19,11 @@ module Insika
       # cache, carried separately because it is the number that explains a total.
       # Only the refinement gate reads them (records a run's cost); the
       # report and the exit code are untouched.
-      RunCase = Struct.new(:result, :timings, :tokens, :cached, keyword_init: true)
+      # `turns` and `store` are what the case actually SAW: every turn's reply and
+      # tool calls, and the store as it stood afterwards. The verdict alone says a
+      # cell failed; only these say what the harness answered, and a cross-harness
+      # table is unreadable without being able to go and look.
+      RunCase = Struct.new(:result, :timings, :tokens, :cached, :turns, :store, keyword_init: true)
 
       # judge: an Evals::Judge (optional). When set, a case with a rubric whose turn
       # ran cleanly gets a subjective verdict attached on top of the deterministic pass.
@@ -35,12 +39,18 @@ module Insika
       # pairwise: an Evals::Pairwise (optional). Only cases carrying a
       # `reference:` are compared, and the verdict never touches pass/fail — it is the
       # answer to "can we replace it", reported beside the suite's own verdict.
-      def initialize(transport:, judge: nil, conv_map: {}, capabilities: nil, pairwise: nil)
+      #
+      # store_reader: how a case's `store_state:` is graded — anything answering
+      # `#read(collection)` with the collection's rows. Only the cases that declare a
+      # `store_state:` ask for it; a case that declares one with no reader configured
+      # is SKIPPED with the reason, never run and passed on its reply alone.
+      def initialize(transport:, judge: nil, conv_map: {}, capabilities: nil, pairwise: nil, store_reader: nil)
         @transport = transport
         @judge = judge
         @conv_map = conv_map || {}
         @capabilities = capabilities
         @pairwise = pairwise
+        @store_reader = store_reader
       end
 
       # [Golden] -> [RunCase]. Each RunCase carries the CaseResult (for the report) +
@@ -62,10 +72,37 @@ module Insika
         skip = skip_reason(golden)
         return RunCase.new(result: Assertions.skip(golden, skip), timings: []) if skip
 
+        if golden.store_state? && @store_reader.nil?
+          return RunCase.new(result: Assertions.skip(golden, "no store reader configured for `store_state`"),
+                             timings: [])
+        end
+
         # A backend that resolves state from a pre-existing conversation (e.g. a
         # consumer needing a real Chat UUID as X-Chat-Id) supplies it via conv_map; otherwise the
         # synthetic "eval-<id>" keeps the adapter's own multi-turn continuation.
+        # A reader that caches its snapshot is opening a new case now.
+        @store_reader.forget if @store_reader.respond_to?(:forget)
         conv = @conv_map[golden.id] || "eval-#{golden.id}"
+        if golden.seeded?
+          # The snapshot goes in BEFORE turn 1 — on the same conversation id the turns
+          # continue. A transport that cannot seed (A2A: a remote agent has no seed
+          # route) SKIPS the case with the reason, never runs it against an empty
+          # state and passes. A deployment that refuses seeding (the setting is off)
+          # is the same outcome: skipped and reported, which is what the `requires`
+          # discipline already promises for anything the deployment lacks.
+          unless @transport.respond_to?(:seed)
+            return RunCase.new(result: Assertions.skip(golden, "transport cannot seed state"), timings: [])
+          end
+
+          begin
+            @transport.seed(conv, golden.state)
+          rescue SeedRefused => e
+            return RunCase.new(result: Assertions.skip(golden, "deployment refuses seeding — #{e.message}"), timings: [])
+          rescue Insika::Error => e
+            failed = TurnResult.new(output_text: "", tool_calls: [], error: "seed failed: #{e.message}")
+            return RunCase.new(result: Assertions.evaluate(golden, failed), timings: [])
+          end
+        end
         turns = []
         timings = []
         spent = []
@@ -81,7 +118,10 @@ module Insika
         last = turns.last
         # Tool/content assertions read the last turn (unchanged); the policy checks
         # read every turn — "one question per reply" is a rule about each of them.
-        result = Assertions.evaluate(golden, last, turns: turns)
+        # The store is read AFTER the last turn: what the conversation left behind.
+        store, store_error = read_store(golden) if golden.store_state? && last.error.nil?
+        result = Assertions.evaluate(golden, last, turns: turns, store: store, store_error: store_error,
+                                     tools_reported: tools_reported?)
         # Subjective layer: only when a judge is configured, the case has a rubric, and
         # the turn ran cleanly (nothing to judge on an errored turn).
         result.judge = @judge.score(golden: golden, result: last) if @judge && result.rubric && result.error.nil?
@@ -90,11 +130,31 @@ module Insika
         # comparison for a reason that has nothing to do with the agent.
         result.pairwise = @pairwise.compare(golden: golden, turns: turns) if @pairwise && result.error.nil?
         RunCase.new(result: result, timings: timings, tokens: sum_tokens(spent),
-                    cached: sum_tokens(cached))
+                    cached: sum_tokens(cached), turns: turns, store: store)
       end
 
       private
 
+      # Does the transport see tool calls at all? Only a benched harness answers no
+      # (an adapter that can report the text and nothing else). Anything that does
+      # not answer the question is assumed to report them, which is what every
+      # transport in the engine does.
+      def tools_reported?
+        return true unless @transport.respond_to?(:reports_tool_calls?)
+
+        @transport.reports_tool_calls?
+      end
+
+      # -> [{ collection => [rows] }, nil] or [nil, why]. A reader that raises fails
+      # the store graders WITH the reason rather than killing the run: one unreadable
+      # cell of a bench table must not throw away the other thirty-nine, and a store
+      # nobody could read is never a store that matched.
+      def read_store(golden)
+        [golden.store_collections.to_h { |c| [c, Array(@store_reader.read(c))] }, nil]
+      rescue StandardError => e
+        [nil, "#{e.class}: #{e.message}"]
+      end
+
       # nil when NO turn reported usage; otherwise the sum of the ones that did. A
       # partially-metered case is reported as what was actually measured, low rather
       # than absent — a budget under-counting is a smaller lie than a budget that

data/lib/insika/evals/simulator.rb

@@ -95,11 +95,20 @@ module Insika
         @safety = safety
       end
 
-      # Runs one simulated conversation. -> SimulatedRun.
-      def run(persona:, agent:, conv:)
+      # Runs one simulated conversation. -> SimulatedRun. `state` is the case's
+      # snapshot (Golden#state), loaded into the conversation before the persona's
+      # opening line — a simulated customer can start from "already saw three
+      # products" too. Empty/nil = the conversation starts empty, as before.
+      def run(persona:, agent:, conv:, state: nil)
         reason = @safety.refusal
         raise UnsafeTarget, reason if reason
 
+        if state && !state.empty?
+          raise Insika::Error, "transport cannot seed state" unless @transport.respond_to?(:seed)
+
+          @transport.seed(conv, state)
+        end
+
         transcript = []
         message = persona.opens_with
         stop = :max_turns
@@ -157,12 +166,21 @@ module Insika
       # [tool names] the agent can reach that are marked side-effect, sorted.
       def side_effect_tools(profile, registry)
         allowed = if profile.tools_allow.nil? then Array(registry.names)
-                  else Array(profile.tools_allow).map(&:to_s)
+                  else Array(profile.tools_allow).map(&:to_s) | group_tools(profile, registry)
                   end
         denied = Array(profile.tools_deny).map(&:to_s)
         (allowed - denied).select { |name| registry.side_effect?(name) }.sort
       end
 
+      # Tools reachable through tools_allow_groups (an MCP server, a kit's groups):
+      # the registry entries whose metadata group the agent allows.
+      def group_tools(profile, registry)
+        groups = Array(profile.tools_allow_groups).map(&:to_s)
+        return [] if groups.empty? || !registry.respond_to?(:entries)
+
+        registry.entries.select { |e| groups.include?(e.metadata[:group].to_s) }.map { |e| e.name.to_s }
+      end
+
       # -> bool: can a simulated run touch this agent without a staging
       # declaration (no reachable side-effect tool)?
       def safe?(profile, registry)