insika 0.9.0 → 1.0.1

data/lib/insika/evals/assertions.rb

@@ -32,11 +32,12 @@ module Insika
 
       # A tool call whose status is anything but a success ("ok"/2xx/"success"). A
       # BLOCKED call is not an error: the tool never ran, a gate held it — that is
-      # the `blocked_gates` grader's business, not `must_not: tool_error`'s.
+      # the `blocked_gates` grader's business, not `must_not: tool_error`'s. Nor is a
+      # HELD one: it waits for the customer, and `phantom_action` reads the reply.
       def errored_tools
         Array(tool_calls).reject do |t|
           status = t["status"] || t[:status]
-          Assertions.ok_status?(status) || status.to_s == "blocked"
+          Assertions.ok_status?(status) || %w[blocked held].include?(status.to_s)
         end
       end
 
@@ -46,7 +47,16 @@ module Insika
     end
 
     # A single check within a case (e.g. "tool:shipping_quote", "must_not:pii_leak").
-    Check = Struct.new(:name, :pass, :detail, keyword_init: true)
+    #
+    # `skipped` is the third outcome, per check rather than per case: the transport
+    # could not observe what this grader reads (a harness that reports no tool
+    # calls), so the check neither held nor broke. It carries `pass: true` so it
+    # cannot fail a case it never examined, and the report prints it as skipped —
+    # the same discipline `requires` already applies to a whole case: reported,
+    # never a silent pass.
+    Check = Struct.new(:name, :pass, :detail, :skipped, keyword_init: true) do
+      def skipped? = skipped == true
+    end
 
     # The verdict for one golden case. `judge` (a Judge::Verdict) is attached AFTER
     # the deterministic pass when the case has a rubric and a judge is configured;
@@ -69,6 +79,7 @@ module Insika
       def skipped? = !skipped.nil?
       def pass? = !skipped? && error.nil? && checks.all?(&:pass) && (judge.nil? || judge.pass)
       def failures = checks.reject(&:pass)
+      def skipped_checks = checks.select(&:skipped?)
       # Has a rubric to score but no verdict yet (judge disabled / not run). A skipped
       # case is not pending anything — nobody is going to judge a turn that never ran.
       def judge_pending? = !skipped? && !rubric.to_s.strip.empty? && judge.nil?
@@ -100,6 +111,11 @@ module Insika
       # That is also the honest scope — a question nobody received is not a question.
       POLICIES = {
         # "UMA PERGUNTA POR VEZ" — the rule Insika broke twice under a 28 KB prompt.
+        # `max:` is the store's, default 1. Measured across six harnesses on one
+        # greeting task: with max 1 every reply that says "tudo bem?" fails and every
+        # reply that does not passes, which grades a courtesy formula rather than the
+        # rule. A store whose customers are greeted sets 2 and keeps the check for
+        # what it is for — the form ("qual seu nome, CPF e número do pedido?").
         "ask_once" => "at most one question per reply",
         # Establish the objective before acting on a vague opener.
         "investigate_first" => "asks before calling a tool, on the first turn",
@@ -156,7 +172,16 @@ module Insika
       # all of them — "one question per reply" is a rule about every reply, and the
       # violation that motivated this was on the FIRST turn. Defaults to the single
       # result so existing callers keep working.
-      def evaluate(golden, result, turns: nil)
+      # `store` is the post-turn snapshot ({ collection => [rows] }) when the case
+      # declares a `store_state:` and the runner had a reader; nil otherwise.
+      #
+      # `tools_reported` is false when the TRANSPORT cannot see tool calls at all —
+      # a benched harness that answers text and nothing else. Its tool-shaped graders
+      # are then skipped with the reason instead of passing (they would: an empty
+      # list satisfies `never_calls`) or failing (it would: nothing satisfies
+      # `tools_called`). Both readings would be a claim about a harness nobody
+      # measured.
+      def evaluate(golden, result, turns: nil, store: nil, store_error: nil, tools_reported: true)
         if result.error
           return CaseResult.new(id: golden.id, agent: golden.agent, error: result.error, rubric: nil, judge: nil,
                                 checks: [Check.new(name: "turn", pass: false, detail: "turn error: #{result.error}")])
@@ -165,12 +190,101 @@ module Insika
         # NOT `Array(turns)`: TurnResult is a Struct, so Array() would explode a single
         # one into its members and hand the policy checks three strings.
         conversation = turns.nil? || turns.empty? ? [result] : turns
-        checks = tool_checks(golden, result) + call_checks(golden, result) + reply_checks(golden, result) +
-                 ui_checks(golden, result) + must_not_checks(golden, result) + policy_checks(golden, conversation)
+        checks = tool_checks(golden, result) + turn_tool_checks(golden, conversation) + call_checks(golden, result) +
+                 reply_checks(golden, result) + ui_checks(golden, result) +
+                 must_not_checks(golden, result, conversation) + policy_checks(golden, conversation)
+        checks = skip_tool_shaped(checks) unless tools_reported
+        checks += store_state_checks(golden, store, error: store_error) if golden.store_state?
         CaseResult.new(id: golden.id, agent: golden.agent, error: nil, checks: checks,
                        rubric: golden.rubric, judge: nil)
       end
 
+      # Graders that read the turn's tool calls. Named by prefix in ONE place: a
+      # grader added without a line here would be silently mis-skipped, and the
+      # policies are in the list because `investigate_first`/`act_fast` are questions
+      # about whether a tool was called, not about the text.
+      TOOL_SHAPED = /\A(?:tool:|never_calls:|calls_one_of\z|first_tool:|max_tool_calls:|blocked_gates:|
+                        must_not:(?:tool_error|phantom_action)\z|policy:(?:investigate_first|act_fast)\z)/x
+
+      def skip_tool_shaped(checks)
+        checks.map do |c|
+          next c unless c.name.match?(TOOL_SHAPED)
+
+          Check.new(name: c.name, pass: true, skipped: true,
+                    detail: "skipped: this harness does not report tool calls")
+        end
+      end
+
+      # WHAT THE STORE LOOKS LIKE AFTER THE TURN. A confident answer with no order
+      # is a fail, and no reply grader can say so. `snapshot` is
+      # { collection => [rows] } as the reader returned it; nil means the reader
+      # gave nothing back, which fails rather than passes — an unread store must
+      # never read as a satisfied one.
+      def store_state_checks(golden, snapshot, error: nil)
+        if snapshot.nil?
+          return [Check.new(name: "store_state", pass: false,
+                            detail: "no store snapshot was read#{" — #{error}" if error}")]
+        end
+
+        rows = Insika::Coercion.deep_stringify(snapshot)
+        spec = golden.store_state
+        count_checks(spec["count"], rows) + record_checks(spec["records"], rows) +
+          absent_checks(spec["absent"], rows)
+      end
+
+      def count_checks(spec, rows)
+        Array(spec).map do |collection, expected|
+          actual = Array(rows[collection.to_s]).size
+          Check.new(name: "store_state:count:#{collection}", pass: actual == expected,
+                    detail: "expected #{expected}, saw #{actual}")
+        end
+      end
+
+      def record_checks(spec, rows)
+        Array(spec).flat_map do |collection, expected_rows|
+          present = Array(rows[collection.to_s])
+          expected_rows.each_with_index.map do |expected, i|
+            hit = present.any? { |row| record_match?(expected, row) }
+            Check.new(name: "store_state:#{collection}[#{i}]", pass: hit,
+                      detail: hit ? "matched" : "no row of #{present.size} in #{collection} matches #{expected.inspect}")
+          end
+        end
+      end
+
+      def absent_checks(spec, rows)
+        Array(spec).flat_map do |collection, expected_rows|
+          present = Array(rows[collection.to_s])
+          expected_rows.each_with_index.map do |expected, i|
+            hits = present.count { |row| record_match?(expected, row) }
+            Check.new(name: "store_state:absent:#{collection}[#{i}]", pass: hits.zero?,
+                      detail: hits.zero? ? "absent" : "#{hits} row(s) in #{collection} match #{expected.inspect}")
+          end
+        end
+      end
+
+      # A row matches when every field the case NAMES matches — a store row carries
+      # ids, timestamps and totals nobody wrote the case about, and demanding the
+      # whole row would make every task un-authorable.
+      def record_match?(expected, row)
+        row.is_a?(Hash) && expected.all? { |k, v| value_match?(v, row[k.to_s]) }
+      end
+
+      # Numbers compare as numbers (189.9 == 189.90, and a store that answers a
+      # string "189.90" is still the same money); a list means "each of these is in
+      # there", not "these are all of them" — `count:` is where exactness is stated.
+      def value_match?(expected, actual)
+        case expected
+        when Hash then record_match?(expected, actual)
+        when Array then actual.is_a?(Array) && expected.all? { |e| actual.any? { |a| value_match?(e, a) } }
+        when Numeric then numeric(actual) && (expected.to_f - actual.to_f).abs < 1e-6
+        else expected.to_s == actual.to_s
+        end
+      end
+
+      def numeric(value)
+        value.is_a?(Numeric) || (value.is_a?(String) && !Float(value, exception: false).nil?)
+      end
+
       # Each REQUIRED expected tool must appear in the turn's tool calls. Optional
       # ("name?") tools are informational — present or not, they never fail.
       def tool_checks(golden, result)
@@ -187,6 +301,20 @@ module Insika
       # The graders over the turn's CALLS. Each is its own Check, so the report names
       # the one that failed instead of "tool checks failed". All read the LAST turn,
       # like `tools_called`.
+      # The calls a specific turn had to make (`turns[i].tools_called`). The case-level
+      # grader reads the last turn; a two-turn case where turn one claimed an action
+      # and turn two repaired the store passes it, and this is what catches turn one.
+      def turn_tool_checks(golden, turns)
+        turns.each_with_index.flat_map do |turn, i|
+          names = turn.tool_names
+          golden.turn_tools_called(i).reject { |t| t[:optional] }.map do |t|
+            present = names.include?(t[:name])
+            Check.new(name: "tool:#{t[:name]}@turn#{i + 1}", pass: present,
+                      detail: present ? "called on turn #{i + 1}" : "turn #{i + 1} called #{names.empty? ? 'nothing' : names.join(', ')}")
+          end
+        end
+      end
+
       def call_checks(golden, result)
         names = result.tool_names
         checks = golden.never_calls.map do |name|
@@ -249,14 +377,40 @@ module Insika
                             detail: shown.empty? ? "nothing shown" : "shown: #{shown.join(', ')}")
       end
 
-      # `must_not` detectors. "tool_error" is special (inspects statuses); the rest
-      # are content detectors over the output text.
-      def must_not_checks(golden, result)
-        golden.must_not.map do |name|
+      # A reply that says it did something the store never saw happen. The case's
+      # `claims:` names each mutation and the words a reply uses to report it — the
+      # store's language, so it lives in the case, not here. Read after the turn,
+      # cumulative over the conversation ("já adicionei" on turn two is true when
+      # turn one did it) — and "ran" means a call that ENDED ok: a claim over a
+      # failed or blocked call is the same lie. Store-level arithmetic and duplicates
+      # are `store_state:`'s business; this is only whether the words match the calls.
+      def phantom_action_checks(golden, turns)
+        claims = golden.claims.transform_values { |pattern| Regexp.new(pattern, Regexp::IGNORECASE) }
+        ran = []
+        offences = turns.each_with_index.filter_map do |turn, i|
+          ran += Array(turn.tool_calls).select { |t| Assertions.ok_status?(t["status"] || t[:status]) }
+                                       .map { |t| (t["name"] || t[:name]).to_s }
+          claimed = claims.find { |tool, verbs| turn.output_text.to_s.match?(verbs) && !ran.include?(tool) }
+          next unless claimed
+
+          tool = claimed.first
+          how = turn.tool_names.include?(tool) ? "#{tool} did not succeed" : "#{tool} never ran"
+          "turn #{i + 1} claims an action but #{how}: #{turn.output_text.to_s.strip[0, 120].inspect}"
+        end
+        [Check.new(name: "must_not:phantom_action", pass: offences.empty?,
+                   detail: offences.empty? ? "every claimed action ran" : offences.join(" | "))]
+      end
+
+      # `must_not` detectors. "tool_error" and "phantom_action" are special (they read
+      # the calls); the rest are content detectors over the output text.
+      def must_not_checks(golden, result, turns = [result])
+        golden.must_not.flat_map do |name|
           if name == "tool_error"
             bad = result.errored_tools
             Check.new(name: "must_not:tool_error", pass: bad.empty?,
                       detail: bad.empty? ? "no tool errors" : "errored: #{bad.map { |t| t['name'] || t[:name] }.join(', ')}")
+          elsif name == "phantom_action"
+            phantom_action_checks(golden, turns)
           else
             hit = detect(name, result.output_text.to_s)
             Check.new(name: "must_not:#{name}", pass: hit.nil?,
@@ -274,7 +428,7 @@ module Insika
         # Exhaustive on purpose: a policy added to POLICIES without a rule here would
         # otherwise fall into whichever branch was last and check the wrong thing.
         pass, detail = case name
-                       when "ask_once" then ask_once(turns)
+                       when "ask_once" then ask_once(turns, max: golden.policy_options.fetch("max", 1).to_i)
                        when "investigate_first" then investigate_first(turns.first)
                        when "act_fast" then act_fast(turns.first)
                        else raise ArgumentError, "policy #{name.inspect} has no rule"
@@ -282,14 +436,15 @@ module Insika
         [Check.new(name: "policy:#{name}", pass: pass, detail: detail)]
       end
 
-      # Every reply asks at most one question. Reported with the offending turn and
+      # Every reply asks at most `max` questions. Reported with the offending turn and
       # the reply itself — "2 questions" alone sends the reader digging.
-      def ask_once(turns)
-        offender = turns.each_with_index.find { |t, _| count_questions(t.output_text) > 1 }
-        return [true, "at most one question per reply"] unless offender
+      def ask_once(turns, max: 1)
+        max = 1 if max < 1
+        offender = turns.each_with_index.find { |t, _| count_questions(t.output_text) > max }
+        return [true, "at most #{max} question(s) per reply"] unless offender
 
         turn, i = offender
-        [false, "turn #{i + 1} asked #{count_questions(turn.output_text)} questions: " \
+        [false, "turn #{i + 1} asked #{count_questions(turn.output_text)} questions (max #{max}): " \
                 "#{turn.output_text.to_s.strip[0, 160].inspect}"]
       end
 

data/lib/insika/evals/golden.rb

@@ -29,7 +29,7 @@ module Insika
     # contradiction from six turns ago) becomes reproducible. {} = the case starts
     # empty, as every case did before the key existed.
     Golden = Struct.new(:id, :agent, :turns, :expect, :requires, :reference, :source, :persona, :tenant,
-                        :state, keyword_init: true) do
+                        :state, :store_state, keyword_init: true) do
       # The user messages to replay, in order. Empty for a persona case: a generated
       # conversation has no scripted turns.
       def user_turns = turns.map { |t| t["user"] }
@@ -49,9 +49,23 @@ module Insika
         end
       end
 
+      # The tools turn `i` (0-based) must call, same `name?` optional marker as above.
+      # [] for a turn that pins nothing.
+      def turn_tools_called(i)
+        Array(turns.dig(i, "tools_called")).map do |ref|
+          s = ref.to_s
+          optional = s.end_with?("?")
+          { name: optional ? s[0..-2] : s, optional: optional }
+        end
+      end
+
       # Names of the negative assertions to run (e.g. "pii_leak", "tool_error").
       def must_not = Array(expect["must_not"]).map(&:to_s)
 
+      # `claims:` — tool name => the words a reply uses to say it did that (a regex
+      # source, matched case-insensitively). What `must_not: phantom_action` reads.
+      def claims = (expect["claims"] || {}).transform_keys(&:to_s)
+
       # The graders over the turn's CALLS and its published REPLY — each optional,
       # each deterministic. Every positive has a negative: `never_calls` pins what a
       # correct turn does NOT do, which `tools_called` alone can never say.
@@ -72,9 +86,40 @@ module Insika
       def state = self[:state] || {}
       def seeded? = !state.empty?
 
+      # What the STORE must look like after the turn — the half of a commerce case
+      # no reply can prove. `records` are rows that must be there, `count` the exact
+      # size of a collection (a duplicated order fails it), `absent` rows that must
+      # not exist. Read after the last turn by a store reader the runner is given;
+      # {} = the case says nothing about the store, as every case did before.
+      def store_state = self[:store_state] || {}
+      def store_state? = !store_state.empty?
+
+      # The collections this case talks about — what the reader is asked for, and
+      # nothing else: a bench task about orders should not pull the catalogue.
+      def store_collections
+        (Array(store_state["records"]&.keys) + Array(store_state["count"]&.keys) +
+          Array(store_state["absent"]&.keys)).map(&:to_s).uniq
+      end
+
       # How much the agent should ask before acting. nil = the store
       # has no opinion and only the rubric decides.
-      def policy = GoldenLoader.presence(expect["policy"])
+      def policy = policy_pair.first
+
+      # What the store set the policy TO. The engine ships the mechanism (count the
+      # questions); the number is the store's, because "one question per reply" is a
+      # tolerance, not a law — a greeting that says "tudo bem?" spends one on
+      # courtesy in half the languages a store sells in.
+      def policy_options = policy_pair.last
+
+      # `policy: ask_once` and `policy: { ask_once: { max: 2 } }` are the same
+      # declaration with and without a number. -> [name, options]
+      def policy_pair
+        value = expect["policy"]
+        return [GoldenLoader.presence(value), {}] unless value.is_a?(Hash)
+
+        name, options = value.first
+        [GoldenLoader.presence(name), (options || {}).transform_keys(&:to_s)]
+      end
 
       # What the DEPLOYMENT must have for this case to mean anything.
       # Empty = runs everywhere.
@@ -141,6 +186,7 @@ module Insika
         validate_policy!(expect["policy"], id: id, source: source)
         validate_graders!(expect, id: id, source: source)
         state = normalize_state(raw["state"], id: id, source: source)
+        store_state = normalize_store_state(raw["store_state"], id: id, source: source)
         requires = raw["requires"] || {}
         unless requires.is_a?(Hash)
           raise InvalidGolden, "#{source}: 'requires' must be a mapping (case '#{id}')"
@@ -151,7 +197,7 @@ module Insika
 
         Golden.new(id: id, agent: agent, turns: turns, expect: expect,
                    requires: requires, reference: reference, source: source, persona: persona,
-                   tenant: tenant, state: state)
+                   tenant: tenant, state: state, store_state: store_state)
       end
 
       # `persona:` is the alternative shape to `turns:`: the
@@ -234,6 +280,55 @@ module Insika
         state
       end
 
+      STORE_STATE_KEYS = %w[records count absent].freeze
+
+      # store_state: what the store must look like AFTER the turn, graded by code
+      # against a snapshot the runner reads. Three keys, each a mapping keyed by
+      # collection: `records` (rows that must exist), `count` (the exact size of a
+      # collection) and `absent` (rows that must not exist). A closed vocabulary on
+      # purpose — a case that wrote `orders:` at the top level would grade nothing
+      # and pass, which is the failure this key exists to catch.
+      def normalize_store_state(raw, id:, source:)
+        return {} if raw.nil?
+
+        where = "#{source}: store_state (case '#{id}')"
+        raise InvalidGolden, "#{where} must be a mapping" unless raw.is_a?(Hash)
+
+        unknown = raw.keys.map(&:to_s) - STORE_STATE_KEYS
+        unless unknown.empty?
+          raise InvalidGolden, "#{where}: unknown key(s) #{unknown.join(', ')} — " \
+                               "known: #{STORE_STATE_KEYS.join(', ')}"
+        end
+
+        state = raw.compact
+        rows_by_collection!(state["records"], "#{where}.records")
+        rows_by_collection!(state["absent"], "#{where}.absent")
+        counts!(state["count"], "#{where}.count")
+        state
+      end
+
+      def rows_by_collection!(value, where)
+        return if value.nil?
+        raise InvalidGolden, "#{where} must be a mapping of collection -> rows" unless value.is_a?(Hash)
+
+        value.each do |collection, rows|
+          next if rows.is_a?(Array) && !rows.empty? && rows.all?(Hash)
+
+          raise InvalidGolden, "#{where}.#{collection} must be a non-empty list of mappings"
+        end
+      end
+
+      def counts!(value, where)
+        return if value.nil?
+        raise InvalidGolden, "#{where} must be a mapping of collection -> integer" unless value.is_a?(Hash)
+
+        value.each do |collection, n|
+          next if n.is_a?(Integer) && n >= 0
+
+          raise InvalidGolden, "#{where}.#{collection} must be a non-negative integer (got #{n.inspect})"
+        end
+      end
+
       def mapping_with!(value, key, type, where)
         return if value.nil?
         raise InvalidGolden, "#{where} must be a mapping" unless value.is_a?(Hash)
@@ -261,6 +356,28 @@ module Insika
 
           raise InvalidGolden, "#{source}: blocked_gates entries are 'tool:gate' (got #{pair.inspect}, case '#{id}')"
         end
+
+        validate_claims!(expect, id: id, source: source)
+      end
+
+      # A phantom-action check with no words to look for would pass everything, and a
+      # pattern that does not compile would fail every cell at grading time instead of
+      # the one author at load time.
+      def validate_claims!(expect, id:, source:)
+        claims = expect["claims"]
+        phantom = Array(expect["must_not"]).map(&:to_s).include?("phantom_action")
+        if phantom && !(claims.is_a?(Hash) && !claims.empty?)
+          raise InvalidGolden, "#{source}: must_not: phantom_action needs a non-empty 'claims:' map (case '#{id}')"
+        end
+        return if claims.nil?
+
+        raise InvalidGolden, "#{source}: 'claims' must be a map of tool => pattern (case '#{id}')" unless claims.is_a?(Hash)
+
+        claims.each do |tool, pattern|
+          Regexp.new(pattern.to_s)
+        rescue RegexpError => e
+          raise InvalidGolden, "#{source}: claims.#{tool} is not a valid pattern — #{e.message} (case '#{id}')"
+        end
       end
 
       # A typo'd policy must not silently mean "no policy" — the case would go on
@@ -268,6 +385,16 @@ module Insika
       # `Assertions` constant is resolved at CALL time (this file loads first, and
       # assertions.rb touches `Safety::Detectors` at load time).
       def validate_policy!(value, id:, source:)
+        if value.is_a?(Hash)
+          raise InvalidGolden, "#{source}: policy takes ONE name (case '#{id}')" unless value.size == 1
+
+          name, options = value.first
+          unless options.nil? || options.is_a?(Hash)
+            raise InvalidGolden, "#{source}: policy #{name.inspect} options must be a map (case '#{id}')"
+          end
+
+          value = name
+        end
         name = presence(value)
         return if name.nil? || Assertions::POLICIES.key?(name)
 
@@ -277,6 +404,11 @@ module Insika
 
       # turns: a non-empty array of { "user" => String }. Rejects anything else so a
       # typo (e.g. `users:`) surfaces at load time, not as an empty replay.
+      #
+      # A turn may also carry `tools_called: [names]` — the calls THAT turn must make.
+      # The case-level `tools_called` reads the last turn only, and the bench found a
+      # reply that said "adicionado ao carrinho" on turn one without ever calling
+      # add_to_cart, then recovered on turn two so the final store looked right.
       def normalize_turns(turns, id:, source:)
         unless turns.is_a?(Array) && !turns.empty?
           raise InvalidGolden, "#{source}: 'turns' must be a non-empty array (case '#{id}')"
@@ -285,7 +417,12 @@ module Insika
         turns.each_with_index.map do |t, i|
           user = t.is_a?(Hash) ? presence(t["user"]) : nil
           user || (raise InvalidGolden, "#{source}: turns[#{i}] needs a non-empty 'user' (case '#{id}')")
-          { "user" => user }
+          tools = t["tools_called"]
+          unless tools.nil? || (tools.is_a?(Array) && tools.all? { |n| presence(n) })
+            raise InvalidGolden, "#{source}: turns[#{i}].tools_called must be a list of tool names (case '#{id}')"
+          end
+
+          tools.nil? ? { "user" => user } : { "user" => user, "tools_called" => tools.map(&:to_s) }
         end
       end
 

data/lib/insika/evals/judge.rb

@@ -37,8 +37,13 @@ module Insika
       # rubric alone decides, and inventing a default here would be inventing an
       # opinion for someone else's store.
       POLICY_INSTRUCTIONS = {
-        "ask_once" => "This store allows AT MOST ONE question per reply. Two questions in " \
-                      "one message is a failure even if the content is otherwise good.",
+        # %<max>s is the store's number, not ours. Stated to the judge in the same
+        # words and with the same threshold the deterministic layer uses — a judge
+        # told "at most one" while the counter allows two grades a different rule.
+        "ask_once" => "This store allows AT MOST %<max>s question(s) per reply. One more " \
+                      "than that in a single message is a failure even if the content is " \
+                      "otherwise good. A greeting formula that asks nothing the assistant " \
+                      "needs (\"tudo bem?\", \"how are you?\") is courtesy, not a question.",
         "investigate_first" => "This store wants the objective established BEFORE acting: on a " \
                                "vague request the assistant should ask (once or twice, not a " \
                                "form), not search immediately.",
@@ -66,7 +71,8 @@ module Insika
         rubric = golden.rubric.to_s.strip
         return nil if rubric.empty?
 
-        prompt = build_prompt(rubric, golden.user_turns, result.output_text.to_s, golden.policy)
+        prompt = build_prompt(rubric, golden.user_turns, result.output_text.to_s, golden.policy,
+                              golden.respond_to?(:policy_options) ? golden.policy_options : {})
         score_prompt(prompt, golden.min_score || DEFAULT_MIN_SCORE)
       end
 
@@ -75,11 +81,12 @@ module Insika
       # [{ role: "user"|"assistant", text: }] in order. -> Verdict, or nil without
       # a rubric. Same panel, same rules as `score`: the rubric that scores a
       # scripted replay scores a generated one.
-      def score_conversation(rubric:, transcript:, policy: nil, min_score: DEFAULT_MIN_SCORE)
+      def score_conversation(rubric:, transcript:, policy: nil, policy_options: {},
+                             min_score: DEFAULT_MIN_SCORE)
         rubric = rubric.to_s.strip
         return nil if rubric.empty?
 
-        prompt = conversation_prompt(rubric, transcript, policy)
+        prompt = conversation_prompt(rubric, transcript, policy, policy_options)
         score_prompt(prompt, min_score)
       end
 
@@ -102,7 +109,7 @@ module Insika
       # user turn and every assistant reply interleaved. A rubric about the whole
       # exchange ("does it discover the objective before recommending?") is
       # unanswerable from the last reply alone.
-      def conversation_prompt(rubric, transcript, policy)
+      def conversation_prompt(rubric, transcript, policy, policy_options = {})
         lines = Array(transcript).map do |m|
           role = m[:role].to_s == "user" ? "customer" : "assistant"
           "#{role}: #{m[:text].to_s.strip}"
@@ -114,7 +121,7 @@ module Insika
 
           RUBRIC:
           #{rubric}
-          #{policy_clause(policy)}
+          #{policy_clause(policy, policy_options)}
           CONVERSATION (in order):
           #{lines}
 
@@ -144,14 +151,14 @@ module Insika
       # much this store wants the agent to ask before acting is a per-store decision,
       # and a judge that is not TOLD it will guess — half the time wrongly. The
       # deterministic half is `Assertions.policy_checks`; this is the other half.
-      def build_prompt(rubric, user_turns, reply, policy = nil)
+      def build_prompt(rubric, user_turns, reply, policy = nil, policy_options = {})
         <<~PROMPT
           You are a strict QA judge for a customer-service AI assistant. Judge the
           ASSISTANT REPLY against the RUBRIC — nothing else.
 
           RUBRIC:
           #{rubric}
-          #{policy_clause(policy)}
+          #{policy_clause(policy, policy_options)}
           CONVERSATION (user turns, in order):
           #{user_turns.map { |t| "- #{t}" }.join("\n")}
 
@@ -164,10 +171,12 @@ module Insika
         PROMPT
       end
 
-      def policy_clause(policy)
+      def policy_clause(policy, policy_options = {})
         instruction = POLICY_INSTRUCTIONS[policy.to_s]
         return "" unless instruction
 
+        instruction = format(instruction, max: (policy_options || {}).fetch("max", 1)) if
+          instruction.include?("%<max>s")
         "\nSTORE POLICY (weigh this as part of the rubric):\n#{instruction}\n"
       end
 

data/lib/insika/evals/moderator_compare.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Insika
+  module Evals
+    # Runs each moderator over the same labeled messages and counts how they
+    # differ from the label. Pure: the moderators are injected.
+    class ModeratorCompare
+      BLOCKS = %w[refuse escalate].freeze
+
+      def initialize(moderators:)
+        @moderators = moderators
+      end
+
+      def run(cases)
+        @moderators.to_h { |name, mod| [name, score(mod, cases)] }
+      end
+
+      private
+
+      def score(mod, cases)
+        r = { "correct" => 0, "total" => cases.size, "false_blocks" => 0, "missed_blocks" => 0,
+              "unavailable" => 0, "misses" => [], "verdicts" => {} }
+        times = cases.map do |c|
+          t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          v = mod.classify(c["message"])
+          tally(r, c, v)
+          (Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0) * 1000
+        end.sort
+        r.merge("p50_ms" => pct(times, 0.5), "p95_ms" => pct(times, 0.95))
+      end
+
+      def tally(r, c, v)
+        r["verdicts"][c["id"]] = { "action" => v.action.to_s, "category" => v.category.to_s, "reason" => v.reason.to_s }
+        r["unavailable"] += 1 if v.unavailable?
+        want_block = BLOCKS.include?(c["expected"])
+        ok = want_block ? v.block? : !v.block? && !v.unavailable?
+        return r["correct"] += 1 if ok
+
+        r["misses"] << c["id"]
+        r["false_blocks"] += 1 if !want_block && v.block?
+        r["missed_blocks"] += 1 if want_block && !v.block?
+      end
+
+      def pct(sorted, q) = sorted.empty? ? 0 : sorted[((sorted.size - 1) * q).round].round(1)
+    end
+  end
+end