insika 0.8.0 → 1.0.1

data/lib/insika/evals/assertions.rb

@@ -13,19 +13,50 @@ module Insika
     # offline without a server.
     #
     #   output_text: the final assistant text (for content checks)
-    #   tool_calls:  [{ "name" =>, "status" => }] captured from the stream's tool events
+    #   tool_calls:  [{ "name" =>, "arguments" =>, "status" =>, "gate" => }] captured
+    #                from the stream's tool events. `arguments` (a Hash) and
+    #                `status` ("ok" | "error" | "blocked", with `gate` naming what
+    #                held a blocked call) arrive from the item frames; an older
+    #                deployment sends names only and they stay nil.
+    #   ui:          [{ "component" =>, "count" => }] — one per presentation call
+    #                (the `insika.ui` frames); nil/[] when the turn showed nothing
     #   error:       transport/turn error string, or nil on a clean turn
-    TurnResult = Struct.new(:output_text, :tool_calls, :error, keyword_init: true) do
+    TurnResult = Struct.new(:output_text, :tool_calls, :ui, :error, keyword_init: true) do
       def tool_names = Array(tool_calls).map { |t| (t["name"] || t[:name]).to_s }
 
-      # A tool call whose status is anything but a success ("ok"/2xx/"success").
+      # Components that actually put a card in front of the customer (count > 0).
+      def shown_components
+        Array(ui).select { |u| (u["count"] || u[:count]).to_i.positive? }
+                 .map { |u| (u["component"] || u[:component]).to_s }
+      end
+
+      # A tool call whose status is anything but a success ("ok"/2xx/"success"). A
+      # BLOCKED call is not an error: the tool never ran, a gate held it — that is
+      # the `blocked_gates` grader's business, not `must_not: tool_error`'s. Nor is a
+      # HELD one: it waits for the customer, and `phantom_action` reads the reply.
       def errored_tools
-        Array(tool_calls).reject { |t| Assertions.ok_status?(t["status"] || t[:status]) }
+        Array(tool_calls).reject do |t|
+          status = t["status"] || t[:status]
+          Assertions.ok_status?(status) || %w[blocked held].include?(status.to_s)
+        end
+      end
+
+      def blocked_tools
+        Array(tool_calls).select { |t| (t["status"] || t[:status]).to_s == "blocked" }
       end
     end
 
     # A single check within a case (e.g. "tool:shipping_quote", "must_not:pii_leak").
-    Check = Struct.new(:name, :pass, :detail, keyword_init: true)
+    #
+    # `skipped` is the third outcome, per check rather than per case: the transport
+    # could not observe what this grader reads (a harness that reports no tool
+    # calls), so the check neither held nor broke. It carries `pass: true` so it
+    # cannot fail a case it never examined, and the report prints it as skipped —
+    # the same discipline `requires` already applies to a whole case: reported,
+    # never a silent pass.
+    Check = Struct.new(:name, :pass, :detail, :skipped, keyword_init: true) do
+      def skipped? = skipped == true
+    end
 
     # The verdict for one golden case. `judge` (a Judge::Verdict) is attached AFTER
     # the deterministic pass when the case has a rubric and a judge is configured;
@@ -48,6 +79,7 @@ module Insika
       def skipped? = !skipped.nil?
       def pass? = !skipped? && error.nil? && checks.all?(&:pass) && (judge.nil? || judge.pass)
       def failures = checks.reject(&:pass)
+      def skipped_checks = checks.select(&:skipped?)
       # Has a rubric to score but no verdict yet (judge disabled / not run). A skipped
       # case is not pending anything — nobody is going to judge a turn that never ran.
       def judge_pending? = !skipped? && !rubric.to_s.strip.empty? && judge.nil?
@@ -79,6 +111,11 @@ module Insika
       # That is also the honest scope — a question nobody received is not a question.
       POLICIES = {
         # "UMA PERGUNTA POR VEZ" — the rule Insika broke twice under a 28 KB prompt.
+        # `max:` is the store's, default 1. Measured across six harnesses on one
+        # greeting task: with max 1 every reply that says "tudo bem?" fails and every
+        # reply that does not passes, which grades a courtesy formula rather than the
+        # rule. A store whose customers are greeted sets 2 and keeps the check for
+        # what it is for — the form ("qual seu nome, CPF e número do pedido?").
         "ask_once" => "at most one question per reply",
         # Establish the objective before acting on a vague opener.
         "investigate_first" => "asks before calling a tool, on the first turn",
@@ -135,7 +172,16 @@ module Insika
       # all of them — "one question per reply" is a rule about every reply, and the
       # violation that motivated this was on the FIRST turn. Defaults to the single
       # result so existing callers keep working.
-      def evaluate(golden, result, turns: nil)
+      # `store` is the post-turn snapshot ({ collection => [rows] }) when the case
+      # declares a `store_state:` and the runner had a reader; nil otherwise.
+      #
+      # `tools_reported` is false when the TRANSPORT cannot see tool calls at all —
+      # a benched harness that answers text and nothing else. Its tool-shaped graders
+      # are then skipped with the reason instead of passing (they would: an empty
+      # list satisfies `never_calls`) or failing (it would: nothing satisfies
+      # `tools_called`). Both readings would be a claim about a harness nobody
+      # measured.
+      def evaluate(golden, result, turns: nil, store: nil, store_error: nil, tools_reported: true)
         if result.error
           return CaseResult.new(id: golden.id, agent: golden.agent, error: result.error, rubric: nil, judge: nil,
                                 checks: [Check.new(name: "turn", pass: false, detail: "turn error: #{result.error}")])
@@ -144,12 +190,101 @@ module Insika
         # NOT `Array(turns)`: TurnResult is a Struct, so Array() would explode a single
         # one into its members and hand the policy checks three strings.
         conversation = turns.nil? || turns.empty? ? [result] : turns
-        checks = tool_checks(golden, result) + must_not_checks(golden, result) +
-                 policy_checks(golden, conversation)
+        checks = tool_checks(golden, result) + turn_tool_checks(golden, conversation) + call_checks(golden, result) +
+                 reply_checks(golden, result) + ui_checks(golden, result) +
+                 must_not_checks(golden, result, conversation) + policy_checks(golden, conversation)
+        checks = skip_tool_shaped(checks) unless tools_reported
+        checks += store_state_checks(golden, store, error: store_error) if golden.store_state?
         CaseResult.new(id: golden.id, agent: golden.agent, error: nil, checks: checks,
                        rubric: golden.rubric, judge: nil)
       end
 
+      # Graders that read the turn's tool calls. Named by prefix in ONE place: a
+      # grader added without a line here would be silently mis-skipped, and the
+      # policies are in the list because `investigate_first`/`act_fast` are questions
+      # about whether a tool was called, not about the text.
+      TOOL_SHAPED = /\A(?:tool:|never_calls:|calls_one_of\z|first_tool:|max_tool_calls:|blocked_gates:|
+                        must_not:(?:tool_error|phantom_action)\z|policy:(?:investigate_first|act_fast)\z)/x
+
+      def skip_tool_shaped(checks)
+        checks.map do |c|
+          next c unless c.name.match?(TOOL_SHAPED)
+
+          Check.new(name: c.name, pass: true, skipped: true,
+                    detail: "skipped: this harness does not report tool calls")
+        end
+      end
+
+      # WHAT THE STORE LOOKS LIKE AFTER THE TURN. A confident answer with no order
+      # is a fail, and no reply grader can say so. `snapshot` is
+      # { collection => [rows] } as the reader returned it; nil means the reader
+      # gave nothing back, which fails rather than passes — an unread store must
+      # never read as a satisfied one.
+      def store_state_checks(golden, snapshot, error: nil)
+        if snapshot.nil?
+          return [Check.new(name: "store_state", pass: false,
+                            detail: "no store snapshot was read#{" — #{error}" if error}")]
+        end
+
+        rows = Insika::Coercion.deep_stringify(snapshot)
+        spec = golden.store_state
+        count_checks(spec["count"], rows) + record_checks(spec["records"], rows) +
+          absent_checks(spec["absent"], rows)
+      end
+
+      def count_checks(spec, rows)
+        Array(spec).map do |collection, expected|
+          actual = Array(rows[collection.to_s]).size
+          Check.new(name: "store_state:count:#{collection}", pass: actual == expected,
+                    detail: "expected #{expected}, saw #{actual}")
+        end
+      end
+
+      def record_checks(spec, rows)
+        Array(spec).flat_map do |collection, expected_rows|
+          present = Array(rows[collection.to_s])
+          expected_rows.each_with_index.map do |expected, i|
+            hit = present.any? { |row| record_match?(expected, row) }
+            Check.new(name: "store_state:#{collection}[#{i}]", pass: hit,
+                      detail: hit ? "matched" : "no row of #{present.size} in #{collection} matches #{expected.inspect}")
+          end
+        end
+      end
+
+      def absent_checks(spec, rows)
+        Array(spec).flat_map do |collection, expected_rows|
+          present = Array(rows[collection.to_s])
+          expected_rows.each_with_index.map do |expected, i|
+            hits = present.count { |row| record_match?(expected, row) }
+            Check.new(name: "store_state:absent:#{collection}[#{i}]", pass: hits.zero?,
+                      detail: hits.zero? ? "absent" : "#{hits} row(s) in #{collection} match #{expected.inspect}")
+          end
+        end
+      end
+
+      # A row matches when every field the case NAMES matches — a store row carries
+      # ids, timestamps and totals nobody wrote the case about, and demanding the
+      # whole row would make every task un-authorable.
+      def record_match?(expected, row)
+        row.is_a?(Hash) && expected.all? { |k, v| value_match?(v, row[k.to_s]) }
+      end
+
+      # Numbers compare as numbers (189.9 == 189.90, and a store that answers a
+      # string "189.90" is still the same money); a list means "each of these is in
+      # there", not "these are all of them" — `count:` is where exactness is stated.
+      def value_match?(expected, actual)
+        case expected
+        when Hash then record_match?(expected, actual)
+        when Array then actual.is_a?(Array) && expected.all? { |e| actual.any? { |a| value_match?(e, a) } }
+        when Numeric then numeric(actual) && (expected.to_f - actual.to_f).abs < 1e-6
+        else expected.to_s == actual.to_s
+        end
+      end
+
+      def numeric(value)
+        value.is_a?(Numeric) || (value.is_a?(String) && !Float(value, exception: false).nil?)
+      end
+
       # Each REQUIRED expected tool must appear in the turn's tool calls. Optional
       # ("name?") tools are informational — present or not, they never fail.
       def tool_checks(golden, result)
@@ -163,14 +298,119 @@ module Insika
         end
       end
 
-      # `must_not` detectors. "tool_error" is special (inspects statuses); the rest
-      # are content detectors over the output text.
-      def must_not_checks(golden, result)
-        golden.must_not.map do |name|
+      # The graders over the turn's CALLS. Each is its own Check, so the report names
+      # the one that failed instead of "tool checks failed". All read the LAST turn,
+      # like `tools_called`.
+      # The calls a specific turn had to make (`turns[i].tools_called`). The case-level
+      # grader reads the last turn; a two-turn case where turn one claimed an action
+      # and turn two repaired the store passes it, and this is what catches turn one.
+      def turn_tool_checks(golden, turns)
+        turns.each_with_index.flat_map do |turn, i|
+          names = turn.tool_names
+          golden.turn_tools_called(i).reject { |t| t[:optional] }.map do |t|
+            present = names.include?(t[:name])
+            Check.new(name: "tool:#{t[:name]}@turn#{i + 1}", pass: present,
+                      detail: present ? "called on turn #{i + 1}" : "turn #{i + 1} called #{names.empty? ? 'nothing' : names.join(', ')}")
+          end
+        end
+      end
+
+      def call_checks(golden, result)
+        names = result.tool_names
+        checks = golden.never_calls.map do |name|
+          hit = names.include?(name)
+          Check.new(name: "never_calls:#{name}", pass: !hit, detail: hit ? "called #{name}" : "not called")
+        end
+
+        unless golden.calls_one_of.empty?
+          hit = golden.calls_one_of & names
+          checks << Check.new(name: "calls_one_of", pass: !hit.empty?,
+                              detail: hit.empty? ? "none of #{golden.calls_one_of.join(', ')} called (saw: #{names.join(', ')})"
+                                                 : "called #{hit.join(', ')}")
+        end
+
+        if (first = golden.first_tool)
+          checks << Check.new(name: "first_tool:#{first}", pass: names.first == first,
+                              detail: "first call was #{names.first || 'none'}")
+        end
+
+        if (max = golden.max_tool_calls)
+          checks << Check.new(name: "max_tool_calls:#{max}", pass: names.size <= max,
+                              detail: "#{names.size} call(s): #{names.join(', ')}")
+        end
+
+        blocked = result.blocked_tools.map { |t| "#{t['name'] || t[:name]}:#{t['gate'] || t[:gate]}" }
+        checks + golden.blocked_gates.map do |pair|
+          held = blocked.include?(pair)
+          Check.new(name: "blocked_gates:#{pair}", pass: held,
+                    detail: held ? "held" : "not held (blocked: #{blocked.empty? ? 'none' : blocked.join(', ')})")
+        end
+      end
+
+      # Case-insensitive substrings over the PUBLISHED answer. `reply_omits` is where
+      # an internal id, a CPF or a raw tag leaking into the customer's text is pinned —
+      # the negative half of `reply_includes`.
+      def reply_checks(golden, result)
+        text = result.output_text.to_s.downcase
+        golden.reply_includes.map do |s|
+          hit = text.include?(s.downcase)
+          Check.new(name: "reply_includes:#{s}", pass: hit, detail: hit ? "present" : "missing from the reply")
+        end + golden.reply_omits.map do |s|
+          hit = text.include?(s.downcase)
+          Check.new(name: "reply_omits:#{s}", pass: !hit, detail: hit ? "present in the reply" : "absent")
+        end
+      end
+
+      # The graders over what the turn SHOWED. `ui_components` pins that a presentation
+      # tool put at least one card of that component in front of the customer;
+      # `no_ui` is its negative — a turn that must answer in text alone.
+      def ui_checks(golden, result)
+        shown = result.shown_components
+        checks = golden.ui_components.map do |component|
+          hit = shown.include?(component)
+          Check.new(name: "ui_components:#{component}", pass: hit,
+                    detail: hit ? "shown" : "not shown (ui: #{shown.empty? ? 'none' : shown.join(', ')})")
+        end
+        return checks unless golden.no_ui?
+
+        checks << Check.new(name: "no_ui", pass: shown.empty?,
+                            detail: shown.empty? ? "nothing shown" : "shown: #{shown.join(', ')}")
+      end
+
+      # A reply that says it did something the store never saw happen. The case's
+      # `claims:` names each mutation and the words a reply uses to report it — the
+      # store's language, so it lives in the case, not here. Read after the turn,
+      # cumulative over the conversation ("já adicionei" on turn two is true when
+      # turn one did it) — and "ran" means a call that ENDED ok: a claim over a
+      # failed or blocked call is the same lie. Store-level arithmetic and duplicates
+      # are `store_state:`'s business; this is only whether the words match the calls.
+      def phantom_action_checks(golden, turns)
+        claims = golden.claims.transform_values { |pattern| Regexp.new(pattern, Regexp::IGNORECASE) }
+        ran = []
+        offences = turns.each_with_index.filter_map do |turn, i|
+          ran += Array(turn.tool_calls).select { |t| Assertions.ok_status?(t["status"] || t[:status]) }
+                                       .map { |t| (t["name"] || t[:name]).to_s }
+          claimed = claims.find { |tool, verbs| turn.output_text.to_s.match?(verbs) && !ran.include?(tool) }
+          next unless claimed
+
+          tool = claimed.first
+          how = turn.tool_names.include?(tool) ? "#{tool} did not succeed" : "#{tool} never ran"
+          "turn #{i + 1} claims an action but #{how}: #{turn.output_text.to_s.strip[0, 120].inspect}"
+        end
+        [Check.new(name: "must_not:phantom_action", pass: offences.empty?,
+                   detail: offences.empty? ? "every claimed action ran" : offences.join(" | "))]
+      end
+
+      # `must_not` detectors. "tool_error" and "phantom_action" are special (they read
+      # the calls); the rest are content detectors over the output text.
+      def must_not_checks(golden, result, turns = [result])
+        golden.must_not.flat_map do |name|
           if name == "tool_error"
             bad = result.errored_tools
             Check.new(name: "must_not:tool_error", pass: bad.empty?,
                       detail: bad.empty? ? "no tool errors" : "errored: #{bad.map { |t| t['name'] || t[:name] }.join(', ')}")
+          elsif name == "phantom_action"
+            phantom_action_checks(golden, turns)
           else
             hit = detect(name, result.output_text.to_s)
             Check.new(name: "must_not:#{name}", pass: hit.nil?,
@@ -188,7 +428,7 @@ module Insika
         # Exhaustive on purpose: a policy added to POLICIES without a rule here would
         # otherwise fall into whichever branch was last and check the wrong thing.
         pass, detail = case name
-                       when "ask_once" then ask_once(turns)
+                       when "ask_once" then ask_once(turns, max: golden.policy_options.fetch("max", 1).to_i)
                        when "investigate_first" then investigate_first(turns.first)
                        when "act_fast" then act_fast(turns.first)
                        else raise ArgumentError, "policy #{name.inspect} has no rule"
@@ -196,14 +436,15 @@ module Insika
         [Check.new(name: "policy:#{name}", pass: pass, detail: detail)]
       end
 
-      # Every reply asks at most one question. Reported with the offending turn and
+      # Every reply asks at most `max` questions. Reported with the offending turn and
       # the reply itself — "2 questions" alone sends the reader digging.
-      def ask_once(turns)
-        offender = turns.each_with_index.find { |t, _| count_questions(t.output_text) > 1 }
-        return [true, "at most one question per reply"] unless offender
+      def ask_once(turns, max: 1)
+        max = 1 if max < 1
+        offender = turns.each_with_index.find { |t, _| count_questions(t.output_text) > max }
+        return [true, "at most #{max} question(s) per reply"] unless offender
 
         turn, i = offender
-        [false, "turn #{i + 1} asked #{count_questions(turn.output_text)} questions: " \
+        [false, "turn #{i + 1} asked #{count_questions(turn.output_text)} questions (max #{max}): " \
                 "#{turn.output_text.to_s.strip[0, 160].inspect}"]
       end
 

data/lib/insika/evals/golden.rb

@@ -20,8 +20,16 @@ module Insika
     # single-tenant default, like `save_artifact`'s own binding_tenant) unless the
     # case declares one. `run_persona_eval` uses it to keep a QA agent from ever
     # running (or even seeing) another tenant's persona case in the same store.
+    #
+    # `state`: the snapshot the conversation starts from — evidence ids, memory
+    # facts and notes, prior history, briefing fields — loaded into the session
+    # BEFORE turn 1. It is how a case tests "the customer already saw three products
+    # and says 'add the second one'" without replaying the search turn: no dependence
+    # on the model's first answer, one turn cheaper, and a messy state (a
+    # contradiction from six turns ago) becomes reproducible. {} = the case starts
+    # empty, as every case did before the key existed.
     Golden = Struct.new(:id, :agent, :turns, :expect, :requires, :reference, :source, :persona, :tenant,
-                        keyword_init: true) do
+                        :state, :store_state, keyword_init: true) do
       # The user messages to replay, in order. Empty for a persona case: a generated
       # conversation has no scripted turns.
       def user_turns = turns.map { |t| t["user"] }
@@ -41,12 +49,77 @@ module Insika
         end
       end
 
+      # The tools turn `i` (0-based) must call, same `name?` optional marker as above.
+      # [] for a turn that pins nothing.
+      def turn_tools_called(i)
+        Array(turns.dig(i, "tools_called")).map do |ref|
+          s = ref.to_s
+          optional = s.end_with?("?")
+          { name: optional ? s[0..-2] : s, optional: optional }
+        end
+      end
+
       # Names of the negative assertions to run (e.g. "pii_leak", "tool_error").
       def must_not = Array(expect["must_not"]).map(&:to_s)
 
+      # `claims:` — tool name => the words a reply uses to say it did that (a regex
+      # source, matched case-insensitively). What `must_not: phantom_action` reads.
+      def claims = (expect["claims"] || {}).transform_keys(&:to_s)
+
+      # The graders over the turn's CALLS and its published REPLY — each optional,
+      # each deterministic. Every positive has a negative: `never_calls` pins what a
+      # correct turn does NOT do, which `tools_called` alone can never say.
+      def never_calls = Array(expect["never_calls"]).map(&:to_s)
+      def calls_one_of = Array(expect["calls_one_of"]).map(&:to_s)
+      def first_tool = GoldenLoader.presence(expect["first_tool"])
+      def max_tool_calls = expect["max_tool_calls"]
+      def reply_includes = Array(expect["reply_includes"]).map(&:to_s)
+      def reply_omits = Array(expect["reply_omits"]).map(&:to_s)
+      # "tool:gate" pairs that must appear among the turn's BLOCKED calls.
+      def blocked_gates = Array(expect["blocked_gates"]).map(&:to_s)
+      # UI components a presentation tool must have shown (`insika.ui` frames with
+      # at least one card), and its negative: `no_ui: true` pins a turn that shows
+      # nothing.
+      def ui_components = Array(expect["ui_components"]).map(&:to_s)
+      def no_ui? = expect["no_ui"] == true
+
+      def state = self[:state] || {}
+      def seeded? = !state.empty?
+
+      # What the STORE must look like after the turn — the half of a commerce case
+      # no reply can prove. `records` are rows that must be there, `count` the exact
+      # size of a collection (a duplicated order fails it), `absent` rows that must
+      # not exist. Read after the last turn by a store reader the runner is given;
+      # {} = the case says nothing about the store, as every case did before.
+      def store_state = self[:store_state] || {}
+      def store_state? = !store_state.empty?
+
+      # The collections this case talks about — what the reader is asked for, and
+      # nothing else: a bench task about orders should not pull the catalogue.
+      def store_collections
+        (Array(store_state["records"]&.keys) + Array(store_state["count"]&.keys) +
+          Array(store_state["absent"]&.keys)).map(&:to_s).uniq
+      end
+
       # How much the agent should ask before acting. nil = the store
       # has no opinion and only the rubric decides.
-      def policy = GoldenLoader.presence(expect["policy"])
+      def policy = policy_pair.first
+
+      # What the store set the policy TO. The engine ships the mechanism (count the
+      # questions); the number is the store's, because "one question per reply" is a
+      # tolerance, not a law — a greeting that says "tudo bem?" spends one on
+      # courtesy in half the languages a store sells in.
+      def policy_options = policy_pair.last
+
+      # `policy: ask_once` and `policy: { ask_once: { max: 2 } }` are the same
+      # declaration with and without a number. -> [name, options]
+      def policy_pair
+        value = expect["policy"]
+        return [GoldenLoader.presence(value), {}] unless value.is_a?(Hash)
+
+        name, options = value.first
+        [GoldenLoader.presence(name), (options || {}).transform_keys(&:to_s)]
+      end
 
       # What the DEPLOYMENT must have for this case to mean anything.
       # Empty = runs everywhere.
@@ -111,6 +184,9 @@ module Insika
         raise InvalidGolden, "#{source}: 'expect' must be a mapping (case '#{id}')" unless expect.is_a?(Hash)
 
         validate_policy!(expect["policy"], id: id, source: source)
+        validate_graders!(expect, id: id, source: source)
+        state = normalize_state(raw["state"], id: id, source: source)
+        store_state = normalize_store_state(raw["store_state"], id: id, source: source)
         requires = raw["requires"] || {}
         unless requires.is_a?(Hash)
           raise InvalidGolden, "#{source}: 'requires' must be a mapping (case '#{id}')"
@@ -121,7 +197,7 @@ module Insika
 
         Golden.new(id: id, agent: agent, turns: turns, expect: expect,
                    requires: requires, reference: reference, source: source, persona: persona,
-                   tenant: tenant)
+                   tenant: tenant, state: state, store_state: store_state)
       end
 
       # `persona:` is the alternative shape to `turns:`: the
@@ -174,11 +250,151 @@ module Insika
         { "role" => role, "text" => text }.merge(origin ? { "origin" => origin } : {})
       end
 
+      STATE_KEYS = %w[evidence memory history briefing].freeze
+
+      # state: the snapshot a case starts from. Absent -> {}. Only the four known
+      # keys, each in its own shape — a typo'd key (`evidences:`) would seed nothing
+      # and the case would go on passing against the wrong precondition. Refused at
+      # LOAD, not at seed time: a case that half-seeds is a hole in the net.
+      def normalize_state(raw, id:, source:)
+        return {} if raw.nil?
+
+        where = "#{source}: state (case '#{id}')"
+        raise InvalidGolden, "#{where} must be a mapping" unless raw.is_a?(Hash)
+
+        unknown = raw.keys.map(&:to_s) - STATE_KEYS
+        unless unknown.empty?
+          raise InvalidGolden, "#{where}: unknown key(s) #{unknown.join(', ')} — known: #{STATE_KEYS.join(', ')}"
+        end
+
+        state = raw.compact
+        mapping_with!(state["evidence"], "ids", Array, "#{where}.evidence")
+        mapping_with!(state["memory"], "facts", Hash, "#{where}.memory")
+        mapping_with!(state["memory"], "notes", Array, "#{where}.memory")
+        mapping_with!(state["briefing"], "fields", Hash, "#{where}.briefing")
+        history = state["history"]
+        unless history.nil? || (history.is_a?(Array) && history.all? { |m| history_message?(m) })
+          raise InvalidGolden, "#{where}.history must be [{ role: user|assistant, content: '…' }]"
+        end
+
+        state
+      end
+
+      STORE_STATE_KEYS = %w[records count absent].freeze
+
+      # store_state: what the store must look like AFTER the turn, graded by code
+      # against a snapshot the runner reads. Three keys, each a mapping keyed by
+      # collection: `records` (rows that must exist), `count` (the exact size of a
+      # collection) and `absent` (rows that must not exist). A closed vocabulary on
+      # purpose — a case that wrote `orders:` at the top level would grade nothing
+      # and pass, which is the failure this key exists to catch.
+      def normalize_store_state(raw, id:, source:)
+        return {} if raw.nil?
+
+        where = "#{source}: store_state (case '#{id}')"
+        raise InvalidGolden, "#{where} must be a mapping" unless raw.is_a?(Hash)
+
+        unknown = raw.keys.map(&:to_s) - STORE_STATE_KEYS
+        unless unknown.empty?
+          raise InvalidGolden, "#{where}: unknown key(s) #{unknown.join(', ')} — " \
+                               "known: #{STORE_STATE_KEYS.join(', ')}"
+        end
+
+        state = raw.compact
+        rows_by_collection!(state["records"], "#{where}.records")
+        rows_by_collection!(state["absent"], "#{where}.absent")
+        counts!(state["count"], "#{where}.count")
+        state
+      end
+
+      def rows_by_collection!(value, where)
+        return if value.nil?
+        raise InvalidGolden, "#{where} must be a mapping of collection -> rows" unless value.is_a?(Hash)
+
+        value.each do |collection, rows|
+          next if rows.is_a?(Array) && !rows.empty? && rows.all?(Hash)
+
+          raise InvalidGolden, "#{where}.#{collection} must be a non-empty list of mappings"
+        end
+      end
+
+      def counts!(value, where)
+        return if value.nil?
+        raise InvalidGolden, "#{where} must be a mapping of collection -> integer" unless value.is_a?(Hash)
+
+        value.each do |collection, n|
+          next if n.is_a?(Integer) && n >= 0
+
+          raise InvalidGolden, "#{where}.#{collection} must be a non-negative integer (got #{n.inspect})"
+        end
+      end
+
+      def mapping_with!(value, key, type, where)
+        return if value.nil?
+        raise InvalidGolden, "#{where} must be a mapping" unless value.is_a?(Hash)
+        return if value[key].nil? || value[key].is_a?(type)
+
+        raise InvalidGolden, "#{where}.#{key} must be #{type == Array ? 'a list' : 'a mapping'}"
+      end
+
+      def history_message?(message)
+        message.is_a?(Hash) && %w[user assistant].include?(message["role"].to_s) &&
+          !presence(message["content"]).nil?
+      end
+
+      # The graders with a shape to get wrong: a non-integer `max_tool_calls` would
+      # compare against nil and pass; a `blocked_gates` entry without its gate would
+      # never match anything and pass. Refused at load, like `policy`.
+      def validate_graders!(expect, id:, source:)
+        max = expect["max_tool_calls"]
+        unless max.nil? || (max.is_a?(Integer) && max >= 0)
+          raise InvalidGolden, "#{source}: max_tool_calls must be a non-negative integer (case '#{id}')"
+        end
+
+        Array(expect["blocked_gates"]).each do |pair|
+          next if pair.to_s.match?(/\A[^:\s]+:[^:\s]+\z/)
+
+          raise InvalidGolden, "#{source}: blocked_gates entries are 'tool:gate' (got #{pair.inspect}, case '#{id}')"
+        end
+
+        validate_claims!(expect, id: id, source: source)
+      end
+
+      # A phantom-action check with no words to look for would pass everything, and a
+      # pattern that does not compile would fail every cell at grading time instead of
+      # the one author at load time.
+      def validate_claims!(expect, id:, source:)
+        claims = expect["claims"]
+        phantom = Array(expect["must_not"]).map(&:to_s).include?("phantom_action")
+        if phantom && !(claims.is_a?(Hash) && !claims.empty?)
+          raise InvalidGolden, "#{source}: must_not: phantom_action needs a non-empty 'claims:' map (case '#{id}')"
+        end
+        return if claims.nil?
+
+        raise InvalidGolden, "#{source}: 'claims' must be a map of tool => pattern (case '#{id}')" unless claims.is_a?(Hash)
+
+        claims.each do |tool, pattern|
+          Regexp.new(pattern.to_s)
+        rescue RegexpError => e
+          raise InvalidGolden, "#{source}: claims.#{tool} is not a valid pattern — #{e.message} (case '#{id}')"
+        end
+      end
+
       # A typo'd policy must not silently mean "no policy" — the case would go on
       # passing while the rule it was written for stopped being checked. The
       # `Assertions` constant is resolved at CALL time (this file loads first, and
       # assertions.rb touches `Safety::Detectors` at load time).
       def validate_policy!(value, id:, source:)
+        if value.is_a?(Hash)
+          raise InvalidGolden, "#{source}: policy takes ONE name (case '#{id}')" unless value.size == 1
+
+          name, options = value.first
+          unless options.nil? || options.is_a?(Hash)
+            raise InvalidGolden, "#{source}: policy #{name.inspect} options must be a map (case '#{id}')"
+          end
+
+          value = name
+        end
         name = presence(value)
         return if name.nil? || Assertions::POLICIES.key?(name)
 
@@ -188,6 +404,11 @@ module Insika
 
       # turns: a non-empty array of { "user" => String }. Rejects anything else so a
       # typo (e.g. `users:`) surfaces at load time, not as an empty replay.
+      #
+      # A turn may also carry `tools_called: [names]` — the calls THAT turn must make.
+      # The case-level `tools_called` reads the last turn only, and the bench found a
+      # reply that said "adicionado ao carrinho" on turn one without ever calling
+      # add_to_cart, then recovered on turn two so the final store looked right.
       def normalize_turns(turns, id:, source:)
         unless turns.is_a?(Array) && !turns.empty?
           raise InvalidGolden, "#{source}: 'turns' must be a non-empty array (case '#{id}')"
@@ -196,7 +417,12 @@ module Insika
         turns.each_with_index.map do |t, i|
           user = t.is_a?(Hash) ? presence(t["user"]) : nil
           user || (raise InvalidGolden, "#{source}: turns[#{i}] needs a non-empty 'user' (case '#{id}')")
-          { "user" => user }
+          tools = t["tools_called"]
+          unless tools.nil? || (tools.is_a?(Array) && tools.all? { |n| presence(n) })
+            raise InvalidGolden, "#{source}: turns[#{i}].tools_called must be a list of tool names (case '#{id}')"
+          end
+
+          tools.nil? ? { "user" => user } : { "user" => user, "tools_called" => tools.map(&:to_s) }
         end
       end