insika 0.7.0 → 0.9.0

data/lib/insika/doctor.rb

@@ -179,7 +179,9 @@ module Insika
                     check_relay_channel check_web_widget check_skill_eager check_skill_drift check_shadow_parity
                     check_soak_envelope check_turn_timing check_grounding check_cache_layers
                     check_memory_scopes check_funnel_declarations check_followup check_distill
-                    check_harvest check_schedules check_guardrail_corpora]
+                    check_compaction check_harvest check_schedules check_guardrail_corpora
+                    check_tool_allowlist_policy check_fencing check_eval_seeding
+                    check_presentation_tools check_provenance]
 
     def safe(check)
       Array(send(check))
@@ -722,26 +724,59 @@ module Insika
 # the mangled prompt on every turn, and nothing else would ever say so: the file is
 # present, non-empty, and the agent answers — worse than a crash. Found on the pilot
 # by an `insika refine` report, three weeks after the fact.
+#
+# The same sweep also WARNS (never errors) on a file that outgrew a prompt.
+# Merchant packs are LLM-generated (generate-merchant-pack), and generated prose
+# bloats: the pilot's 28 KB AGENTS.md is the local example, and the ETH Zurich
+# instruction-file study puts the cost of that shape at 20%+ extra tokens per
+# turn for no extra instruction-following. The thresholds are deliberately
+# generous — a hand-written file never meets them; only the generated shape does.
 def check_prompt_files
   return [] unless @agent_file_store
 
   agents = @agent_file_store.agents
-  wrapped = agents.flat_map do |agent|
-    @agent_file_store.list(agent).filter_map do |name|
-      next unless wrapped_content?(@agent_file_store.read(agent, name))
-
-      Finding.new(check: "prompt-files", severity: :error, fix: nil,
-                  message: "agent '#{agent}' file '#{name}' holds a serialized object, not text — " \
-                           "the model receives `{\"content\" => …}` on one line, escapes and all. " \
-                           "Recover the markdown from inside the wrapper and write it back.")
+  wrapped = []
+  oversized = []
+  agents.each do |agent|
+    @agent_file_store.list(agent).each do |name|
+      content = @agent_file_store.read(agent, name)
+      if wrapped_content?(content)
+        wrapped << Finding.new(check: "prompt-files", severity: :error, fix: nil,
+                               message: "agent '#{agent}' file '#{name}' holds a serialized object, not text — " \
+                                        "the model receives `{\"content\" => …}` on one line, escapes and all. " \
+                                        "Recover the markdown from inside the wrapper and write it back.")
+      elsif (finding = oversized_prompt(agent, name, content))
+        oversized << finding
+      end
     end
   end
-  return wrapped if wrapped.any?
+  return wrapped + oversized if wrapped.any? || oversized.any?
 
   total = agents.sum { |a| @agent_file_store.list(a).length }
   [ok("prompt-files", "#{total} prompt file(s) across #{agents.length} agent(s): all text")]
 end
 
+# WARN thresholds for one prompt file. ~6 000 estimated tokens (~24 KB) or 600
+# lines: the pilot's generated 28 KB / 292-line AGENTS.md trips the token bar,
+# every hand-written demo file stays far under both. Estimate = chars/4, the
+# same yardstick as Insika::TokenEstimator — cheap and honest about being ±15%.
+PROMPT_FILE_WARN_TOKENS = 6_000
+PROMPT_FILE_WARN_LINES = 600
+
+def oversized_prompt(agent, name, content)
+  text = content.to_s
+  tokens = Insika::TokenEstimator.estimate(text)
+  lines = text.lines.count
+  return nil if tokens <= PROMPT_FILE_WARN_TOKENS && lines <= PROMPT_FILE_WARN_LINES
+
+  Finding.new(check: "prompt-files", severity: :warn, fix: nil,
+              message: "agent '#{agent}' file '#{name}' is ~#{tokens} tokens over #{lines} line(s) " \
+                       "(threshold: #{PROMPT_FILE_WARN_TOKENS} tokens / #{PROMPT_FILE_WARN_LINES} lines) — " \
+                       "a prompt this large costs 20%+ more tokens on every turn for no better " \
+                       "instruction-following. Trim it, or split the reference material into skills " \
+                       "the agent loads on demand.")
+end
+
 # Cheap and specific: Ruby's inspect of a Hash whose first key is a string. A real
 # prompt does not open with `{"…" =>`.
 def wrapped_content?(content) = /\A\s*\{\s*"[^"]+"\s*=>/.match?(content.to_s)
@@ -802,6 +837,7 @@ def wrapped_content?(content) = /\A\s*\{\s*"[^"]+"\s*=>/.match?(content.to_s)
     # identity block bills a cache write every turn).
     IDENTITY_BUILTINS = %w[
       Insika::Context::Providers::Prompt
+      Insika::Context::Providers::FenceNotice
       Insika::Context::Providers::Skill
       Insika::Context::Providers::ToolSearch
     ].freeze
@@ -868,6 +904,111 @@ def wrapped_content?(content) = /\A\s*\{\s*"[^"]+"\s*=>/.match?(content.to_s)
     # harmless but useless — every claim passes and the audit reads zero. A
     # warning, never an error: the pack owns matcher quality; the engine refuses
     # only uncompileable data.
+    # A stored agent that declares `tools_allow` / `tools_deny` /
+    # `tools_allow_groups` but does not name the `tool_allowlist` policy. Only
+    # that policy applies the lists, and the Policy::Engine runs ONLY the
+    # policies a profile names — so as written on disk the lists are inert and
+    # the model receives every registered tool (hundreds of schemas, tens of
+    # thousands of tokens, per request).
+    #
+    # AgentProfile.build now adds the policy whenever a list is declared, so a
+    # live turn off this record is already safe. The record itself stays wrong
+    # until something re-saves it, and anything that reads the stored shape
+    # directly — an older engine, a pack export, an operator auditing the
+    # Studio form — still sees an allowlist that does nothing. That is why this
+    # reads the RAW record and not `all`.
+    def check_tool_allowlist_policy
+      return [] unless @profile_source.respond_to?(:all_raw)
+
+      records = @profile_source.all_raw
+      bad = records.select { |r| tool_lists_declared?(r) && !names_tool_allowlist?(r) }
+      return [ok("tool-allowlist", "#{records.length} stored agent(s): every declared tool list names the policy")] if bad.empty?
+
+      bad.map do |r|
+        Finding.new(check: "tool-allowlist", severity: :error, fix: nil,
+                    message: "agent '#{r["id"]}' declares a tool allow/deny list but its policies " \
+                             "(#{Array(r["policies"]).join(", ").then { |p| p.empty? ? "none" : p }}) " \
+                             "do not include tool_allowlist — as stored, the list is never applied and " \
+                             "the model receives every registered tool. The engine repairs this when it " \
+                             "loads the agent; re-save it (Studio, or the pack) so the stored policies " \
+                             "match the intent.")
+      end
+    end
+
+    # Presence, not emptiness, for the two nil-able lists: `tools_allow: []`
+    # means "no tools" and is as much a declaration as a list of names.
+    # `tools_deny` has no nil state, so only a non-empty one counts.
+    def tool_lists_declared?(raw)
+      !raw["tools_allow"].nil? || !raw["tools_allow_groups"].nil? || !Array(raw["tools_deny"]).empty?
+    end
+
+    def names_tool_allowlist?(raw) = Array(raw["policies"]).any? { |p| p.to_s == "tool_allowlist" }
+
+    # A stored agent that allows a PRESENTATION tool but no tool declaring
+    # `evidence`. The presentation tool shows only cards an evidence tool returned
+    # this session, so with no evidence tool in reach every call drops every id —
+    # the model keeps asking to show products and nothing ever appears. Silent
+    # unless a presentation tool exists. Data tools only: a code tool's evidence
+    # lives in registry metadata the doctor does not read.
+    def check_presentation_tools
+      return [] unless @tool_store && @profile_source
+
+      raws = @tool_store.all_raw
+      presenting = raws.select { |r| r["presentation"] }.map { |r| r["name"].to_s }
+      return [] if presenting.empty?
+
+      evidence = raws.select { |r| r["evidence"] }.map { |r| r["name"].to_s }
+      bad = @profile_source.all.filter_map do |profile|
+        allowed = allowed_data_tools(profile, raws)
+        shows = presenting & allowed
+        next if shows.empty? || !(evidence & allowed).empty?
+
+        Finding.new(check: "presentation-tools", severity: :warn, fix: nil,
+                    message: "agent '#{profile.id}' allows #{shows.join(', ')} but no tool declaring " \
+                             "evidence — a presentation tool can only show cards an evidence tool " \
+                             "returned this session, so every call would drop every id. Allow the " \
+                             "search tool that declares `evidence`, or drop the presentation tool.")
+      end
+      return bad unless bad.empty?
+
+      [ok("presentation-tools", "#{presenting.length} presentation tool(s): every agent that shows cards " \
+                                "also allows an evidence tool")]
+    end
+
+    # The data tools a profile may call, by the allowlist's own rules: nil lists =
+    # all; otherwise names ∪ groups; deny wins.
+    def allowed_data_tools(profile, raws)
+      names = profile.tools_allow.nil? ? nil : Array(profile.tools_allow).map(&:to_s)
+      groups = profile.tools_allow_groups.nil? ? nil : Array(profile.tools_allow_groups).map(&:to_s)
+      allowed = if names.nil? && groups.nil?
+                  raws.map { |r| r["name"].to_s }
+                else
+                  Array(names) | raws.select { |r| Array(groups).include?(r["group"].to_s) }.map { |r| r["name"].to_s }
+                end
+      allowed - Array(profile.tools_deny).map(&:to_s)
+    end
+
+    def check_provenance
+      return [] unless @tool_store && @profile_source.respond_to?(:all_raw)
+
+      tools = @tool_store.all_raw
+      @profile_source.all_raw.filter_map do |raw|
+        allowed = tools.select do |tool|
+          name = tool["name"]
+          unrestricted = raw["tools_allow"].nil? && raw["tools_allow_groups"].nil?
+          (unrestricted || Array(raw["tools_allow"]).include?(name) ||
+            Array(raw["tools_allow_groups"]).include?(tool["group"])) &&
+            !Array(raw["tools_deny"]).include?(name)
+        end
+        gated = allowed.select { |tool| tool["requires_evidence"] }
+        next if gated.empty? || allowed.any? { |tool| tool["evidence"] }
+
+        Finding.new(check: "provenance", severity: :warn, fix: nil,
+                    message: "agent '#{raw["id"]}' allows #{gated.map { |t| t["name"] }.join(', ')} " \
+                             "with requires_evidence but no stored evidence tool — verify that an allowed code tool supplies evidence")
+      end
+    end
+
     def check_grounding
       return [] unless @profile_source
 
@@ -1230,6 +1371,68 @@ def wrapped_content?(content) = /\A\s*\{\s*"[^"]+"\s*=>/.match?(content.to_s)
         stale: @proposal_store.stale(limit: 10_000).size }
     end
 
+    # the in-session compaction check (RFC-0044): platform-gated,
+    # so ONE line — enabled with no resolvable model (no compaction.model, no
+    # platform utility_model) can never run; the warn is the same "declared
+    # but dead" signal as check_distill's (the engine never guesses a model).
+    def check_compaction
+      return [] unless @settings_store
+
+      settings = @settings_store.get
+      config = settings["compaction"] || {}
+      return [ok("compaction", "in-session compaction off")] unless Coercion.truthy?(config["enabled"])
+
+      if Coercion.presence(config["model"]).nil? && Coercion.presence(settings["utility_model"]).nil?
+        [Finding.new(check: "compaction", severity: :warn, fix: nil,
+                     message: "in-session compaction is enabled but has no model slot — it will " \
+                              "never run (set compaction.model or the platform utility_model).")]
+      else
+        [ok("compaction", "in-session compaction on — keep_last #{config['keep_last']}, " \
+                          "compact_after #{config['compact_after']}")]
+      end
+    end
+
+    # `evals.seeding` opens POST /v1/conversations/:id/seed: a snapshot written
+    # into a real conversation under the tenant token, before its first turn. Right
+    # on the machine running snapshot evals; wrong left on in production, where any
+    # consumer holding the token could write a customer's history. A warning, not
+    # an error — the operator may be running the evals right now.
+    def check_eval_seeding
+      return [] unless @settings_store
+
+      on = ((@settings_store.get["evals"] || {})["seeding"]) == true
+      return [ok("eval-seeding", "evals.seeding off — conversations cannot be seeded")] unless on
+
+      [Finding.new(check: "eval-seeding", severity: :warn, fix: nil,
+                   message: "evals.seeding is ON — POST /v1/conversations/:id/seed accepts a fabricated " \
+                            "conversation state under the tenant token. Fine while running snapshot " \
+                            "evals; turn it off (Studio > Settings, or `update_settings`) in production.")]
+    end
+
+    # A customer-facing agent — one reachable through an inbound channel: any
+    # agent when the relay is mounted (the event names the agent), the listed
+    # ones for the widget — with `fencing` off reads third-party bytes as-is.
+    # A warning, never an error: the default is off this release on purpose
+    # (the goldens were baselined unfenced), so the message says exactly what
+    # is unsanitized today. No inbound channel = nothing customer-facing = silent.
+    def check_fencing
+      return [] unless @profile_source
+
+      relay = Insika::Coercion.present?(@env["INSIKA_RELAY_TOKEN"])
+      widget = @env["INSIKA_WIDGET_AGENTS"].to_s.split(",").map(&:strip).reject(&:empty?)
+      return [] unless relay || widget.any?
+
+      exposed = @profile_source.all.select { |p| relay || widget.include?(p.id.to_s) }
+      unfenced = exposed.reject { |p| Insika::Fence.enabled?(p) }.map(&:id)
+      return [ok("fencing", "fencing on for every agent behind an inbound channel")] if unfenced.empty?
+
+      [Finding.new(check: "fencing", severity: :warn, fix: nil,
+                   message: "agent(s) #{unfenced.join(', ')}: reachable through an inbound channel with " \
+                            "`fencing` off — tool results, <memory> facts and <knowledge> concepts reach " \
+                            "the model byte-for-byte (invisible characters, forged turn markers and " \
+                            "transcript-shaped tags included). Set `fencing true` on the agent.")]
+    end
+
     # the harvest check — per profile WITH a harvest hash:
     #   declared-without-model warn (D12), no grounding matcher warn (D3),
     #   malformed negative list error (D4), else ok with the pending counts.

data/lib/insika/dsl/runtime.rb

@@ -231,11 +231,16 @@ module Insika
             base: "", catalog: c[:prompt_catalog],
             agent_files: c[:agent_file_store], system_files: c[:system_file_store]
           ),
+          Insika::Context::Providers::FenceNotice.new,
           Insika::Context::Providers::Skill.new(catalog: c[:skill_catalog]),
           Insika::Context::Providers::SkillTrigger.new(catalog: c[:skill_catalog]),
           Insika::Context::Providers::ToolSearch.new(catalog: c[:tool_catalog]),
           Insika::Context::Providers::Memory.new(store: spine.memory_store),
           Insika::Context::Providers::Knowledge.new(store: spine.knowledge_store),
+          # Session briefing: read path. Inert for agents without briefing_fields.
+          # Before Session, so the durable head renders above the transcript and
+          # the tail recitation lands after it.
+          Insika::Context::Providers::Briefing.new(session_store: spine.session_store),
           Insika::Context::Providers::Session.new(session_store: spine.session_store)
         ] # NOT frozen: load_plugins appends plugin providers at boot
       end

data/lib/insika/dsl.rb

@@ -417,6 +417,12 @@ module Insika
       # SEES: repeated identical tool results collapse to a back-reference.
       def tool_output_compression(on = true) = @config[:tool_output_compression] = on
 
+      # Third-party text is sanitized before the model reads it (tool results,
+      # memory, knowledge) and a one-sentence notice names those blocks as data.
+      # Opt-in this release — the goldens were baselined on unfenced bytes:
+      #   fencing true
+      def fencing(on = true) = @config[:fencing] = on
+
       # Content-safety guardrails — opt-in and configurable per agent.
       # Pure config-over-code: the hash is stored on the profile and consumed by
       # Safety::Config.from_profile. Merges, so repeated calls accumulate.

data/lib/insika/edge_limiter.rb

@@ -263,12 +263,15 @@ module Insika
 
     # Appends the note to the assembled system prompt: the real Data package is
     # immutable (with), the specs' minimal Struct is mutable — both duck-typed.
+    # The note is per-turn data, so it also lands in the volatile layer — below
+    # the cache breakpoint, never inside the cached identity prefix.
     def inject_budget_note(state, note)
       ctx = state.context
       return if ctx.nil?
 
       if ctx.respond_to?(:with)
-        state.context = ctx.with(system: "#{ctx.system}\n\n#{note}")
+        volatile = [ctx.system_volatile, note].reject { |s| s.to_s.empty? }.join("\n\n")
+        state.context = ctx.with(system: "#{ctx.system}\n\n#{note}", system_volatile: volatile)
       elsif ctx.respond_to?(:system=)
         ctx.system = "#{ctx.system}\n\n#{note}"
       end

data/lib/insika/env_schema.rb

@@ -72,10 +72,10 @@ module Insika
 
     # Prefixes the engine fully OWNS: an unknown key under one of these is a typo, not
     # a foreign var. INSIKA_ (current) and HARNESS_ (legacy, still honored during the
-    # deprecation window). Deliberately NOT OPENCLAW_ (shared with the OpenClaw gateway
-    # product, which sets its own OPENCLAW_HOME/_STATE_DIR/… — the engine merely borrows
-    # 3 names for interop), nor LITESTREAM_ (the sidecar owns it), nor OTEL_ (the
-    # OpenTelemetry SDK owns its env).
+    # deprecation window). Deliberately NOT OPENCLAW_ (the OpenClaw gateway product
+    # sets its own OPENCLAW_HOME/_STATE_DIR/… on the same host; the engine reads none
+    # of them), nor LITESTREAM_ (the sidecar owns it), nor OTEL_ (the OpenTelemetry
+    # SDK owns its env).
     OWNED_PREFIXES = [PREFIX, LEGACY_PREFIX].freeze
 
     BOOLEANS = %w[1 0 true false yes no on off].freeze
@@ -142,8 +142,7 @@ module Insika
       spec(name: "INSIKA_ROUTER_BACKEND_TIMEOUT", type: :integer, description: "`insika-router`'s connect/read timeout to a backend, in seconds (default 10)."),
       spec(name: "INSIKA_ROUTER_HOST", description: "bind address for `insika-router` itself (default 0.0.0.0)."),
       spec(name: "INSIKA_ROUTER_PORT", type: :integer, description: "listen port for `insika-router` itself (default 9090)."),
-      spec(name: "OPENCLAW_GATEWAY_TOKEN", secret: true, description: "Bearer for /v1 + /a2a (falls back to ADMIN_TOKEN)."),
-      spec(name: "OPENCLAW_AGENTS_DIR", type: :path, description: "Directory of OpenClaw-style agent packs."),
+      spec(name: "INSIKA_GATEWAY_TOKEN", secret: true, description: "Bearer for /v1 + /a2a (falls back to ADMIN_TOKEN when unset)."),
       spec(name: "INSIKA_PLUGIN_DIR", type: :path, description: "Workspace plugin root (directories with insika.plugin.yml). Loaded at boot; ids still need INSIKA_PLUGINS."),
       spec(name: "INSIKA_PLUGINS", type: :csv, description: "Plugin ids to enable from the workspace/bundled roots. Announced gems are enabled by installing them."),
       spec(name: "INSIKA_PLUGINS_DISABLED", type: :csv, description: "Plugin ids that never load — the absolute veto, wins over INSIKA_PLUGINS and over an announced gem."),

data/lib/insika/errors.rb

@@ -8,6 +8,7 @@ module Insika
 
   class ValidationError < Error; end  # Malformed Command -> HTTP 422, no Task created
   class NotFoundError   < Error; end  # nonexistent session/task/agent -> HTTP 404
+  class ConflictError   < Error; end  # the write contradicts state that already exists -> HTTP 409
 
   # Policy Engine denied -> :policy_denied event, task :failed
   class PolicyDenied < Error

data/lib/insika/evals/assertions.rb

@@ -13,14 +13,35 @@ module Insika
     # offline without a server.
     #
     #   output_text: the final assistant text (for content checks)
-    #   tool_calls:  [{ "name" =>, "status" => }] captured from the stream's tool events
+    #   tool_calls:  [{ "name" =>, "arguments" =>, "status" =>, "gate" => }] captured
+    #                from the stream's tool events. `arguments` (a Hash) and
+    #                `status` ("ok" | "error" | "blocked", with `gate` naming what
+    #                held a blocked call) arrive from the item frames; an older
+    #                deployment sends names only and they stay nil.
+    #   ui:          [{ "component" =>, "count" => }] — one per presentation call
+    #                (the `insika.ui` frames); nil/[] when the turn showed nothing
     #   error:       transport/turn error string, or nil on a clean turn
-    TurnResult = Struct.new(:output_text, :tool_calls, :error, keyword_init: true) do
+    TurnResult = Struct.new(:output_text, :tool_calls, :ui, :error, keyword_init: true) do
       def tool_names = Array(tool_calls).map { |t| (t["name"] || t[:name]).to_s }
 
-      # A tool call whose status is anything but a success ("ok"/2xx/"success").
+      # Components that actually put a card in front of the customer (count > 0).
+      def shown_components
+        Array(ui).select { |u| (u["count"] || u[:count]).to_i.positive? }
+                 .map { |u| (u["component"] || u[:component]).to_s }
+      end
+
+      # A tool call whose status is anything but a success ("ok"/2xx/"success"). A
+      # BLOCKED call is not an error: the tool never ran, a gate held it — that is
+      # the `blocked_gates` grader's business, not `must_not: tool_error`'s.
       def errored_tools
-        Array(tool_calls).reject { |t| Assertions.ok_status?(t["status"] || t[:status]) }
+        Array(tool_calls).reject do |t|
+          status = t["status"] || t[:status]
+          Assertions.ok_status?(status) || status.to_s == "blocked"
+        end
+      end
+
+      def blocked_tools
+        Array(tool_calls).select { |t| (t["status"] || t[:status]).to_s == "blocked" }
       end
     end
 
@@ -144,8 +165,8 @@ module Insika
         # NOT `Array(turns)`: TurnResult is a Struct, so Array() would explode a single
         # one into its members and hand the policy checks three strings.
         conversation = turns.nil? || turns.empty? ? [result] : turns
-        checks = tool_checks(golden, result) + must_not_checks(golden, result) +
-                 policy_checks(golden, conversation)
+        checks = tool_checks(golden, result) + call_checks(golden, result) + reply_checks(golden, result) +
+                 ui_checks(golden, result) + must_not_checks(golden, result) + policy_checks(golden, conversation)
         CaseResult.new(id: golden.id, agent: golden.agent, error: nil, checks: checks,
                        rubric: golden.rubric, judge: nil)
       end
@@ -163,6 +184,71 @@ module Insika
         end
       end
 
+      # The graders over the turn's CALLS. Each is its own Check, so the report names
+      # the one that failed instead of "tool checks failed". All read the LAST turn,
+      # like `tools_called`.
+      def call_checks(golden, result)
+        names = result.tool_names
+        checks = golden.never_calls.map do |name|
+          hit = names.include?(name)
+          Check.new(name: "never_calls:#{name}", pass: !hit, detail: hit ? "called #{name}" : "not called")
+        end
+
+        unless golden.calls_one_of.empty?
+          hit = golden.calls_one_of & names
+          checks << Check.new(name: "calls_one_of", pass: !hit.empty?,
+                              detail: hit.empty? ? "none of #{golden.calls_one_of.join(', ')} called (saw: #{names.join(', ')})"
+                                                 : "called #{hit.join(', ')}")
+        end
+
+        if (first = golden.first_tool)
+          checks << Check.new(name: "first_tool:#{first}", pass: names.first == first,
+                              detail: "first call was #{names.first || 'none'}")
+        end
+
+        if (max = golden.max_tool_calls)
+          checks << Check.new(name: "max_tool_calls:#{max}", pass: names.size <= max,
+                              detail: "#{names.size} call(s): #{names.join(', ')}")
+        end
+
+        blocked = result.blocked_tools.map { |t| "#{t['name'] || t[:name]}:#{t['gate'] || t[:gate]}" }
+        checks + golden.blocked_gates.map do |pair|
+          held = blocked.include?(pair)
+          Check.new(name: "blocked_gates:#{pair}", pass: held,
+                    detail: held ? "held" : "not held (blocked: #{blocked.empty? ? 'none' : blocked.join(', ')})")
+        end
+      end
+
+      # Case-insensitive substrings over the PUBLISHED answer. `reply_omits` is where
+      # an internal id, a CPF or a raw tag leaking into the customer's text is pinned —
+      # the negative half of `reply_includes`.
+      def reply_checks(golden, result)
+        text = result.output_text.to_s.downcase
+        golden.reply_includes.map do |s|
+          hit = text.include?(s.downcase)
+          Check.new(name: "reply_includes:#{s}", pass: hit, detail: hit ? "present" : "missing from the reply")
+        end + golden.reply_omits.map do |s|
+          hit = text.include?(s.downcase)
+          Check.new(name: "reply_omits:#{s}", pass: !hit, detail: hit ? "present in the reply" : "absent")
+        end
+      end
+
+      # The graders over what the turn SHOWED. `ui_components` pins that a presentation
+      # tool put at least one card of that component in front of the customer;
+      # `no_ui` is its negative — a turn that must answer in text alone.
+      def ui_checks(golden, result)
+        shown = result.shown_components
+        checks = golden.ui_components.map do |component|
+          hit = shown.include?(component)
+          Check.new(name: "ui_components:#{component}", pass: hit,
+                    detail: hit ? "shown" : "not shown (ui: #{shown.empty? ? 'none' : shown.join(', ')})")
+        end
+        return checks unless golden.no_ui?
+
+        checks << Check.new(name: "no_ui", pass: shown.empty?,
+                            detail: shown.empty? ? "nothing shown" : "shown: #{shown.join(', ')}")
+      end
+
       # `must_not` detectors. "tool_error" is special (inspects statuses); the rest
       # are content detectors over the output text.
       def must_not_checks(golden, result)

data/lib/insika/evals/golden.rb

@@ -20,8 +20,16 @@ module Insika
     # single-tenant default, like `save_artifact`'s own binding_tenant) unless the
     # case declares one. `run_persona_eval` uses it to keep a QA agent from ever
     # running (or even seeing) another tenant's persona case in the same store.
+    #
+    # `state`: the snapshot the conversation starts from — evidence ids, memory
+    # facts and notes, prior history, briefing fields — loaded into the session
+    # BEFORE turn 1. It is how a case tests "the customer already saw three products
+    # and says 'add the second one'" without replaying the search turn: no dependence
+    # on the model's first answer, one turn cheaper, and a messy state (a
+    # contradiction from six turns ago) becomes reproducible. {} = the case starts
+    # empty, as every case did before the key existed.
     Golden = Struct.new(:id, :agent, :turns, :expect, :requires, :reference, :source, :persona, :tenant,
-                        keyword_init: true) do
+                        :state, keyword_init: true) do
       # The user messages to replay, in order. Empty for a persona case: a generated
       # conversation has no scripted turns.
       def user_turns = turns.map { |t| t["user"] }
@@ -44,6 +52,26 @@ module Insika
       # Names of the negative assertions to run (e.g. "pii_leak", "tool_error").
       def must_not = Array(expect["must_not"]).map(&:to_s)
 
+      # The graders over the turn's CALLS and its published REPLY — each optional,
+      # each deterministic. Every positive has a negative: `never_calls` pins what a
+      # correct turn does NOT do, which `tools_called` alone can never say.
+      def never_calls = Array(expect["never_calls"]).map(&:to_s)
+      def calls_one_of = Array(expect["calls_one_of"]).map(&:to_s)
+      def first_tool = GoldenLoader.presence(expect["first_tool"])
+      def max_tool_calls = expect["max_tool_calls"]
+      def reply_includes = Array(expect["reply_includes"]).map(&:to_s)
+      def reply_omits = Array(expect["reply_omits"]).map(&:to_s)
+      # "tool:gate" pairs that must appear among the turn's BLOCKED calls.
+      def blocked_gates = Array(expect["blocked_gates"]).map(&:to_s)
+      # UI components a presentation tool must have shown (`insika.ui` frames with
+      # at least one card), and its negative: `no_ui: true` pins a turn that shows
+      # nothing.
+      def ui_components = Array(expect["ui_components"]).map(&:to_s)
+      def no_ui? = expect["no_ui"] == true
+
+      def state = self[:state] || {}
+      def seeded? = !state.empty?
+
       # How much the agent should ask before acting. nil = the store
       # has no opinion and only the rubric decides.
       def policy = GoldenLoader.presence(expect["policy"])
@@ -111,6 +139,8 @@ module Insika
         raise InvalidGolden, "#{source}: 'expect' must be a mapping (case '#{id}')" unless expect.is_a?(Hash)
 
         validate_policy!(expect["policy"], id: id, source: source)
+        validate_graders!(expect, id: id, source: source)
+        state = normalize_state(raw["state"], id: id, source: source)
         requires = raw["requires"] || {}
         unless requires.is_a?(Hash)
           raise InvalidGolden, "#{source}: 'requires' must be a mapping (case '#{id}')"
@@ -121,7 +151,7 @@ module Insika
 
         Golden.new(id: id, agent: agent, turns: turns, expect: expect,
                    requires: requires, reference: reference, source: source, persona: persona,
-                   tenant: tenant)
+                   tenant: tenant, state: state)
       end
 
       # `persona:` is the alternative shape to `turns:`: the
@@ -174,6 +204,65 @@ module Insika
         { "role" => role, "text" => text }.merge(origin ? { "origin" => origin } : {})
       end
 
+      STATE_KEYS = %w[evidence memory history briefing].freeze
+
+      # state: the snapshot a case starts from. Absent -> {}. Only the four known
+      # keys, each in its own shape — a typo'd key (`evidences:`) would seed nothing
+      # and the case would go on passing against the wrong precondition. Refused at
+      # LOAD, not at seed time: a case that half-seeds is a hole in the net.
+      def normalize_state(raw, id:, source:)
+        return {} if raw.nil?
+
+        where = "#{source}: state (case '#{id}')"
+        raise InvalidGolden, "#{where} must be a mapping" unless raw.is_a?(Hash)
+
+        unknown = raw.keys.map(&:to_s) - STATE_KEYS
+        unless unknown.empty?
+          raise InvalidGolden, "#{where}: unknown key(s) #{unknown.join(', ')} — known: #{STATE_KEYS.join(', ')}"
+        end
+
+        state = raw.compact
+        mapping_with!(state["evidence"], "ids", Array, "#{where}.evidence")
+        mapping_with!(state["memory"], "facts", Hash, "#{where}.memory")
+        mapping_with!(state["memory"], "notes", Array, "#{where}.memory")
+        mapping_with!(state["briefing"], "fields", Hash, "#{where}.briefing")
+        history = state["history"]
+        unless history.nil? || (history.is_a?(Array) && history.all? { |m| history_message?(m) })
+          raise InvalidGolden, "#{where}.history must be [{ role: user|assistant, content: '…' }]"
+        end
+
+        state
+      end
+
+      def mapping_with!(value, key, type, where)
+        return if value.nil?
+        raise InvalidGolden, "#{where} must be a mapping" unless value.is_a?(Hash)
+        return if value[key].nil? || value[key].is_a?(type)
+
+        raise InvalidGolden, "#{where}.#{key} must be #{type == Array ? 'a list' : 'a mapping'}"
+      end
+
+      def history_message?(message)
+        message.is_a?(Hash) && %w[user assistant].include?(message["role"].to_s) &&
+          !presence(message["content"]).nil?
+      end
+
+      # The graders with a shape to get wrong: a non-integer `max_tool_calls` would
+      # compare against nil and pass; a `blocked_gates` entry without its gate would
+      # never match anything and pass. Refused at load, like `policy`.
+      def validate_graders!(expect, id:, source:)
+        max = expect["max_tool_calls"]
+        unless max.nil? || (max.is_a?(Integer) && max >= 0)
+          raise InvalidGolden, "#{source}: max_tool_calls must be a non-negative integer (case '#{id}')"
+        end
+
+        Array(expect["blocked_gates"]).each do |pair|
+          next if pair.to_s.match?(/\A[^:\s]+:[^:\s]+\z/)
+
+          raise InvalidGolden, "#{source}: blocked_gates entries are 'tool:gate' (got #{pair.inspect}, case '#{id}')"
+        end
+      end
+
       # A typo'd policy must not silently mean "no policy" — the case would go on
       # passing while the rule it was written for stopped being checked. The
       # `Assertions` constant is resolved at CALL time (this file loads first, and

data/lib/insika/evals/runner.rb

@@ -66,6 +66,26 @@ module Insika
         # consumer needing a real Chat UUID as X-Chat-Id) supplies it via conv_map; otherwise the
         # synthetic "eval-<id>" keeps the adapter's own multi-turn continuation.
         conv = @conv_map[golden.id] || "eval-#{golden.id}"
+        if golden.seeded?
+          # The snapshot goes in BEFORE turn 1 — on the same conversation id the turns
+          # continue. A transport that cannot seed (A2A: a remote agent has no seed
+          # route) SKIPS the case with the reason, never runs it against an empty
+          # state and passes. A deployment that refuses seeding (the setting is off)
+          # is the same outcome: skipped and reported, which is what the `requires`
+          # discipline already promises for anything the deployment lacks.
+          unless @transport.respond_to?(:seed)
+            return RunCase.new(result: Assertions.skip(golden, "transport cannot seed state"), timings: [])
+          end
+
+          begin
+            @transport.seed(conv, golden.state)
+          rescue SeedRefused => e
+            return RunCase.new(result: Assertions.skip(golden, "deployment refuses seeding — #{e.message}"), timings: [])
+          rescue Insika::Error => e
+            failed = TurnResult.new(output_text: "", tool_calls: [], error: "seed failed: #{e.message}")
+            return RunCase.new(result: Assertions.evaluate(golden, failed), timings: [])
+          end
+        end
         turns = []
         timings = []
         spent = []