insika 0.8.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +75 -0
  3. data/README.md +5 -3
  4. data/bin/insika +1 -1
  5. data/docs/AGENTS.md +52 -11
  6. data/docs/API.md +73 -0
  7. data/docs/ARCHITECTURE.md +45 -44
  8. data/docs/CHANNELS.md +19 -2
  9. data/docs/CONTEXT.md +33 -27
  10. data/docs/DEPLOY.md +13 -2
  11. data/docs/EVALS.md +98 -8
  12. data/docs/FACTS.md +4 -0
  13. data/docs/KNOWLEDGE.md +7 -0
  14. data/docs/OBSERVABILITY.md +21 -6
  15. data/docs/POLICY.md +4 -1
  16. data/docs/RELEASING.md +4 -0
  17. data/docs/SECURITY.md +27 -1
  18. data/docs/TOOLS.md +127 -33
  19. data/docs/prompts/ADD-TOOL.md +12 -2
  20. data/docs/prompts/DIAGNOSE-TURN.md +3 -0
  21. data/docs/prompts/GO-LIVE.md +3 -1
  22. data/lib/insika/agent_profile.rb +21 -9
  23. data/lib/insika/channels/web/widget.js +33 -0
  24. data/lib/insika/channels/web.rb +5 -2
  25. data/lib/insika/chat_builder.rb +62 -20
  26. data/lib/insika/commands/agent_payload.rb +1 -1
  27. data/lib/insika/commands/run_distillation.rb +5 -8
  28. data/lib/insika/commands/seed_session.rb +118 -0
  29. data/lib/insika/context/builder.rb +29 -9
  30. data/lib/insika/context/priority.rb +2 -0
  31. data/lib/insika/context/provider.rb +5 -0
  32. data/lib/insika/context/providers/briefing.rb +11 -8
  33. data/lib/insika/context/providers/fence_notice.rb +27 -0
  34. data/lib/insika/context/providers/knowledge.rb +7 -4
  35. data/lib/insika/context/providers/memory.rb +8 -4
  36. data/lib/insika/context/providers/session.rb +7 -3
  37. data/lib/insika/doctor.rb +109 -1
  38. data/lib/insika/dsl/runtime.rb +1 -0
  39. data/lib/insika/dsl.rb +6 -0
  40. data/lib/insika/edge_limiter.rb +4 -1
  41. data/lib/insika/errors.rb +1 -0
  42. data/lib/insika/evals/assertions.rb +92 -6
  43. data/lib/insika/evals/golden.rb +91 -2
  44. data/lib/insika/evals/runner.rb +20 -0
  45. data/lib/insika/evals/simulator.rb +11 -2
  46. data/lib/insika/evals/transport.rb +117 -15
  47. data/lib/insika/evidence.rb +79 -12
  48. data/lib/insika/executor.rb +30 -23
  49. data/lib/insika/fence.rb +96 -0
  50. data/lib/insika/golden_store.rb +3 -0
  51. data/lib/insika/mcp_store.rb +5 -2
  52. data/lib/insika/mcp_tool_registry.rb +8 -1
  53. data/lib/insika/memory_store.rb +12 -0
  54. data/lib/insika/overlay_tool_registry.rb +5 -0
  55. data/lib/insika/prefix_fingerprint.rb +32 -27
  56. data/lib/insika/profile_source.rb +1 -0
  57. data/lib/insika/server/app.rb +43 -1
  58. data/lib/insika/server/rack_app.rb +2 -0
  59. data/lib/insika/server/responses.rb +31 -4
  60. data/lib/insika/session_store.rb +4 -1
  61. data/lib/insika/settings_store.rb +10 -1
  62. data/lib/insika/spoken_transcript.rb +31 -0
  63. data/lib/insika/studio/app.rb +6 -2
  64. data/lib/insika/studio/forms.rb +18 -3
  65. data/lib/insika/studio/views/_agent_tab_config.erb +5 -1
  66. data/lib/insika/studio/views/session.erb +1 -1
  67. data/lib/insika/studio/views/tool_edit.erb +6 -2
  68. data/lib/insika/telemetry/recorder.rb +13 -1
  69. data/lib/insika/tool_assembly.rb +21 -13
  70. data/lib/insika/tool_definition.rb +73 -10
  71. data/lib/insika/tool_envelope.rb +102 -2
  72. data/lib/insika/tool_store.rb +9 -4
  73. data/lib/insika/tool_trace_store.rb +1 -1
  74. data/lib/insika/tool_usage_report.rb +12 -2
  75. data/lib/insika/tools/data_defined_tool.rb +1 -0
  76. data/lib/insika/tools/present.rb +122 -0
  77. data/lib/insika/tools/run_persona_eval.rb +6 -1
  78. data/lib/insika/tools/tool_search.rb +4 -2
  79. data/lib/insika/turn_state.rb +13 -1
  80. data/lib/insika/version.rb +1 -1
  81. data/lib/insika/wiring/graph.rb +7 -0
  82. data/lib/insika/wiring/graph_chat.rb +4 -0
  83. data/lib/insika.rb +4 -0
  84. metadata +6 -1
data/docs/DEPLOY.md CHANGED
@@ -188,7 +188,7 @@ insika doctor --json # machine-readable (CI / monitoring)
188
188
  insika doctor --fix # applies the safe autofixes and re-diagnoses
189
189
  insika env # lists known keys + current values (secrets masked)
190
190
  insika tools:report # tool audit over the stored traces: never-called
191
- # allowlisted tools, error rate > 30%, stale tools —
191
+ # allowlisted tools, error rate > 30%, stale tools, gate refusals
192
192
  # read-only, the operator removes ([--agent ID] [--days N] [--json])
193
193
  ```
194
194
 
@@ -203,7 +203,18 @@ text rather than a serialized object** (a file whose content is a stringified Ha
203
203
  serves a mangled prompt on every turn while looking perfectly healthy — present,
204
204
  non-empty, and the agent still answers), **a prompt file that outgrew a prompt**
205
205
  (WARN past ~6 000 estimated tokens or 600 lines — the LLM-generated pack shape
206
- that costs 20%+ extra tokens per turn for no better instruction-following), and
206
+ that costs 20%+ extra tokens per turn for no better instruction-following),
207
+ **in-session compaction enabled with no model slot** (WARN — neither
208
+ `compaction.model` nor the platform `utility_model` is set, so it can never run;
209
+ see [Context](CONTEXT.md)), **eval seeding left on** (WARN — `evals.seeding`
210
+ opens `POST /v1/conversations/:id/seed`, which writes a fabricated conversation
211
+ state under the tenant token; fine while running snapshot evals, off in
212
+ production — see [Evals](EVALS.md#state--a-case-starts-from-a-snapshot)),
213
+ **provenance and presentation tools without an allowed evidence data tool**
214
+ (WARN — the check only sees stored data declarations; verify any code-tool source),
215
+ **fencing off for agents exposed by relay/widget environment configuration**,
216
+ **cache-layer declarations** (known volatile providers marked identity are errors;
217
+ custom identity providers need a byte-stability review), and
207
218
  **skill drift** — a shared skill whose
208
219
  body names one store, a prompt file routing to a skill the agent cannot load, a broken
209
220
  companion pair, a stale `eager:` key (see
data/docs/EVALS.md CHANGED
@@ -42,6 +42,99 @@ expect:
42
42
  Turns replay **in order** under one conversation, so a case can build context ("what
43
43
  about the shipping?" after "I want the 70% bar"); the assertions run on the last turn.
44
44
 
45
+ ### State — a case starts from a snapshot
46
+
47
+ Every case above starts from an **empty** conversation. To test "the customer already
48
+ saw three products and says *add the second one*", such a case would first have to
49
+ replay the search turn — which makes it depend on the model's first answer, costs a
50
+ turn, and cannot reproduce a messy state (a contradiction from six turns ago, a
51
+ stored preference). `state:` is the precondition, loaded into the conversation
52
+ **before turn 1**:
53
+
54
+ ```yaml
55
+ id: loja-chocolates-add-seen
56
+ agent: loja-chocolates
57
+ requires:
58
+ tools: [add_to_cart]
59
+ state:
60
+ evidence:
61
+ ids: ["SKU-70-DARK"]
62
+ cards:
63
+ - { type: card, url: "https://shop.example/70", id: "SKU-70-DARK", caption: "Dark chocolate" }
64
+ memory: { facts: { preference: "dark chocolate" }, notes: [] }
65
+ history:
66
+ - { role: user, content: "quero chocolate amargo" }
67
+ - { role: assistant, content: "O chocolate 70% é SKU-70-DARK." }
68
+ briefing: { fields: { cep: "01311-000" } }
69
+ turns:
70
+ - user: "adiciona uma unidade desse chocolate 70% no carrinho"
71
+ expect:
72
+ tools_called: [add_to_cart]
73
+ never_calls: [search_products]
74
+ reply_omits: ["SKU-70-DARK"]
75
+ ```
76
+
77
+ Only those four `state` keys are accepted by the case loader. A persona case may
78
+ carry `state` too. Adapt the tool names, arguments and briefing fields to the agent.
79
+
80
+ The evidence ledger is a runtime precondition, not a list shown to the model:
81
+ include the relevant product and ID in `history` or the user's request. `cards`
82
+ seeds the cards a search would have returned (each needs a `url` and an `id`; the
83
+ id counts as seen), so a presentation case needs no lookup in the turn.
84
+
85
+ With the HTTP transport, the eval stays a **client**: the runner never writes a
86
+ store. Seeding goes through
87
+ `POST /v1/conversations/:id/seed` (same Bearer as the turn, same id namespacing for a
88
+ tenant), and the deployment accepts it **only while the platform setting
89
+ `evals.seeding` is on** — off by default, because a seeded conversation is a
90
+ fabricated precondition, and the doctor warns while it is on. With the setting off
91
+ the replay Runner **skips** seeded cases with a reason; it also skips when its
92
+ transport has no seed support. Other seed errors fail the replay. In-process
93
+ `GraphTransport` dispatches `seed_session` directly and bypasses `evals.seeding`;
94
+ it needs a runtime exposing the graph. The persona
95
+ Simulator surfaces seed refusal as an error instead of the Runner's skip result.
96
+
97
+ A session with messages returns `409`. Replays default to `eval-<case-id>`; pass
98
+ `--conv-map FILE` with a JSON mapping such as
99
+ `{"loja-chocolates-add-seen":"eval-add-seen-run-2"}` to use a fresh ID on reruns.
100
+ Seeded memory may share a tenant cell even with fresh IDs; see
101
+ [the seed API](API.md#seeding-an-eval-conversation) for scope and payload details.
102
+
103
+ #### Graders — what a turn's calls and reply are checked against
104
+
105
+ The turn's SSE stream reports each tool call with its arguments and how it ended
106
+ (`ok`, `error`, or `blocked` plus the gate that held it), so the deterministic
107
+ layer can check more than "was the tool called". All optional; each is its own
108
+ check, and the report names the one that failed.
109
+
110
+ | Key | Checks |
111
+ |-----|--------|
112
+ | `tools_called: [name, name?]` | each required tool was called (`?` = optional, never fails) |
113
+ | `never_calls: [names]` | none of these was called — the negative every `tools_called` needs |
114
+ | `calls_one_of: [names]` | at least one of these was called |
115
+ | `first_tool: name` | the first call's name |
116
+ | `max_tool_calls: N` | a ceiling on the turn's calls |
117
+ | `reply_includes: [substrings]` | each appears in the published answer (case-insensitive) |
118
+ | `reply_omits: [substrings]` | none appears — where an internal id, a CPF or a raw tag leaking into the customer's text is pinned |
119
+ | `blocked_gates: ["tool:gate"]` | each pair appears among the turn's blocked calls |
120
+ | `ui_components: [names]` | a presentation tool showed at least one card of each component (the `insika.ui` frames) |
121
+ | `no_ui: true` | the turn showed nothing — the negative of `ui_components` |
122
+ | `must_not: [detectors]` | the negative detectors (`pii_leak`, `tool_error`, …); a blocked call is not a tool error |
123
+
124
+ **Every positive has a negative.** A case that only says `tools_called: [add_to_cart]`
125
+ passes an agent that also re-searched, or that echoed the SKU to the customer. Pin
126
+ what a correct turn does *not* do — `never_calls`, `reply_omits`, `max_tool_calls` —
127
+ in the same case. Snapshots avoid the model calls needed to reconstruct the setup;
128
+ use the simulator when later customer messages need to branch on the replies.
129
+
130
+ `tools_called` confirms an attempted call, not a successful backend mutation.
131
+ Arguments are collected for inspection; there is no generic argument or cart-state
132
+ grader. `blocked_gates` needs completion statuses from the HTTP transport. The
133
+ in-process `GraphTransport` records tool names and UI events, but not completion
134
+ statuses or arguments. Neither transport counts `load_skill` / `load_knowledge` as
135
+ tool calls. `ui_components` requires a nonempty selection; `no_ui`
136
+ accepts absent UI events and events with zero items.
137
+
45
138
  ### `requires` — a case that cannot run here is skipped, not failed
46
139
 
47
140
  Deployments differ. Some stores have order tracking wired, some do not; some run
@@ -269,14 +362,11 @@ The model only sees `case_id`, enumerated with the ids the tool can actually run
269
362
  (never a free string it could invent) — every **simulated** case in the store, the same
270
363
  `persona:` shape as above.
271
364
 
272
- **Safety is derived here too, but there is no swap yet.** The tool computes the
273
- target's reachable side-effect tools (`Evals::EvalProfile`, the same derivation the CLI
274
- uses) and **refuses outright** naming the tools if that list is non-empty. Unlike
275
- the CLI, nothing here actually swaps a side-effect tool for a dry-run: `Evals::
276
- EvalProfile.registry` (the overlay) exists for exactly that, but nothing calls it yet.
277
- So `run_persona_eval` only runs against **read-only** target agents today; wiring the
278
- overlay into an in-process turn (so a target WITH a write tool can be tested safely) is
279
- follow-up work, not something this tool claims to do.
365
+ **Side effects are replaced for local graph runs.** The tool derives the target's
366
+ reachable side-effect tools and builds a throwaway executor with recorder tools
367
+ under those names. Read-only tools and the other graph collaborators are shared.
368
+ If side effects are reachable but no graph was supplied, it refuses the run.
369
+ This substitution prevents real writes; it does not prove a backend mutation.
280
370
 
281
371
  **Budget**: the persona model + judge model calls are the cost of running the eval,
282
372
  charged to the **calling** agent's own turn — never the target's (the target's own
data/docs/FACTS.md CHANGED
@@ -38,6 +38,10 @@ decides; the engine never applies its own proposal.
38
38
  5. Approved facts join the customer's memory cell and are injected by the
39
39
  Memory provider on the next turn of any session of that customer.
40
40
 
41
+ The extraction transcript contains only nonblank `user` and `assistant` text,
42
+ with original message indexes preserved. Tool payloads are excluded regardless
43
+ of `fencing`; an assistant's repetition of tool text can still be included.
44
+
41
45
  ## Enabling it — the `distill:` block
42
46
 
43
47
  Distillation is pack data on the agent, exactly like `refinement:` or
data/docs/KNOWLEDGE.md CHANGED
@@ -27,6 +27,13 @@ and an operator can see, edit and resolve all of it in the Studio. Only the
27
27
  optional FTS5 index remains, deferred with a measured trigger — see
28
28
  [What's not here yet](#whats-not-here-yet).
29
29
 
30
+ Extraction reads only nonblank `user` and `assistant` text, retaining original
31
+ message indexes. Direct tool payloads are excluded regardless of `fencing`;
32
+ assistant paraphrases can still reach the extractor. The names and descriptions
33
+ in the injected `<knowledge>` block are sanitized when
34
+ [fencing](AGENTS.md#fencing--third-party-text-is-data-never-instructions) is on.
35
+ The full body returned by `load_knowledge` is not fenced.
36
+
30
37
  ## The concept format
31
38
 
32
39
  One concept is one record — a markdown document with a YAML frontmatter
@@ -25,15 +25,17 @@ authoring writes (`:golden_written`, `:agent_file_written`, …), queue bookkeep
25
25
  channel delivery (`:channel_delivered` — see [Channels](CHANNELS.md))
26
26
  travel the same stream and are **ignored** by the bridge: they open no span and
27
27
  touch no instrument, because they are not part of a turn's latency or cost. Any
28
- other subscriber still sees them. The one mid-turn event the bridge does consume
29
- is `:tool_loop_intervened` it feeds the `insika.tool.loop_intervened` counter
30
- (no span: the intervention is a fact about the turn, not a timed operation).
28
+ other subscriber still sees them. Additional events feed counters without opening
29
+ spans: `:tool_loop_intervened`, `:tool_blocked` and `:context_compacted`.
31
30
 
32
31
  They are worth subscribing to even so, because each is the ONLY record of
33
32
  something that left no task of its own behind:
34
33
 
35
34
  | Event | Data | What it answers |
36
35
  |---|---|---|
36
+ | `:tool_blocked` | `name`, `gate`, `param` | a provenance gate refused a call before execution; no rejected value in this event |
37
+ | `:ui` | `component`, `title`, `items`, `count`, `dropped` | the presentation selection, including customer-facing card content; see [API](API.md#tool-and-presentation-sse-events) |
38
+ | `:session_seeded` | `session_id`, `tenant`, `keys` | an eval snapshot was loaded; no snapshot contents |
37
39
  | `:turn_coalesced` | `task_id`, `merged`, `arrivals[]` | the fragments a customer typed in a row arrived as separate messages, and when |
38
40
  | `:turn_steered` | `task_id`, `count`, `total` | a message arrived mid-run and was appended to the turn in flight |
39
41
  | `:turn_steer_released` | `task_id`, `released_as`, `count` | the run could not absorb it, so it became the turn `released_as` |
@@ -70,8 +72,9 @@ and correct while the customer got nothing, because delivery is a separate,
70
72
  retried, out-of-band step. `status: "failed"` means the reply is sitting in the
71
73
  outbox and the customer is still waiting.
72
74
 
73
- Counts, ids and times only never message content. The text lives in the
74
- transcript, which is the surface that is allowed to carry it.
75
+ Operational audit events use metadata rather than transcript bodies. Presentation
76
+ `:ui` events intentionally contain customer-facing card content; tool-call events
77
+ can contain arguments. Do not treat the whole event stream as content-free.
75
78
 
76
79
  The bridge speaks the standard the market already runs on: point any OTLP backend
77
80
  at Insika and a real turn shows up as a full trace, next to counters and histograms
@@ -82,6 +85,14 @@ backend config, no vendor file. It ships a stable set of attribute and instrumen
82
85
  names, and the recipes below tell you what to chart against them — in whatever you
83
86
  already run.
84
87
 
88
+ ### Blocked tool calls
89
+
90
+ `tool_blocked` carries `name`, `gate`, and `param`, with task/session correlation
91
+ in event metadata. It never carries the rejected value. The `insika.tool.blocked`
92
+ counter (unit `{call}`) uses the turn's agent/tenant/command labels plus
93
+ `insika.tool` and `insika.gate`. Session traces keep the `gate` field alongside the
94
+ masked result; `insika tools:report` lists blocked calls separately from errors.
95
+
85
96
  ## Contents
86
97
 
87
98
  - [Turning it on](#turning-it-on-opt-in-parity-when-off)
@@ -145,6 +156,7 @@ knows its outcome.
145
156
  | `insika.turn.duration` | histogram | `s` | same, when both timestamps are known |
146
157
  | `insika.tokens` | counter | `{token}` | the turn reported usage |
147
158
  | `insika.cost` | counter | `{USD}` | the turn's model is priced (see below) |
159
+ | `insika.tool.blocked` | counter | `{call}` | a `tool_blocked` gate refusal |
148
160
  | `insika.tool.calls` | counter | `{call}` | a tool call completes |
149
161
  | `insika.tool.duration` | histogram | `s` | a `tool_call`/`tool_result` pair completes |
150
162
  | `insika.cache.hit_rate` | histogram | `%` | a turn reported billed prompt tokens (see below) |
@@ -291,7 +303,10 @@ climbing while `input` stays flat.
291
303
  `insika.tokens` filtered to `insika.token.type="cached"` over the same counter
292
304
  filtered to `input`. This is the number that moves your bill. For the per-turn
293
305
  distribution (does every turn hit, or do fleet averages hide cold agents?), chart
294
- `insika.cache.hit_rate` — p50 by `insika.agent`; a healthy agent sits near 100.
306
+ `insika.cache.hit_rate` — p50 by `insika.agent`. Compare with that agent's baseline:
307
+ cache eligibility, expiry and changing history affect the ratio. The
308
+ [identity-prefix fingerprint](CONTEXT.md#the-observable-cache-fingerprints-and-the-invalidation-reason)
309
+ explains identity/tool-schema changes; volatile category digests are separate.
295
310
 
296
311
  **Loop interventions**
297
312
  `insika.tool.loop_intervened`, rate, grouped by `insika.agent` and `insika.tool`.
data/docs/POLICY.md CHANGED
@@ -34,7 +34,10 @@ Builtins cover tool-, skill-, and workflow-allowlisting, plus
34
34
  does not allow or deny but *tags* a tool as needing human approval. Set
35
35
  `approvals_required: [tool names]`; the gate then fires when the model tries to
36
36
  call that tool, suspending the turn until an operator approves it in the Studio.
37
- See [Security](SECURITY.md#human-approval).
37
+ A data tool's `requires_evidence` check runs before this approval gate; an unknown
38
+ ID is blocked without asking the operator. See
39
+ [Tools](TOOLS.md#provenance-checking-ids-before-a-write) and
40
+ [Security](SECURITY.md#human-approval).
38
41
 
39
42
  ## Layer 3: Guardrails (content safety)
40
43
 
data/docs/RELEASING.md CHANGED
@@ -19,6 +19,10 @@ is invisible to it. Do not publish on rspec alone.
19
19
  3. Every new `lib/` file is **tracked in git**. The gemspec's `files` come from
20
20
  `git ls-files`: an untracked file builds without a warning and the installed
21
21
  gem fails at `require` — this is exactly the failure this proof exists to catch.
22
+ 4. If changing the `fencing` default (currently off), first run the deployment's
23
+ golden cases with it enabled, retain the comparison report, and document the
24
+ behavior change in `CHANGELOG.md`. A default change needs evaluation evidence;
25
+ a scheduled version number alone is not the release gate.
22
26
 
23
27
  ## Cut the gem
24
28
 
data/docs/SECURITY.md CHANGED
@@ -205,6 +205,32 @@ category: the agent's category reply → the agent's default → the builtin
205
205
  category → the builtin default. All of it is editable in the Studio Configuration
206
206
  form. See [Agents §Layer 3](POLICY.md#layer-3-guardrails-content-safety).
207
207
 
208
+ ## Third-party text is data (fencing)
209
+
210
+ Input guardrails cover the incoming message. The opt-in `fencing` flag also
211
+ sanitizes ordinary tool result strings, memory text and the injected knowledge
212
+ names/descriptions, and places a
213
+ fixed notice under the identity. It reduces known markup and Unicode tricks;
214
+ it does not make arbitrary third-party instructions safe.
215
+
216
+ The user/assistant-only extraction filter applies even when fencing is off.
217
+ Attachment captions and `load_knowledge` bodies are not fenced. The extraction
218
+ filter excludes direct tool messages,
219
+ but assistant paraphrases can still become extraction input. See
220
+ [Agents](AGENTS.md#fencing--third-party-text-is-data-never-instructions) for the
221
+ exact scope, exceptions, default and size cap.
222
+
223
+ ## Write provenance and serial execution
224
+
225
+ A data tool's `requires_evidence` gate checks IDs against the session ledger
226
+ before approval or backend execution. A known ID proves a prior lookup; it does
227
+ not authorize access or establish current stock, price or quantity limits.
228
+ Those checks remain the backend's responsibility.
229
+
230
+ Tools marked `side_effect` execute one at a time within a session, including in
231
+ parallel batches. This is not a cross-session or distributed backend lock.
232
+ See [Tools](TOOLS.md#provenance-checking-ids-before-a-write).
233
+
208
234
  ## Human approval
209
235
 
210
236
  Some tool calls should not happen unattended. Mark them with
@@ -242,7 +268,7 @@ defense-in-depth: without it, `ALLOW_PRIVATE` opens *any* private destination.
242
268
  > the request never leaves the process, and the conversation *looks* fine. Verify
243
269
  > tool health by the Studio session **trace** (a healthy call shows the backend's
244
270
  > `200`), never by the model's reply. Full detail in
245
- > [Tools §Egress](TOOLS.md#egress-the-ssrf-guard-and-its-silent-failure).
271
+ > [Tools §Egress](TOOLS.md#egress-the-ssrf-guard).
246
272
 
247
273
  ## Sandbox: confined execution
248
274
 
data/docs/TOOLS.md CHANGED
@@ -12,10 +12,10 @@ kinds, and the distinction that matters is **who can change one at runtime**:
12
12
 
13
13
  | | **Code tool** | **Data tool** | **MCP tool** |
14
14
  |---|---|---|---|
15
- | What | a Ruby class (`< RubyLLM::Tool`) | an HTTP call described by config, no Ruby | an MCP server's tool, called LIVE |
15
+ | What | a Ruby class (`< RubyLLM::Tool`) | an HTTP call or card presentation described by config | an MCP server's tool, called LIVE |
16
16
  | Lives | in the deployment image | as a row in SQLite | on the MCP server, behind a live client |
17
17
  | Editable at runtime | no (shipped in the image) | **yes** (DSL / API / manifest / Studio) | **yes** — enable/edit the *instance* (DSL / CLI / API / JSON import / Studio); the server owns its own tools |
18
- | Reach for it when | logic must run in-process (file edit, shell, subagent) | calling an external HTTP API | adopting a whole external MCP server's toolset |
18
+ | Reach for it when | logic must run in-process (file edit, shell, subagent) | calling an HTTP API or selecting evidence cards | adopting a whole external MCP server's toolset |
19
19
 
20
20
  **MCP tools are not data tools.** Configuring an enabled MCP **instance** (any
21
21
  surface below) is enough — its tools appear automatically, tagged
@@ -120,7 +120,7 @@ once, at ingestion — see the gotcha below before reaching for it:
120
120
  (`object/array/string/number/integer/boolean`); `oneOf`/`anyOf`/`allOf`/`$ref`/
121
121
  `if`/`then`/`else` are forbidden (not every provider supports them).
122
122
  - `side_effect` defaults from the method (GET/HEAD → false, else true) and drives
123
- checkpoint/replay semantics (a completed side-effecting tool is not re-run on
123
+ serial execution within a session and checkpoint/replay semantics (a completed side-effecting tool is not re-run on
124
124
  resume — see [Architecture](ARCHITECTURE.md#durability-checkpoints-and-resume)).
125
125
 
126
126
  ### `halt_when`: when the answer is already out
@@ -199,12 +199,10 @@ backend, not of whoever calls it. Every agent sharing the tool gets the same val
199
199
 
200
200
  ## Evidence: the lean envelope and grounding
201
201
 
202
- A catalog tool returns products; the model should only ever quote the ones the tool
203
- actually returned the store dies of a SKU the model invented. `evidence` is the
204
- declaration that makes "no claim without a tool ID" an engine rule instead of a
205
- prompt convention. One declaration does **both** jobs: the engine strips the result
206
- down to what the model sees (the lean envelope) **and** records every returned id on
207
- the session's evidence ledger. There is no "lean but not evidence" mode.
202
+ An `evidence` declaration reshapes a tool result into a lean list and records its
203
+ IDs in the session ledger. The ledger supplies the write gate, presentation tools
204
+ and optional output grounding below. Declaring evidence alone does not prevent
205
+ unsupported claims in the final answer.
208
206
 
209
207
  ```jsonc
210
208
  { "name": "search_products",
@@ -226,14 +224,63 @@ the session's evidence ledger. There is no "lean but not evidence" mode.
226
224
  never a null. A malformed evidence result becomes `{ "error": … }` back to the
227
225
  model — a correctable tool answer, exactly like a malformed call.
228
226
  - **Attachments** are the optional second half: `[{ "type": "card"|"image",
229
- "url": "…", "caption": "…" }]` (≤ 16, url ≤ 500 chars, malformed dropped). They
227
+ "url": "…", "caption": "…", "id": "…" }]` (≤ 16, url ≤ 500 chars, malformed dropped). They
230
228
  **never** reach the model context or the transcript — they ride the channel
231
229
  delivery as an additive `attachments` key on the outbox payload, and the channel
232
- (or its consumer) decides what a card looks like.
230
+ (or its consumer) decides what a card looks like. Supply an explicit `id` when
231
+ cards are not one-to-one with items in the same order. Without one, an attachment
232
+ takes its item's ID at the original position, before malformed cards are dropped.
233
+ - Lean `line` passes through the tool-result sanitizer only when `fencing` is on.
234
+ Attachment captions are normalized to UTF-8 but are not fenced. See
235
+ [Fencing](AGENTS.md#fencing--third-party-text-is-data-never-instructions).
233
236
  - A **code tool** opts in the same way: it either returns `{ items, attachments }`
234
237
  directly and declares `evidence` in its registry metadata, or exposes an
235
238
  `evidence` reader. No declaration = today's tool behavior, byte for byte.
236
239
 
240
+ ### Provenance: checking IDs before a write
241
+
242
+ Declare `requires_evidence` on a data-defined tool to accept only IDs previously
243
+ returned by an `evidence` tool in the same session:
244
+
245
+ ```json
246
+ { "requires_evidence": ["product_id"] }
247
+ ```
248
+
249
+ The full form is `{ "requires_evidence": { "params": ["product_id"] } }`.
250
+ The list must be non-empty and name declared top-level parameters. Scalar values
251
+ and every element of an array are converted to strings and compared exactly:
252
+ `SKU-1` and `sku-1` are different IDs. IDs typed by a customer do not count.
253
+ The ledger includes earlier turns and completed evidence results in the current
254
+ turn, capped at the latest 1,000 distinct IDs. Every declared parameter the call
255
+ carries must pass before any write occurs; a parameter the schema marks optional
256
+ and the model leaves out has nothing to check, a required one left out blocks.
257
+ Search first, then write in a later batch: a search and write in the same parallel
258
+ batch have no dependency ordering guarantee.
259
+
260
+ An unknown ID returns `status: "blocked"`, `gate: "provenance"`, the parameter,
261
+ the value, and an instruction to search or look it up before retrying. The backend
262
+ is never called and no operator approval is requested. A missing ledger blocks the
263
+ call too. Omit the declaration to keep the existing behavior; MCP tools do not
264
+ support this declaration.
265
+
266
+ The Studio tool editor exposes `requires_evidence`. Blocked calls appear in session
267
+ traces and `insika tools:report`, emit `tool_blocked` with name/gate/parameter only,
268
+ and increment `insika.tool.blocked`. `insika doctor` warns when an agent allows a
269
+ gated data tool without an allowed data tool declaring `evidence`.
270
+
271
+ ### Side effects in parallel batches
272
+
273
+ With `limits.tool_concurrency > 1`, tools marked `side_effect` execute one at a time
274
+ within the session's runtime. Unmarked tools still run concurrently, including
275
+ while a write is running. MCP tools are marked `side_effect: true` unless the server
276
+ annotates them `readOnlyHint`; a read-only `POST` data tool needs an explicit
277
+ `"side_effect": false` to keep its concurrency.
278
+ A queued write holds no concurrency slot. Different sessions remain independent;
279
+ backend rules such as quantity limits remain the backend's responsibility.
280
+
281
+ The per-tool timeout starts after both gates are acquired. Trace duration includes
282
+ queueing time, so it measures how long the model waited, not just backend execution.
283
+
237
284
  ### Grounding: policing claims against the ledger
238
285
 
239
286
  With the ledger fed, the pack declares how claims are policed — data on the agent,
@@ -263,11 +310,63 @@ grounding mode: :flag, matcher: { sku: '\b[A-Z]{2,4}\d{4,8}\b' }
263
310
  - Grounding is **independent of the guardrails opt-in**: an agent with guardrails
264
311
  off and `grounding.mode: :flag` still gets the check.
265
312
 
313
+ ## Presentation tools: the model picks ids, the engine shows the cards
314
+
315
+ A **presentation tool** selects which evidence cards to show. Declare it with
316
+ `presentation` instead of `request`; it runs in-process and is always
317
+ `side_effect: false`. The model supplies IDs, never card URLs or captions.
318
+
319
+ ```jsonc
320
+ { "name": "present_products",
321
+ "description": "Show product cards to the customer. Pass only ids a search returned.",
322
+ "parameters": [{ "name": "product_ids", "type": "array:string" },
323
+ { "name": "title", "type": "string", "required": false }],
324
+ "presentation": { "component": "product_cards", // what the channel renders
325
+ "ids": "product_ids", // the array:string parameter
326
+ "max": 8 } } // 1..16 (the attachment cap; default 16)
327
+ ```
328
+
329
+ Exactly one of `request` / `presentation`; `ids` must name a declared `array:string`
330
+ parameter; `component` follows the tool-name rule. Create it through the DSL,
331
+ manifest or API. The Studio form preserves an existing `presentation` declaration
332
+ on save but does not expose its fields. Requested IDs are deduplicated in order
333
+ and compared case-sensitively. The engine then:
334
+
335
+ 1. keeps only ids the session's **evidence ledger** has seen — the rest are dropped
336
+ with reason `unknown` (an id the model invented, or the customer typed);
337
+ 2. joins each kept id to the card an evidence tool returned **this session** — this
338
+ turn's first, then the last 64 the ledger kept (cards carry the `id` of the item
339
+ they stand for); a known id with no card is dropped with reason `no_card`;
340
+ 3. truncates to `max` — the overflow is dropped with reason `max`;
341
+ 4. records the selection on the turn and emits `:ui` on the stream (published as
342
+ `insika.ui` on `/v1/responses`, as the `ui` frame on the web channel);
343
+ 5. answers the model with `shown` (IDs) and `dropped` (objects with `id` and `reason`) — plus one
344
+ instruction when nothing could be shown ("name the products in text or search
345
+ again").
346
+
347
+ **Delivery.** When a turn made a presentation call, the outbox `attachments` are
348
+ *exactly the presented cards*, in call order, each stamped with the call's `component`
349
+ and `title`. A turn with no presentation call delivers every hoarded card, as before —
350
+ a pack that declares no presentation tool sees no change. An empty selection also
351
+ suppresses that fallback: it emits `count: 0` and delivers no cards from that call.
352
+ An earlier turn's ID can be shown without a fresh lookup while its card remains
353
+ in the session's 64-card ledger. IDs and cards have separate caps: a known ID can
354
+ still return `no_card`. Cards are stored snapshots, not a stock or price refresh.
355
+
356
+ A presentation tool needs an evidence source to populate the session's cards.
357
+ `insika doctor` warns when an agent allows presentation without an allowed evidence
358
+ data tool (`presentation-tools`); it cannot verify a code-tool evidence source. The line that tells the model
359
+ *when* to show cards ("show cards with present_products, ids only") is the pack's.
360
+
361
+ Not here: partial rendering while arguments stream, per-component enrichment (price
362
+ today, stock — the card is what the evidence tool returned), and a second component
363
+ such as suggestion chips (same mechanism, when a channel asks for it).
364
+
266
365
  ## Registering a tool
267
366
 
268
367
  A tool appears in the Studio panel and enters an agent's tool-loop when it is
269
368
  **registered** in the catalog **and** allowed by the agent's policy allowlist.
270
- Four ways to write a data tool into the store — all **hot** (registry and catalog
369
+ Three ways to write a data tool into the store — all **hot** (registry and catalog
271
370
  reload, no restart):
272
371
 
273
372
  1. **DSL** — `data_tool(name:, …)` in a `Insika.agent { … }` block.
@@ -275,8 +374,6 @@ reload, no restart):
275
374
  3. **Manifest** — `POST /v1/tools/manifest`. Partial failure is isolated: one
276
375
  malformed tool becomes an `errors[]` entry; only a structural manifest error
277
376
  fails the whole request. The response reports `{ version, created, updated, errors }`.
278
- 4. ~~MCP ingestion~~ — retired. An MCP server's tools are no
279
- longer written into this store at all; see [MCP servers](#mcp-servers).
280
377
 
281
378
  ### The one gotcha: env/secret templating is manifest-only
282
379
 
@@ -298,7 +395,7 @@ at turn time, not ingestion).
298
395
  An MCP **instance** is durable config — transport, target, credentials, an
299
396
  `enabled` flag — held in its own store, separate from data tools. Once an
300
397
  instance is enabled, its tools appear in the catalog automatically (group
301
- `mcp:<instance>`, `side_effect: true`), and every call goes straight to the
398
+ `mcp:<instance>`, `side_effect: true` unless annotated `readOnlyHint`), and every call goes straight to the
302
399
  server through a live, held client — the runtime never converts an MCP tool
303
400
  into a stored data tool.
304
401
 
@@ -405,9 +502,10 @@ held client, which does its own discovery on first use regardless of whether
405
502
  A model can ask for several tools in one step. By default the engine runs them one
406
503
  at a time. Set `limits[:tool_concurrency]` above 1 (see
407
504
  [Agents](AGENTS.md#tool_concurrency--parallel-tool-calls)) and the calls in that
408
- batch run concurrently, **at most N in flight**, on the turn's own reactor — so
409
- the wall-clock of a batch of slow data tools approaches the slowest call rather
410
- than their sum. The cap covers every enveloped tool of the turn, including the
505
+ batch run concurrently, **at most N in flight**, on the turn's own reactor.
506
+ Tools marked `side_effect` still execute one at a time per session; queued writes
507
+ acquire that serial gate before taking a shared concurrency slot. See
508
+ [Side effects](#side-effects-in-parallel-batches). The cap covers every enveloped tool, including the
411
509
  ones `tool_search` promotes mid-turn.
412
510
 
413
511
  It applies only to what the *model* fans out. Two primitives already parallelize
@@ -434,7 +532,7 @@ Turning it on changes three things, all of them worth knowing before you do:
434
532
  Approvals and concurrency are mutually exclusive per turn — the approval gate wins
435
533
  and the turn goes serial. That is a deadlock avoided, not a preference.
436
534
 
437
- ## Egress: the SSRF guard (and its silent failure)
535
+ ## Egress: the SSRF guard
438
536
 
439
537
  Data tools make outbound HTTP, so every call passes through the **EgressGuard**, a
440
538
  Server-Side Request Forgery defense. The default posture is **strict: public
@@ -446,17 +544,10 @@ Server-Side Request Forgery defense. The default posture is **strict: public
446
544
  | `INSIKA_EGRESS_ALLOW_HTTP=1` | permit plain `http` — **loopback dev only** |
447
545
  | `INSIKA_EGRESS_ALLOW_PRIVATE=1` | permit private/loopback IPs — **dev only** |
448
546
 
449
- > ⚠️ **Egress failures are silent.** When a tool targets a blocked host (e.g. a
450
- > plain-`http` localhost backend without the opt-ins), the guard turns the block
451
- > into a `{ error: }` returned **to the model** — the request never leaves the
452
- > process, yet the stream still emits a tool call, so the model narrates a
453
- > plausible failure and the conversation *looks* like it worked. You will not see
454
- > an exception.
455
- >
456
- > **Always verify by the trace, never by the reply:** open the Studio session
457
- > viewer — a healthy call shows the request, args, and the backend's `200`; a
458
- > missing or errored call is almost always egress (host not in the allowlist, or
459
- > `http`/private without the opt-in).
547
+ An egress rejection returns `{ error: }` to the model without making the HTTP
548
+ request. Inspect the session trace for the actual error; a plausible reply does
549
+ not prove the backend ran. Provenance refusals instead report `status: "blocked"`
550
+ and `gate: "provenance"`.
460
551
 
461
552
  Egress is **orthogonal** to registration and allowlisting: a tool can be
462
553
  registered, allowed, offered to the model, and still blocked at call time.
@@ -471,8 +562,9 @@ Work down this checklist:
471
562
  `data-tools` check is the only place that says so.
472
563
  2. **Allowed for this agent?** In `tools_allow` (or an allowed group), and not in
473
564
  `tools_deny`?
474
- 3. **Egress?** If it *appears and is called* but "fails", open the trace — a
475
- blocked call is ~99% egress.
565
+ 3. **Call refused or failed?** Inspect the trace. For `gate: "provenance"`, look up
566
+ the ID through an evidence tool before retrying. For an error, check its message
567
+ for schema, egress, timeout or backend failures.
476
568
  4. **URL literal?** For non-manifest tools, an unresolved `{{env.*}}` would have
477
569
  422'd at import — re-check the definition.
478
570
 
@@ -490,7 +582,7 @@ routes, the signed link and the retention/LGPD reach.
490
582
  The per-session trace answers "what did this conversation call"; nothing used to
491
583
  answer "what does this agent carry and never use". The report aggregates the
492
584
  stored traces per agent (tasks → sessions → `tool_traces`, the same read the
493
- Studio does) and flags three shapes:
585
+ Studio does) and flags four shapes:
494
586
 
495
587
  - **`never_called`** — in `tools_allow`, zero calls in any stored trace. Dead
496
588
  weight: its schema ships on every request and buys nothing.
@@ -498,6 +590,8 @@ Studio does) and flags three shapes:
498
590
  the window (default 14 days). Either the tool is broken or the model cannot
499
591
  hold its contract.
500
592
  - **`stale`** — called at some point, but not once inside the window.
593
+ - **`blocked`** — gate refusals in the window, counted by gate. These do not count
594
+ as conventional tool errors.
501
595
 
502
596
  ```bash
503
597
  insika tools:report # every stored agent
@@ -30,6 +30,7 @@ with one turn. Nothing more.
30
30
  | The need | The kind | Where it lives |
31
31
  |---|---|---|
32
32
  | Call an external HTTP API | **data tool** (`data_tool` in the DSL block) | a row in SQLite, editable at runtime |
33
+ | Show selected evidence cards | **presentation data tool** (`presentation`, no `request`) | the tool store; see [Tools](../TOOLS.md#presentation-tools-the-model-picks-ids-the-engine-shows-the-cards) |
33
34
  | Logic must run in-process | **code tool** (a Ruby class `< RubyLLM::Tool`) | the deployment image |
34
35
  | Adopt a whole external MCP server | **`mcp` instance** | durable config; its tools appear tagged `mcp:<name>` |
35
36
  | Teach a procedure (no data fetching) | **skill** (`skill "name", description:, instructions:`) | loads on demand via `load_skill` |
@@ -78,6 +79,12 @@ RULES:
78
79
  verbatim and arguments are checked against it at call time.
79
80
  - Author the FINAL url: the HTTP client does not follow redirects, and the egress guard
80
81
  cleared that host only.
82
+ - For a write that accepts catalog IDs, declare `requires_evidence` for those
83
+ parameters and ensure an allowed lookup declares `evidence`. Mark the write
84
+ `side_effect: true`; the engine serializes marked writes within a session.
85
+ - For presentation, follow the [card contract](../TOOLS.md#presentation-tools-the-model-picks-ids-the-engine-shows-the-cards):
86
+ pass known IDs to the presentation tool. Look up again if their cards are absent
87
+ from the session ledger or current information is needed.
81
88
  - Do not add a second capability "while we're here".
82
89
 
83
90
  ## Step 3 — Make sure it enters the tool-loop
@@ -96,8 +103,11 @@ instead of fighting it.
96
103
  ## Step 4 — Prove it with ONE turn
97
104
 
98
105
  Run one `reply()` whose message forces the call ("how many BRL is 1 USD right now?").
99
- The reply must use what the tool returned if the model answers from imagination, the
100
- tool did not run: re-check Step 3 before touching the prompt.
106
+ Inspect the trace to confirm the call and result; the reply alone is not proof.
107
+ For a gated write, also verify that an unknown ID is blocked before any backend
108
+ request. For presentation, verify the selected `insika.ui` items. Use
109
+ [snapshot eval assertions](../EVALS.md#graders--what-a-turns-calls-and-reply-are-checked-against)
110
+ when the flow needs a repeatable precondition.
101
111
 
102
112
  ## Step 5 — Self-check
103
113
 
@@ -43,6 +43,9 @@ text), roughly **when**, and expected vs. actual. Reproduce once locally if chea
43
43
  | turn completed, customer got nothing | delivery is separate from the turn: check `channel_delivered` / `delivery_failed` | [Channels](../CHANNELS.md) |
44
44
  | freshly created agent returns empty turns | persona overflows the default `context_budget` (8000) | [Context](../CONTEXT.md) |
45
45
  | tool never called (or "missing") | not registered OR not allowed (`tools_allow`) | [Tools](../TOOLS.md) § troubleshooting |
46
+ | `tool_blocked` with `gate: provenance` | ID absent from session evidence; look up before retrying | [Tools](../TOOLS.md#provenance-checking-ids-before-a-write) |
47
+ | presentation shows no cards | inspect dropped IDs: `unknown`, `no_card`, `max`; a known ID may have no retained card | [API](../API.md#tool-and-presentation-sse-events) |
48
+ | cache reuse dropped | compare identity/tool-schema fingerprints with provider cache token accounting | [Context](../CONTEXT.md#the-provider-prefix-cache) |
46
49
  | identical `tool_call` repeated, then abort | the `max_tool_repeat` loop guard | [Agents](../AGENTS.md) § limits |
47
50
  | model gave up after one empty result | `tool_persistence` off (it is ON by default) | same |
48
51
  | `turn_stuck` event | the agent declared it cannot proceed — escalation signal, not a bug | [Agents](../AGENTS.md) § stuck |
@@ -87,7 +87,9 @@ In order, each with evidence:
87
87
 
88
88
  1. `curl https://<host>/up` → `{"status":"ok"}`.
89
89
  2. `bin/insika doctor` against the deployed volume (or via the platform's shell) —
90
- relay its findings verbatim; fix errors before continuing.
90
+ relay its findings verbatim; fix errors before continuing. Check evidence sources
91
+ for gated writes/presentation, review the fencing warning, and leave
92
+ `evals.seeding` off after snapshot evals. See [Deploy](../DEPLOY.md#strict-config-and-insika-doctor).
91
93
  3. One authenticated turn:
92
94
 
93
95
  ```bash