insika 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (185) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +145 -17
  3. data/README.md +7 -1
  4. data/bin/insika +211 -8
  5. data/docs/AGENTS.md +377 -2
  6. data/docs/API.md +56 -0
  7. data/docs/CHANNELS.md +99 -9
  8. data/docs/CONTEXT.md +127 -16
  9. data/docs/DEPLOY.md +5 -2
  10. data/docs/EVALS.md +19 -0
  11. data/docs/FACTS.md +135 -0
  12. data/docs/HARVEST.md +117 -0
  13. data/docs/LOADTEST.md +14 -8
  14. data/docs/OBSERVABILITY.md +54 -0
  15. data/docs/REFINEMENT.md +3 -3
  16. data/docs/RELEASING.md +27 -0
  17. data/docs/RUNNING-LOCAL.md +3 -3
  18. data/docs/SECURITY.md +61 -0
  19. data/docs/SKILLS.md +2 -2
  20. data/docs/SOAK.md +127 -0
  21. data/docs/TOOLS.md +70 -2
  22. data/docs/domain.md +115 -0
  23. data/docs/index.md +1 -1
  24. data/lib/insika/agent_profile.rb +145 -10
  25. data/lib/insika/balloon_splitter.rb +102 -0
  26. data/lib/insika/budget_ledger.rb +34 -3
  27. data/lib/insika/cache_series_store.rb +49 -0
  28. data/lib/insika/channel_delivery.rb +118 -13
  29. data/lib/insika/channels/relay.rb +77 -3
  30. data/lib/insika/channels/web.rb +2 -2
  31. data/lib/insika/chat_builder.rb +84 -1
  32. data/lib/insika/checkpoint_store.rb +16 -0
  33. data/lib/insika/commands/agent_payload.rb +3 -3
  34. data/lib/insika/commands/cancel_followup.rb +49 -0
  35. data/lib/insika/commands/delete_tenant_data.rb +95 -0
  36. data/lib/insika/commands/export_customer_memory.rb +48 -0
  37. data/lib/insika/commands/forget_customer.rb +117 -0
  38. data/lib/insika/commands/freeze_funnel_baseline.rb +113 -0
  39. data/lib/insika/commands/gate_harvest.rb +138 -0
  40. data/lib/insika/commands/gate_refinement.rb +1 -1
  41. data/lib/insika/commands/judge_shadow_pairs.rb +124 -0
  42. data/lib/insika/commands/memory_forget_fact.rb +20 -4
  43. data/lib/insika/commands/memory_put_fact.rb +23 -4
  44. data/lib/insika/commands/promote_harvest.rb +130 -0
  45. data/lib/insika/commands/record_outcome.rb +46 -0
  46. data/lib/insika/commands/record_shadow_reply.rb +68 -0
  47. data/lib/insika/commands/reject_harvest.rb +38 -0
  48. data/lib/insika/commands/resolve_proposal.rb +108 -0
  49. data/lib/insika/commands/revoke_contact.rb +49 -0
  50. data/lib/insika/commands/rollback_harvest.rb +86 -0
  51. data/lib/insika/commands/run_distillation.rb +186 -0
  52. data/lib/insika/commands/run_harvest.rb +393 -0
  53. data/lib/insika/commands/send_message.rb +103 -6
  54. data/lib/insika/commands/session_purge.rb +67 -0
  55. data/lib/insika/contact_store.rb +183 -0
  56. data/lib/insika/context/builder.rb +21 -3
  57. data/lib/insika/context/fragment.rb +7 -3
  58. data/lib/insika/context/priority.rb +3 -0
  59. data/lib/insika/context/provider.rb +17 -3
  60. data/lib/insika/context/providers/briefing.rb +96 -0
  61. data/lib/insika/context/providers/memory.rb +16 -7
  62. data/lib/insika/context/providers/prompt.rb +30 -2
  63. data/lib/insika/context/providers/skill.rb +2 -0
  64. data/lib/insika/context/providers/tool_search.rb +2 -0
  65. data/lib/insika/context_trace_store.rb +38 -2
  66. data/lib/insika/distill.rb +224 -0
  67. data/lib/insika/distill_engine.rb +169 -0
  68. data/lib/insika/doctor.rb +716 -6
  69. data/lib/insika/dsl/runtime.rb +9 -3
  70. data/lib/insika/dsl/server_boot.rb +70 -1
  71. data/lib/insika/dsl.rb +83 -3
  72. data/lib/insika/edge_limiter.rb +28 -7
  73. data/lib/insika/env_schema.rb +9 -2
  74. data/lib/insika/errors.rb +11 -0
  75. data/lib/insika/evals/assertions.rb +3 -2
  76. data/lib/insika/evals/pairwise.rb +18 -6
  77. data/lib/insika/evidence.rb +183 -0
  78. data/lib/insika/executor.rb +687 -70
  79. data/lib/insika/followup_engine.rb +207 -0
  80. data/lib/insika/followup_policy.rb +221 -0
  81. data/lib/insika/followup_store.rb +306 -0
  82. data/lib/insika/funnel_declaration.rb +106 -0
  83. data/lib/insika/funnel_fold.rb +179 -0
  84. data/lib/insika/funnel_store.rb +163 -0
  85. data/lib/insika/golden_store.rb +1 -1
  86. data/lib/insika/grounding/matcher.rb +69 -0
  87. data/lib/insika/grounding.rb +44 -0
  88. data/lib/insika/harvest/conversion_gate.rb +159 -0
  89. data/lib/insika/harvest/criterion.rb +98 -0
  90. data/lib/insika/harvest/gate.rb +194 -0
  91. data/lib/insika/harvest/negative_list.rb +199 -0
  92. data/lib/insika/harvest.rb +241 -0
  93. data/lib/insika/harvest_engine.rb +193 -0
  94. data/lib/insika/harvest_store.rb +548 -0
  95. data/lib/insika/media.rb +298 -0
  96. data/lib/insika/memory_audit_store.rb +85 -0
  97. data/lib/insika/memory_store.rb +264 -23
  98. data/lib/insika/message_origin.rb +6 -1
  99. data/lib/insika/model_visible.rb +87 -0
  100. data/lib/insika/model_visible_trace_store.rb +66 -0
  101. data/lib/insika/onboarding.rb +6 -1
  102. data/lib/insika/outbox_store.rb +42 -4
  103. data/lib/insika/outcome_store.rb +147 -0
  104. data/lib/insika/packaging.rb +163 -0
  105. data/lib/insika/parity/criterion.rb +79 -0
  106. data/lib/insika/parity/verdict.rb +318 -0
  107. data/lib/insika/prefix_fingerprint.rb +58 -0
  108. data/lib/insika/profile_source.rb +22 -1
  109. data/lib/insika/proposal_store.rb +271 -0
  110. data/lib/insika/queue_policy.rb +4 -1
  111. data/lib/insika/refinement/proposer.rb +1 -1
  112. data/lib/insika/reliability.rb +32 -6
  113. data/lib/insika/retention.rb +281 -0
  114. data/lib/insika/routing.rb +101 -0
  115. data/lib/insika/safety/config.rb +44 -4
  116. data/lib/insika/safety/corpus.rb +255 -0
  117. data/lib/insika/safety/detectors.rb +32 -113
  118. data/lib/insika/safety/factory.rb +15 -2
  119. data/lib/insika/safety/grounding_enforcer.rb +59 -0
  120. data/lib/insika/safety/grounding_validator.rb +49 -0
  121. data/lib/insika/safety/input_guardrail.rb +1 -1
  122. data/lib/insika/safety/moderator.rb +1 -1
  123. data/lib/insika/safety/output_filter.rb +9 -5
  124. data/lib/insika/safety/output_validator.rb +11 -5
  125. data/lib/insika/schema_guard.rb +35 -0
  126. data/lib/insika/server/app.rb +144 -4
  127. data/lib/insika/server/rack_app.rb +16 -0
  128. data/lib/insika/server/responses.rb +40 -1
  129. data/lib/insika/session_actor.rb +8 -4
  130. data/lib/insika/session_store.rb +65 -2
  131. data/lib/insika/settings_store.rb +10 -0
  132. data/lib/insika/shadow_pair_store.rb +258 -0
  133. data/lib/insika/skill_catalog.rb +4 -0
  134. data/lib/insika/soak/envelope.rb +140 -0
  135. data/lib/insika/soak/report.rb +392 -0
  136. data/lib/insika/soak/runner.rb +554 -0
  137. data/lib/insika/store.rb +10 -1
  138. data/lib/insika/stores/memory.rb +6 -0
  139. data/lib/insika/stores/sqlite.rb +8 -0
  140. data/lib/insika/studio/app.rb +875 -14
  141. data/lib/insika/studio/assets/dist/application.css +1 -1
  142. data/lib/insika/studio/assets/dist/application.js +22 -22
  143. data/lib/insika/studio/assets/dist/favicon.svg +6 -0
  144. data/lib/insika/studio/forms.rb +262 -5
  145. data/lib/insika/studio/nav_icons.rb +6 -1
  146. data/lib/insika/studio/views/agent_detail.erb +627 -84
  147. data/lib/insika/studio/views/agents.erb +10 -6
  148. data/lib/insika/studio/views/approvals.erb +4 -1
  149. data/lib/insika/studio/views/chats.erb +4 -1
  150. data/lib/insika/studio/views/customer.erb +94 -0
  151. data/lib/insika/studio/views/customers.erb +32 -0
  152. data/lib/insika/studio/views/evals.erb +4 -1
  153. data/lib/insika/studio/views/facts.erb +133 -0
  154. data/lib/insika/studio/views/followups.erb +125 -0
  155. data/lib/insika/studio/views/funnel.erb +106 -0
  156. data/lib/insika/studio/views/harvest.erb +234 -0
  157. data/lib/insika/studio/views/home.erb +2 -1
  158. data/lib/insika/studio/views/layout.erb +1 -0
  159. data/lib/insika/studio/views/parity.erb +147 -0
  160. data/lib/insika/studio/views/playground.erb +7 -1
  161. data/lib/insika/studio/views/session.erb +56 -1
  162. data/lib/insika/studio/views/settings.erb +2 -0
  163. data/lib/insika/studio/views/skills.erb +0 -1
  164. data/lib/insika/studio/views/system_files.erb +1 -1
  165. data/lib/insika/studio/views/task.erb +13 -0
  166. data/lib/insika/studio/views/tasks.erb +4 -1
  167. data/lib/insika/studio/views/tools.erb +0 -1
  168. data/lib/insika/task_store.rb +21 -1
  169. data/lib/insika/testing/store_contract.rb +27 -6
  170. data/lib/insika/tick.rb +25 -1
  171. data/lib/insika/tool_definition.rb +17 -7
  172. data/lib/insika/tool_envelope.rb +69 -0
  173. data/lib/insika/tool_manifest.rb +5 -1
  174. data/lib/insika/tools/data_defined_tool.rb +10 -0
  175. data/lib/insika/tools/generate_image.rb +44 -0
  176. data/lib/insika/tools/schedule_followup.rb +164 -0
  177. data/lib/insika/tools/tts.rb +47 -0
  178. data/lib/insika/tools/update_briefing.rb +126 -0
  179. data/lib/insika/turn_state.rb +38 -1
  180. data/lib/insika/turn_timing.rb +22 -2
  181. data/lib/insika/version.rb +1 -1
  182. data/lib/insika/vitals.rb +84 -0
  183. data/lib/insika/wiring/graph.rb +300 -13
  184. data/lib/insika.rb +92 -3
  185. metadata +77 -1
data/docs/REFINEMENT.md CHANGED
@@ -32,11 +32,11 @@ it reaches anyone. Every part of that is below, including what it cannot catch.
32
32
  From the CLI, against the same database the engine uses:
33
33
 
34
34
  ```bash
35
- INSIKA_DB=insika.db bin/insika refine --agent bia
35
+ INSIKA_DB=insika.db bin/insika refine --agent demo
36
36
  ```
37
37
 
38
38
  ```text
39
- bia — completed (last 200 session(s))
39
+ demo — completed (last 200 session(s))
40
40
  tool_error ×24 shipping_quote failed: cep is required
41
41
  sessions: 9f2c1a04…, 4b7e5590…, c1d0aa31…
42
42
  repetition ×7 customer repeated themselves
@@ -173,7 +173,7 @@ an agent with no configuration at all can be run. The optional block on the agen
173
173
  sets the defaults:
174
174
 
175
175
  ```ruby
176
- Insika.agent "bia" do
176
+ Insika.agent "demo" do
177
177
  model "deepseek-v4-flash"
178
178
  refine window: { last_sessions: 200 }, max_findings: 20
179
179
  end
data/docs/RELEASING.md CHANGED
@@ -59,6 +59,33 @@ GEM_HOME="$T/gemhome" GEM_PATH="$T/gemhome" ruby -e '
59
59
  puts "clean"'
60
60
  ```
61
61
 
62
+ ### The 1.0 release gate — the RFC-0036 install proof (E1)
63
+
64
+ For the 1.0 release the proof above is scripted and its **installed-bytes**
65
+ half is asserted by the domain-boundary suite (C1) on the artifact, not the
66
+ repo. Run the runbook, with a key (the smoke turn is one `reply` through the
67
+ installed gem):
68
+
69
+ ```bash
70
+ DEEPSEEK_API_KEY=sk-... scripts/install_proof/install_proof.sh # prints PASS
71
+ ```
72
+
73
+ The script builds, installs into a FRESH `GEM_HOME`, asserts `gem contents
74
+ insika` carries no `deploy/ packs/ examples/ plugins/ evals/ scripts/ spec/`
75
+ path and no demo-persona-name string, and answers one turn from an app dir that follows
76
+ only the public docs. Archive the PASS output with the release notes — it is
77
+ the 1.0 exit criterion "install proof by the docs alone".
78
+
79
+ The same gate writes the freeze date: a breaking `/v1` change needs a new
80
+ `Insika-Version` entry (server/app.rb), a compatibility branch (RFC-0016 A5)
81
+ and a rewritten `**Frozen as of:**` line in `docs/API.md` — the version-gate
82
+ spec pins the two together, and the 1.0 release writes the date at release
83
+ time.
84
+
85
+ Catalog submission checklist (RFC §6): verify the best-of-Agent-Harnesses
86
+ catalog size at submission time (161 vs 154 — the counts diverge across the
87
+ catalog's own pages) and cite the conformance suite as the `durable` evidence.
88
+
62
89
  ## Publish
63
90
 
64
91
  ```bash
@@ -8,7 +8,7 @@ permalink: /running-local/
8
8
  # Running the Insika locally
9
9
 
10
10
  Boots the engine single-process, serving `/studio` and `/v1/*` against a demo
11
- agent (the `bia` persona on DeepSeek). Every message runs the **same**
11
+ agent (the `demo` agent on DeepSeek). Every message runs the **same**
12
12
  `send_message` the API runs — real tools, skills, and memory.
13
13
 
14
14
  ## Boot
@@ -26,7 +26,7 @@ Open `http://localhost:9292`:
26
26
  | URL | What |
27
27
  |-----|------|
28
28
  | `/studio` | management UI (log in with the token; default `local-demo`) |
29
- | `/studio/chats` | chat with the demo agent (`agent: bia`, `session_id: web`, multi-turn ready) |
29
+ | `/studio/chats` | chat with the demo agent (`agent: demo`, `session_id: web`, multi-turn ready) |
30
30
  | `/studio/tasks` | tasks / approvals console |
31
31
  | `/v1/responses` | OpenAI-Responses ingress (Bearer) — the drop-in API contract |
32
32
  | `/v1/agents` | provisioning by definition/pack (Bearer) — `POST` imports, `DELETE /:id` removes |
@@ -82,7 +82,7 @@ the `model`:
82
82
  curl -N http://localhost:9292/v1/responses \
83
83
  -H "Authorization: Bearer local-demo" \
84
84
  -H "Content-Type: application/json" \
85
- -d '{ "model": "bia", "user": "web", "stream": true, "input": "hello" }'
85
+ -d '{ "model": "demo", "user": "web", "stream": true, "input": "hello" }'
86
86
  ```
87
87
 
88
88
  `user` is the session id (any stable id for a multi-turn conversation).
data/docs/SECURITY.md CHANGED
@@ -367,6 +367,67 @@ rotated key or a typo never takes the whole service down. The `insika doctor`
367
367
  command runs the same checks on demand against a live database. See
368
368
  [Deploy](DEPLOY.md#strict-config-and-insika-doctor).
369
369
 
370
+ ## Memory and the right to be forgotten (LGPD, RFC-0031)
371
+
372
+ Memory is scoped per **`(tenant, customer)`** — the cell `"memory:<tenant>:<customer>"`
373
+ is the isolation boundary. A query against one tenant never touches another's
374
+ cells. A turn with no customer falls back to the session's own **marked** cell
375
+ (`"memory:chat:<session id>"`) — the marker keeps the Studio drill and the doctor
376
+ from ever reading a conversation as a customer (a bare `memory:<id>` cell is
377
+ indistinguishable from a single-tenant customer ref). Cells written before the
378
+ marker (bare session ids) may still appear as customers in the drill until
379
+ forgotten or aged out. Three operations enforce the right to be forgotten:
380
+
381
+ - **Retention** — the `memory_ttl_days` setting (platform default per cell, or an
382
+ ops-authored per-tenant map) sweeps facts older than the window. A per-fact
383
+ `expires_at` override wins over the cell TTL. Both run on the daily retention
384
+ tick, **not** gated by `retention_days` (the conversation-footprint window).
385
+ Note the interaction: an explicit `expires_at` also exempts the fact from the
386
+ `retention_days` age-based sweep — an explicit date owns that fact's life, so a
387
+ far-future date makes the fact immune to *both* age passes. Export and forget
388
+ still see it; the only way to remove it early is the Customers drill.
389
+ - **Export** — `export_customer_memory` returns the full fact + note content to the
390
+ **operator** (the Studio turns it into a JSON download). The emitted event carries
391
+ counts only — the event stream, the audit store and every log stay content-free.
392
+ - **Forget** — `forget_customer` purges the customer's memory cell AND their
393
+ sessions (and everything those sessions left behind: traces, tasks, checkpoints,
394
+ outbox deliveries). The operator-mutation audit store records a digest-free line
395
+ ("a purge happened, with N records") — content-free by construction.
396
+
397
+ ### Distilled facts are personal data (RFC-0034)
398
+
399
+ The distillation loop ([Facts](FACTS.md)) writes **proposals** — a distilled
400
+ fact's name and value are personal data, and they are treated like the rest of
401
+ the memory footprint: `forget_customer` deletes the person's proposals (every
402
+ status), `delete_tenant_data` deletes a tenant's, and the `retention_days`
403
+ sweep ages them out with the transcripts they were distilled from. Provenance
404
+ holds: an approved fact is written with `origin: "distilled:<session_ref>"`
405
+ (the RFC-0031 closed set gains one spelling, never an open string). Events and
406
+ audit carry ids and counts only — a fact value never enters the stream, the
407
+ ledger or a log; the evidence excerpt is a link read from the transcript at
408
+ request time, never a copy.
409
+
410
+ The doctor's `memory-scopes` check flags bare cells only in a `multi_tenant`
411
+ deployment (in `single_tenant` the bare cell is the designed customer shape), and
412
+ never the session-marked cells. See [Context](CONTEXT.md#memory) and
413
+ [Deploy](DEPLOY.md#strict-config-and-insika-doctor).
414
+
415
+ ### Harvest candidates are derived data (RFC-0035)
416
+
417
+ The harvest loop ([Harvest](HARVEST.md)) writes **candidates** — a mined skill
418
+ proposal is behavior instructions, the same trust level as any skill content,
419
+ and it is store-scoped, never customer-scoped: `forget_customer` does NOT reach
420
+ it (a candidate's skill body is not customer content). Candidates reference
421
+ sessions by id and their evidence excerpt is the transcript rendered at request
422
+ time through the output filter — never a copy. `delete_tenant_data` purges a
423
+ tenant's candidates, promotion rows and markers; the `retention_days` sweep
424
+ ages them out with the transcripts they were derived from (they are
425
+ re-derivable — pruning is never data loss). **No auto-application**: nothing
426
+ reaches the catalog without a human click, and the grounding filter refuses any
427
+ product claim the origin sessions' evidence ledger did not see. Events and the
428
+ promotion log carry ids, refs and verdicts only — a skill body never enters
429
+ the stream.
430
+
370
431
  ## See also
371
432
 
372
433
  - [Agents](AGENTS.md) — the five access layers per agent.
data/docs/SKILLS.md CHANGED
@@ -219,8 +219,8 @@ Write one with `agent:`, and remove it the same way (which un-specializes, leavi
219
219
  the shared skill in place):
220
220
 
221
221
  ```ruby
222
- dispatch(:write_skill, { name: "escalation-to-human", agent: "store-cacau", content: md })
223
- dispatch(:delete_skill, { name: "escalation-to-human", agent: "store-cacau" })
222
+ dispatch(:write_skill, { name: "escalation-to-human", agent: "store-demo", content: md })
223
+ dispatch(:delete_skill, { name: "escalation-to-human", agent: "store-demo" })
224
224
  ```
225
225
 
226
226
  In the Studio: **Skills → specialize for this agent**, which seeds the override from
data/docs/SOAK.md ADDED
@@ -0,0 +1,127 @@
1
+ ---
2
+ title: Soak
3
+ parent: Operate & prove it
4
+ nav_order: 4
5
+ permalink: /soak/
6
+ ---
7
+
8
+ # SOAK — does it degrade over uptime?
9
+
10
+ The soak is the third question the measurement toolbox answers, and it is the
11
+ one the other two cannot:
12
+
13
+ | Tool | Question | Drives |
14
+ |------|----------|--------|
15
+ | [`scripts/bench.rb`](BENCHMARK.md) | How much overhead does the engine itself add? | publishable claims |
16
+ | [Load test](LOADTEST.md) | What does a burst look like, end to end? | topology + provider choice |
17
+ | **`insika soak`** | **Does the deploy degrade over 72 hours of steady load?** | the cut argument: "it does not tax uptime" |
18
+
19
+ A load test fires a wave, joins it, and reports. A soak **sustains a declared
20
+ arrival rate for days** and asks whether the *process itself* — resident memory,
21
+ per-turn local work, restarts — degrades with uptime. A leak that only shows at
22
+ hour 40 is invisible to a 10-minute bench.
23
+
24
+ ## The protocol in one paragraph
25
+
26
+ 1. **Freeze the envelope first.** A deployment-side envelope file declares the
27
+ load shape, the gated ceilings and the isolation contract — *before* the
28
+ first turn is fired. Its SHA-256 is stamped into every hourly snapshot;
29
+ editing it mid-run turns the run `invalid` instead of producing a verdict.
30
+ 2. **Preflight.** `insika soak --preflight` refuses to start unless the target
31
+ answers `/v1/vitals` with a readable RSS, turns carry the `INSIKA_TURN_TIMING`
32
+ breakdown *and* a `usage` block, the pid is stable, and the target host
33
+ matches the envelope. Each check is a named refusal, not a runtime warning.
34
+ 3. **Run.** `insika soak --run` fires the arrival process (Poisson, seeded,
35
+ concurrency-capped), polls vitals hourly, and appends every observation to
36
+ `<out>/<run>.jsonl` as it happens — a runner that dies at hour 60 must
37
+ leave 60 usable hours behind. No retry, no self-healing: a failed turn is
38
+ evidence.
39
+ 4. **Read the verdict.** `insika soak --verify <file>.jsonl.gz` recomputes the
40
+ whole verdict offline from the raw records — no network, no target, no
41
+ clock. Two people running it get the same answer; so does CI.
42
+
43
+ ## What is gated, and what is only reported
44
+
45
+ The gate is on the part **Insika owns**:
46
+
47
+ | Gated (binding) | Reported (never gated) |
48
+ |-----------------|------------------------|
49
+ | `rss_growth_ratio` — upper 95% bound of the RSS fit, hour 6 → hour 72 | `rss_slope_mb_per_day`, `rss_peak_mb` |
50
+ | `prep_p95_drift_ratio` — p95 of the local per-turn work (`prep_ms`: context build, policy, guardrails, chat assembly), last 6 h vs first measured 6 h | `ttft_p95_ms`, `total_p95` hourly series |
51
+ | `restarts_max` — any `boot_id` **or** `pid` change is a fail; there is no "declared deploy" exception | `heap_growth_ratio`, `db_growth_mb` |
52
+ | `error_rate_ceiling`, `no_usage_rate_ceiling` | `cost_usd`, `tokens_per_turn` |
53
+ | calibrated absolute ceilings (written from the E1 dry run, ×2) | — |
54
+
55
+ Time-to-first-token is **reported, never gated**: it was measured at ~720 ms and
56
+ shown to be ~100% provider. Gating on a number the engine cannot move would mean
57
+ failing a clean soak because the provider had a bad afternoon — and then
58
+ loosening the envelope, which the protocol forbids.
59
+
60
+ The **`no_usage_rate`** ceiling is the trap detector. The edge limiter answers a
61
+ breach with a canned reply, zero LLM calls and HTTP 200 — so a soak that ran
62
+ entirely into the limiter looks like thousands of successful turns with a
63
+ gorgeous 2 ms p95 and flat memory. A turn with no `usage` block called no model;
64
+ the ceiling keeps that from reading as a pass.
65
+
66
+ ## The isolation contract
67
+
68
+ - **Staging or a dedicated tenant, never a production tenant.** The run writes
69
+ ~4,000 sessions and tasks into the store; against production that is
70
+ synthetic data mixed into real customer history. The envelope names the
71
+ tenant (`soak`), so `delete_tenant_data` cleans the entire run in one
72
+ command.
73
+ - **Its own agent, with the edge limits off.** At a real ~48k tokens per turn,
74
+ the production token ceiling admits about 10 turns per hour — a 60 turns/h
75
+ soak against a production-configured agent measures its own limiter within
76
+ minutes. `chat_rate_limit: 0` and `agent_token_ceiling: 0` on the soak agent
77
+ are part of the frozen envelope, not an afternoon decision.
78
+ - **Deploys are frozen for the window.** Any restart is a fail, including a
79
+ deployment — a marker the operator writes after the fact would turn every
80
+ crash into a deploy.
81
+
82
+ ## Running it
83
+
84
+ ```bash
85
+ # the plan + one sample request, no traffic
86
+ insika soak --dry-run --envelope soak-envelope.md
87
+
88
+ # every precondition, and nothing else
89
+ insika soak --preflight --envelope soak-envelope.md
90
+
91
+ # the run itself (INSIKA_URL + OPENCLAW_GATEWAY_TOKEN, like loadtest.rb)
92
+ INSIKA_URL=https://<target> insika soak --run --envelope soak-envelope.md --out soak-out/
93
+
94
+ # resume after a short outage (the gap is recorded and counts against the window)
95
+ insika soak --run --resume soak-out/staging-2026-08-20T09-00-00Z.jsonl
96
+
97
+ # the verdict, recomputed offline — the only step that needs no target
98
+ insika soak --verify soak-out/staging-2026-08-20T09-00-00Z.jsonl.gz
99
+ ```
100
+
101
+ Duration, rate and ceilings all come from the envelope — there is no
102
+ `--duration` override, so a short run cannot be passed off as a soak.
103
+
104
+ ## Reading the verdict
105
+
106
+ `insika soak --verify` prints one of four, in this order of precedence:
107
+
108
+ - **`invalid`** — the snapshots carry more than one envelope hash, more than
109
+ one process id appeared under one boot generation, or the file is materially
110
+ truncated. **No metric is reported at all**; the file cannot be argued with.
111
+ - **`insufficient`** — coverage is short: a gap over 15 minutes, an hour under
112
+ the turn floor, or a runner that did not reach hour 72. Short coverage
113
+ restarts the window; it is never rounded up into a pass.
114
+ - **`fail`** — a gated ceiling was breached. The report names each one and
115
+ prints the leak-hunt starting point: whether the Ruby heap grew with RSS
116
+ (Ruby-side retention) or stayed flat while RSS climbed (allocator or a
117
+ native buffer).
118
+ - **`pass`** — none of the above.
119
+
120
+ A fail means *find the leak* — never cut, and never loosen the envelope.
121
+
122
+ ## See also
123
+
124
+ - [Observability](OBSERVABILITY.md) — `GET /v1/vitals`, the process readings
125
+ the soak samples hourly.
126
+ - [Deploy](DEPLOY.md) — the process model; `WEB_CONCURRENCY=1` is a soak
127
+ precondition, not only a queue-semantics one.
data/docs/TOOLS.md CHANGED
@@ -91,8 +91,10 @@ retries against. Structure is strict; a scalar may arrive in its lossless string
91
91
 
92
92
  - `{{param}}` — a declared top-level parameter, filled from the model's call.
93
93
  - `{{ctx.*}}` — turn context set **server-side, never by the model**: a closed set
94
- of `chat_id`, `store_id`, `agent_id`, `tenant`. This is how a tool knows *which*
95
- session/agent it is acting for without trusting the model.
94
+ of `chat_id`, `store_id`, `agent_id`, `tenant`, `image_url`. This is how a tool knows *which*
95
+ session/agent it is acting for without trusting the model. `image_url` is the
96
+ first image part on the message (a photo for analysis outside the prompt);
97
+ absent when the turn carried none.
96
98
  - `{{secret.*}}` — allowed **only** inside a header named in `secret_headers`.
97
99
  A secret placeholder anywhere else is rejected (it would leak unmasked). The
98
100
  real secret value is injected at provision time and never lives on disk.
@@ -181,6 +183,72 @@ backend, not of whoever calls it. Every agent sharing the tool gets the same val
181
183
  > there **preserves** it — the form carries the stored values through instead of
182
184
  > replacing the record with only what it shows.
183
185
 
186
+ ## Evidence: the lean envelope and grounding (RFC-0029)
187
+
188
+ A catalog tool returns products; the model should only ever quote the ones the tool
189
+ actually returned — the store dies of a SKU the model invented. `evidence` is the
190
+ declaration that makes "no claim without a tool ID" an engine rule instead of a
191
+ prompt convention. One declaration does **both** jobs: the engine strips the result
192
+ down to what the model sees (the lean envelope) **and** records every returned id on
193
+ the session's evidence ledger. There is no "lean but not evidence" mode.
194
+
195
+ ```jsonc
196
+ { "name": "search_products",
197
+ "response": { "extract": "evidence_envelope" },
198
+ "evidence": "products" } // bare kind
199
+
200
+ { "evidence": { "kind": "products", // full form
201
+ "items": "results", // non-default paths
202
+ "attachments": "cards" } }
203
+ ```
204
+
205
+ - `evidence_envelope` is the canonical extract: the raw response body arrives under
206
+ an engine-only key, the envelope parses `items`/`attachments` out of it, and
207
+ **nothing re-fattens** — the transcript and the tool trace record only the lean
208
+ result. It **requires** the `evidence` declaration (refused at load otherwise).
209
+ - **Wire contract** — the lean result the model sees is always
210
+ `{ "items": [ { "id": "…", "line": "…" } ] }` (≤ 16 items; `line` truncated to
211
+ 200 chars). A tool whose result has no valid items yields `{ "items": [] }`,
212
+ never a null. A malformed evidence result becomes `{ "error": … }` back to the
213
+ model — a correctable tool answer, exactly like a malformed call.
214
+ - **Attachments** are the optional second half: `[{ "type": "card"|"image",
215
+ "url": "…", "caption": "…" }]` (≤ 16, url ≤ 500 chars, malformed dropped). They
216
+ **never** reach the model context or the transcript — they ride the channel
217
+ delivery as an additive `attachments` key on the outbox payload, and the channel
218
+ (or its consumer) decides what a card looks like.
219
+ - A **code tool** opts in the same way: it either returns `{ items, attachments }`
220
+ directly and declares `evidence` in its registry metadata, or exposes an
221
+ `evidence` reader. No declaration = today's tool behavior, byte for byte.
222
+
223
+ ### Grounding: policing claims against the ledger
224
+
225
+ With the ledger fed, the pack declares how claims are policed — data on the agent,
226
+ not a separate code path (see [Agents](AGENTS.md)):
227
+
228
+ ```ruby
229
+ grounding mode: :flag, matcher: { sku: '\b[A-Z]{2,4}\d{4,8}\b' }
230
+ ```
231
+
232
+ - `mode` is `flag` (the default — audit), `enforce` (cut), or `off`. Absent = off.
233
+ - `matcher.sku` is a regex for the store's SKU shape, applied to the final answer;
234
+ every match that is **not** in the evidence ledger is an ungrounded claim.
235
+ Grounding is **SKU-only** by design: a name-based half cannot flag anything
236
+ without a "this is a product name" signal, so the ledger grounds ids, and the
237
+ model quoting a returned product by its *name* is simply outside the check
238
+ (the SKU path is the claim detector). A `sku` that does not compile is refused
239
+ at build; a matcher with no `sku` builds but matches nothing — `insika doctor`
240
+ warns about it.
241
+ - **`flag`** appends an `:ungrounded` flag (category `ungrounded`, source
242
+ `evidence`) to the existing `:guardrail_flagged` event — audit after the fact,
243
+ like every other output flag.
244
+ - **`enforce`** *cuts the sentence* containing an ungrounded claim from the content
245
+ the turn persists and delivers, and the flag carries `action: "cut"` so the audit
246
+ can tell a cut from a flag. It is honest about streaming: on a streaming surface
247
+ the already-streamed bytes are the channel's reality, which is exactly why the
248
+ default is `flag` — ship `enforce` only after a matcher audit proves precision.
249
+ - Grounding is **independent of the guardrails opt-in**: an agent with guardrails
250
+ off and `grounding.mode: :flag` still gets the check.
251
+
184
252
  ## Registering a tool
185
253
 
186
254
  A tool appears in the Studio panel and enters an agent's tool-loop when it is
data/docs/domain.md ADDED
@@ -0,0 +1,115 @@
1
+ ---
2
+ title: The domain-free core
3
+ parent: Operate & prove it
4
+ nav_order: 9
5
+ permalink: /domain/
6
+ ---
7
+
8
+ # The domain-free core — what ships, what a deployment declares, and how to clear it
9
+
10
+ The engine is domain-free by construction (RFC-0036): the gem carries no store
11
+ vocabulary, no persona, and no fixed conversation language. This page is the
12
+ removability map — for every artifact that could make a deployment look like
13
+ "the Brazilian store harness", here is what ships, what the doctor reports, and
14
+ how to clear it.
15
+
16
+ ## What ships and what does not
17
+
18
+ One selection function owns the gem payload: `Insika::Packaging.payload_files`
19
+ — the gemspec and the audit spec (`spec/insika/domain_boundary_spec.rb`) call
20
+ the same function, so the boundary is a fact the suite asserts on, never a
21
+ prose promise. It ships:
22
+
23
+ - `lib/` — the engine, the server, the Studio (the compiled JS only);
24
+ - `docs/` — the public documentation;
25
+ - `README.md`, `LICENSE`, `CHANGELOG.md`, `bin/insika`.
26
+
27
+ It never ships: `deploy/`, `packs/`, `examples/`, `plugins/`, `evals/`,
28
+ `scripts/`, `spec/` — even a tracked pack cannot ship. Packs, personas and
29
+ deployment wiring are checkout material, and that is the point: the reference
30
+ deployment is not the gem.
31
+
32
+ The proof command is `insika doctor --domain`: a read-only inventory of what a
33
+ deployment declares, plus the built-in corpora still in effect. A bare install
34
+ with no agents names nothing; an install that boots an agent reports the
35
+ built-in pt-BR guardrail corpus as `source: gem-default` — the removability
36
+ surface, not a store. The section never fails the doctor's exit code: it is
37
+ informational, the RFC's proof surface.
38
+
39
+ ## The guardrail corpus — clear the shipped pt-BR content
40
+
41
+ The deterministic guardrail patterns ship as language-tagged data
42
+ (`Safety::Corpus`). A deployment clears a language by config, per agent:
43
+
44
+ ```jsonc
45
+ // pack: agent.config.json
46
+ { "guardrails": { "corpora": { "languages": ["en"] } } }
47
+ ```
48
+
49
+ ```ruby
50
+ # DSL
51
+ agent = Insika.agent("store-support") do
52
+ instructions "…"
53
+ guardrails corpora: { languages: ["en"] }
54
+ end
55
+ ```
56
+
57
+ - `languages` filters the shipped families: `nil` = all (the default),
58
+ `[]` = none, `["en"]` = the EN-only corpus. An EN-only deployment drops the
59
+ pt-BR input heuristics **and** the CPF/CNPJ output redaction — a documented
60
+ consequence: the tax-id formats are pt-BR data, and the universal "secret"
61
+ redaction is never cleared.
62
+ - `extra` adds source-string patterns per family:
63
+
64
+ ```jsonc
65
+ { "guardrails": { "corpora": { "extra": { "abuse": ["/\\bdupa\\b/i"] } } } }
66
+ ```
67
+
68
+ - `guardrails.responses` replaces the built-in pt-BR fallback replies (the
69
+ safe refusals). Set a `default` to clear every category at once.
70
+
71
+ See [Security](SECURITY.md) for the guardrail layers; the doctor's domain
72
+ section enumerates any agent still running the built-in pt-BR corpus or the
73
+ built-in replies, with the clear path in each entry. An unknown language or
74
+ family, or a malformed pattern, is refused at boot — `insika doctor` reports it
75
+ as an error and the agent never turns with a broken corpus.
76
+
77
+ ## Marking a deployment — declare, never infer
78
+
79
+ The engine never guesses a store. Domain markers are data a deployment
80
+ declares, and the doctor reads only those declarations:
81
+
82
+ - **Personas/packs** — `metadata.domain` on the agent profile:
83
+
84
+ ```jsonc
85
+ { "metadata": { "domain": "e-commerce-pt-BR" } }
86
+ ```
87
+
88
+ ```ruby
89
+ metadata domain: "e-commerce-pt-BR"
90
+ ```
91
+
92
+ - **Outcome funnel** — `funnel:` on the profile (see
93
+ [Outcomes](AGENTS.md#outcomes--business-results-over-real-traffic-ws7)).
94
+ Vocabulary note: in the gem and the doctor output it is an **outcome
95
+ funnel**, never "conversion" — the stage names are the deployment's, and a
96
+ bare install shows no funnel and no stage names at all.
97
+ - **Evidence** — the `evidence:` declaration on a tool manifest (see
98
+ [Tools](TOOLS.md)); the kinds are the deployment's vocabulary, never gem
99
+ constants.
100
+
101
+ `insika doctor --domain` enumerates all four with their source —
102
+ `deployment` for declared data, `gem-default` for the built-in corpus still in
103
+ effect — and a bare boot names none.
104
+
105
+ ## The conformance claim — model-visible means logged
106
+
107
+ "Every byte that reaches the provider is reconstructable from checkpoints +
108
+ traces" is a spec, not a promise. For each (task, turn) the engine records the
109
+ model-visible payload at the provider boundary — the system text, the tool
110
+ schemas, and the full message stream (`ModelVisibleTraceStore`) — next to the
111
+ durable transcript (the checkpoint). The conformance suite
112
+ (`spec/insika/conformance/model_visible_spec.rb`) drives real turns on a
113
+ capturing chat and asserts a three-way byte identity: what the chat held ==
114
+ the checkpoint transcript == the model trace. A path whose bytes are not
115
+ logged is a fix in the engine, never a waiver in the suite.
data/docs/index.md CHANGED
@@ -36,7 +36,7 @@ require "insika"
36
36
  assistant = Insika.agent("assistant") do
37
37
  model "deepseek-v4-flash"
38
38
  provider :deepseek
39
- instructions "You are Bia, a concise and friendly assistant. Answer briefly."
39
+ instructions "You are a concise and friendly assistant. Answer briefly."
40
40
  end
41
41
 
42
42
  puts assistant.reply("hi, what can you do?") # one turn, in-process