insika 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (185) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +145 -17
  3. data/README.md +7 -1
  4. data/bin/insika +211 -8
  5. data/docs/AGENTS.md +377 -2
  6. data/docs/API.md +56 -0
  7. data/docs/CHANNELS.md +99 -9
  8. data/docs/CONTEXT.md +127 -16
  9. data/docs/DEPLOY.md +5 -2
  10. data/docs/EVALS.md +19 -0
  11. data/docs/FACTS.md +135 -0
  12. data/docs/HARVEST.md +117 -0
  13. data/docs/LOADTEST.md +14 -8
  14. data/docs/OBSERVABILITY.md +54 -0
  15. data/docs/REFINEMENT.md +3 -3
  16. data/docs/RELEASING.md +27 -0
  17. data/docs/RUNNING-LOCAL.md +3 -3
  18. data/docs/SECURITY.md +61 -0
  19. data/docs/SKILLS.md +2 -2
  20. data/docs/SOAK.md +127 -0
  21. data/docs/TOOLS.md +70 -2
  22. data/docs/domain.md +115 -0
  23. data/docs/index.md +1 -1
  24. data/lib/insika/agent_profile.rb +145 -10
  25. data/lib/insika/balloon_splitter.rb +102 -0
  26. data/lib/insika/budget_ledger.rb +34 -3
  27. data/lib/insika/cache_series_store.rb +49 -0
  28. data/lib/insika/channel_delivery.rb +118 -13
  29. data/lib/insika/channels/relay.rb +77 -3
  30. data/lib/insika/channels/web.rb +2 -2
  31. data/lib/insika/chat_builder.rb +84 -1
  32. data/lib/insika/checkpoint_store.rb +16 -0
  33. data/lib/insika/commands/agent_payload.rb +3 -3
  34. data/lib/insika/commands/cancel_followup.rb +49 -0
  35. data/lib/insika/commands/delete_tenant_data.rb +95 -0
  36. data/lib/insika/commands/export_customer_memory.rb +48 -0
  37. data/lib/insika/commands/forget_customer.rb +117 -0
  38. data/lib/insika/commands/freeze_funnel_baseline.rb +113 -0
  39. data/lib/insika/commands/gate_harvest.rb +138 -0
  40. data/lib/insika/commands/gate_refinement.rb +1 -1
  41. data/lib/insika/commands/judge_shadow_pairs.rb +124 -0
  42. data/lib/insika/commands/memory_forget_fact.rb +20 -4
  43. data/lib/insika/commands/memory_put_fact.rb +23 -4
  44. data/lib/insika/commands/promote_harvest.rb +130 -0
  45. data/lib/insika/commands/record_outcome.rb +46 -0
  46. data/lib/insika/commands/record_shadow_reply.rb +68 -0
  47. data/lib/insika/commands/reject_harvest.rb +38 -0
  48. data/lib/insika/commands/resolve_proposal.rb +108 -0
  49. data/lib/insika/commands/revoke_contact.rb +49 -0
  50. data/lib/insika/commands/rollback_harvest.rb +86 -0
  51. data/lib/insika/commands/run_distillation.rb +186 -0
  52. data/lib/insika/commands/run_harvest.rb +393 -0
  53. data/lib/insika/commands/send_message.rb +103 -6
  54. data/lib/insika/commands/session_purge.rb +67 -0
  55. data/lib/insika/contact_store.rb +183 -0
  56. data/lib/insika/context/builder.rb +21 -3
  57. data/lib/insika/context/fragment.rb +7 -3
  58. data/lib/insika/context/priority.rb +3 -0
  59. data/lib/insika/context/provider.rb +17 -3
  60. data/lib/insika/context/providers/briefing.rb +96 -0
  61. data/lib/insika/context/providers/memory.rb +16 -7
  62. data/lib/insika/context/providers/prompt.rb +30 -2
  63. data/lib/insika/context/providers/skill.rb +2 -0
  64. data/lib/insika/context/providers/tool_search.rb +2 -0
  65. data/lib/insika/context_trace_store.rb +38 -2
  66. data/lib/insika/distill.rb +224 -0
  67. data/lib/insika/distill_engine.rb +169 -0
  68. data/lib/insika/doctor.rb +716 -6
  69. data/lib/insika/dsl/runtime.rb +9 -3
  70. data/lib/insika/dsl/server_boot.rb +70 -1
  71. data/lib/insika/dsl.rb +83 -3
  72. data/lib/insika/edge_limiter.rb +28 -7
  73. data/lib/insika/env_schema.rb +9 -2
  74. data/lib/insika/errors.rb +11 -0
  75. data/lib/insika/evals/assertions.rb +3 -2
  76. data/lib/insika/evals/pairwise.rb +18 -6
  77. data/lib/insika/evidence.rb +183 -0
  78. data/lib/insika/executor.rb +687 -70
  79. data/lib/insika/followup_engine.rb +207 -0
  80. data/lib/insika/followup_policy.rb +221 -0
  81. data/lib/insika/followup_store.rb +306 -0
  82. data/lib/insika/funnel_declaration.rb +106 -0
  83. data/lib/insika/funnel_fold.rb +179 -0
  84. data/lib/insika/funnel_store.rb +163 -0
  85. data/lib/insika/golden_store.rb +1 -1
  86. data/lib/insika/grounding/matcher.rb +69 -0
  87. data/lib/insika/grounding.rb +44 -0
  88. data/lib/insika/harvest/conversion_gate.rb +159 -0
  89. data/lib/insika/harvest/criterion.rb +98 -0
  90. data/lib/insika/harvest/gate.rb +194 -0
  91. data/lib/insika/harvest/negative_list.rb +199 -0
  92. data/lib/insika/harvest.rb +241 -0
  93. data/lib/insika/harvest_engine.rb +193 -0
  94. data/lib/insika/harvest_store.rb +548 -0
  95. data/lib/insika/media.rb +298 -0
  96. data/lib/insika/memory_audit_store.rb +85 -0
  97. data/lib/insika/memory_store.rb +264 -23
  98. data/lib/insika/message_origin.rb +6 -1
  99. data/lib/insika/model_visible.rb +87 -0
  100. data/lib/insika/model_visible_trace_store.rb +66 -0
  101. data/lib/insika/onboarding.rb +6 -1
  102. data/lib/insika/outbox_store.rb +42 -4
  103. data/lib/insika/outcome_store.rb +147 -0
  104. data/lib/insika/packaging.rb +163 -0
  105. data/lib/insika/parity/criterion.rb +79 -0
  106. data/lib/insika/parity/verdict.rb +318 -0
  107. data/lib/insika/prefix_fingerprint.rb +58 -0
  108. data/lib/insika/profile_source.rb +22 -1
  109. data/lib/insika/proposal_store.rb +271 -0
  110. data/lib/insika/queue_policy.rb +4 -1
  111. data/lib/insika/refinement/proposer.rb +1 -1
  112. data/lib/insika/reliability.rb +32 -6
  113. data/lib/insika/retention.rb +281 -0
  114. data/lib/insika/routing.rb +101 -0
  115. data/lib/insika/safety/config.rb +44 -4
  116. data/lib/insika/safety/corpus.rb +255 -0
  117. data/lib/insika/safety/detectors.rb +32 -113
  118. data/lib/insika/safety/factory.rb +15 -2
  119. data/lib/insika/safety/grounding_enforcer.rb +59 -0
  120. data/lib/insika/safety/grounding_validator.rb +49 -0
  121. data/lib/insika/safety/input_guardrail.rb +1 -1
  122. data/lib/insika/safety/moderator.rb +1 -1
  123. data/lib/insika/safety/output_filter.rb +9 -5
  124. data/lib/insika/safety/output_validator.rb +11 -5
  125. data/lib/insika/schema_guard.rb +35 -0
  126. data/lib/insika/server/app.rb +144 -4
  127. data/lib/insika/server/rack_app.rb +16 -0
  128. data/lib/insika/server/responses.rb +40 -1
  129. data/lib/insika/session_actor.rb +8 -4
  130. data/lib/insika/session_store.rb +65 -2
  131. data/lib/insika/settings_store.rb +10 -0
  132. data/lib/insika/shadow_pair_store.rb +258 -0
  133. data/lib/insika/skill_catalog.rb +4 -0
  134. data/lib/insika/soak/envelope.rb +140 -0
  135. data/lib/insika/soak/report.rb +392 -0
  136. data/lib/insika/soak/runner.rb +554 -0
  137. data/lib/insika/store.rb +10 -1
  138. data/lib/insika/stores/memory.rb +6 -0
  139. data/lib/insika/stores/sqlite.rb +8 -0
  140. data/lib/insika/studio/app.rb +875 -14
  141. data/lib/insika/studio/assets/dist/application.css +1 -1
  142. data/lib/insika/studio/assets/dist/application.js +22 -22
  143. data/lib/insika/studio/assets/dist/favicon.svg +6 -0
  144. data/lib/insika/studio/forms.rb +262 -5
  145. data/lib/insika/studio/nav_icons.rb +6 -1
  146. data/lib/insika/studio/views/agent_detail.erb +627 -84
  147. data/lib/insika/studio/views/agents.erb +10 -6
  148. data/lib/insika/studio/views/approvals.erb +4 -1
  149. data/lib/insika/studio/views/chats.erb +4 -1
  150. data/lib/insika/studio/views/customer.erb +94 -0
  151. data/lib/insika/studio/views/customers.erb +32 -0
  152. data/lib/insika/studio/views/evals.erb +4 -1
  153. data/lib/insika/studio/views/facts.erb +133 -0
  154. data/lib/insika/studio/views/followups.erb +125 -0
  155. data/lib/insika/studio/views/funnel.erb +106 -0
  156. data/lib/insika/studio/views/harvest.erb +234 -0
  157. data/lib/insika/studio/views/home.erb +2 -1
  158. data/lib/insika/studio/views/layout.erb +1 -0
  159. data/lib/insika/studio/views/parity.erb +147 -0
  160. data/lib/insika/studio/views/playground.erb +7 -1
  161. data/lib/insika/studio/views/session.erb +56 -1
  162. data/lib/insika/studio/views/settings.erb +2 -0
  163. data/lib/insika/studio/views/skills.erb +0 -1
  164. data/lib/insika/studio/views/system_files.erb +1 -1
  165. data/lib/insika/studio/views/task.erb +13 -0
  166. data/lib/insika/studio/views/tasks.erb +4 -1
  167. data/lib/insika/studio/views/tools.erb +0 -1
  168. data/lib/insika/task_store.rb +21 -1
  169. data/lib/insika/testing/store_contract.rb +27 -6
  170. data/lib/insika/tick.rb +25 -1
  171. data/lib/insika/tool_definition.rb +17 -7
  172. data/lib/insika/tool_envelope.rb +69 -0
  173. data/lib/insika/tool_manifest.rb +5 -1
  174. data/lib/insika/tools/data_defined_tool.rb +10 -0
  175. data/lib/insika/tools/generate_image.rb +44 -0
  176. data/lib/insika/tools/schedule_followup.rb +164 -0
  177. data/lib/insika/tools/tts.rb +47 -0
  178. data/lib/insika/tools/update_briefing.rb +126 -0
  179. data/lib/insika/turn_state.rb +38 -1
  180. data/lib/insika/turn_timing.rb +22 -2
  181. data/lib/insika/version.rb +1 -1
  182. data/lib/insika/vitals.rb +84 -0
  183. data/lib/insika/wiring/graph.rb +300 -13
  184. data/lib/insika.rb +92 -3
  185. metadata +77 -1
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 9e7913378bd8be19a7d9422524e62c6786c2abc8d26b5194ef8b1db8fef82be5
4
- data.tar.gz: f1a279b7284d9756d7ad3dcef56329314062ef77c0fbb3d24df98e10c9bcb4ce
3
+ metadata.gz: 9ac198fc4ba91d6f3d3b55ebdf782b326bee455d45be766befaff3fc761009ff
4
+ data.tar.gz: e315b3f30cbe8365bbb3c6db6477209935ac71f5d0cadca26b6a5bc3458fa1cb
5
5
  SHA512:
6
- metadata.gz: dcbf5403a982de5ee9dc8ed0c5fde2b21b5b63667cde0b47d2b9985b99fe7a9708d53fb82d43ce8464f46c38989d5b2b261156a84c8245e16199017515d24191
7
- data.tar.gz: 349d532c16f5baed05fcd79318f3f68056e40f2a931e5e33ff12c339791edaa6fcdfe811d419cf3aff490118bd46698486470f234d9e820f0e568bb875e8b2cc
6
+ metadata.gz: 985b9924a6996aeb9a180ae6c800191029aaaad48ea55c18ffb35e8df159b14fbbc801bf7fc7cc33dc2291c6f8e80ffba078578f715ee062410c0504579819d3
7
+ data.tar.gz: a3a7dd5ff765f9fd652b116f7a8b03c05d3fb29cc582e1a14508e5e0ee5afb088b2d755a194da9e378b262871f84a884adbbc18a8e5e65f8af36b49706b76047
data/CHANGELOG.md CHANGED
@@ -8,44 +8,172 @@ it is released. Entries land with the pull request that makes the change.
8
8
 
9
9
  ## [Unreleased]
10
10
 
11
- Nothing yet.
11
+ ## [0.3.0] - 2026-08-19
12
+
13
+ The proof-and-consultant wave: shadow parity against a frozen criterion, the
14
+ 72h soak, perceived latency (progressive delivery), the session briefing,
15
+ evidence grounding, the layered identity cache, customer memory in the Studio,
16
+ the conversion ledger, `schedule()` with consent, human-gated facts, the gated
17
+ skill harvest — and a domain-free core: the e-commerce defaults are removable,
18
+ and what ships in the gem is asserted by the suite.
19
+
20
+ ### Added
21
+
22
+ - **Shadow mode** — the relay channel can run every turn end to end and deliver
23
+ nothing: the incumbent keeps answering, the engine records what it *would*
24
+ have answered, and the two replies are judged pairwise against a **frozen,
25
+ pre-registered criterion** (the file `INSIKA_PARITY_CRITERION` points at; its
26
+ SHA-256 stamps every pair, and editing it mid-window turns the verdict
27
+ `invalid`, never stale). A panel of judges scores each pair twice with the
28
+ sides swapped, so a preference that flips with presentation order is recorded
29
+ as comparable, not as a preference. The Studio's Parity page folds the
30
+ running verdict on demand. No criterion, no shadow: boot refuses.
31
+ - **The 72h soak** — `insika soak` sustains a declared arrival rate for days and
32
+ asks whether the *process itself* degrades with uptime. A deployment-side
33
+ envelope freezes the load shape and the gated ceilings before the first hour
34
+ (its hash stamps every snapshot); `--preflight` refuses to start on a missing
35
+ precondition; the runner appends hourly vitals as they happen, so a crash at
36
+ hour 60 leaves 60 usable hours; `--verify` recomputes the whole verdict
37
+ offline from the raw records. A fail means *find the leak* — never cut, and
38
+ never loosen the envelope.
39
+ - **Progressive delivery** — the relay can flush the outbox one balloon per
40
+ paragraph (`INSIKA_RELAY_DELIVERY=progressive`), with the first balloon
41
+ posted as soon as the answer exists; the default `:at_end` is byte-identical
42
+ to before. Every channel turn records `first_balloon_ms` (inbound receipt →
43
+ first outbox flush) so the target is measured, not assumed.
44
+ - **Session briefing** — the pack can declare `briefing_fields`; the engine
45
+ keeps the session's working state as data (known fields, still-missing list,
46
+ next step), a tool updates it, and the context provider injects it into the
47
+ turn — the consultant stops re-asking what it already knows.
48
+ - **Grounding** — an `evidence:` contract on the tool envelope: a model claim
49
+ must cite ids that came out of a tool call, with `:enforce` / `:flag` / `:off`
50
+ modes and a per-turn ledger. A claim without evidence is refused or flagged —
51
+ never silently passed through.
52
+ - **Layered identity + observable cache** — context providers are split into
53
+ identity/volatile layers behind a byte-stable cache prefix; the per-agent
54
+ cache-hit series is recorded (`CacheSeriesStore`) and shown on the Studio
55
+ agent detail. Invalidations and hits are measurable, not guessed.
56
+ - **Customer memory in the Studio** — a Customers drill reads and edits the
57
+ per-customer memory cell, with an append-only operator-mutation audit trail
58
+ (content-free digests), provenance metadata, expiry, the LGPD export
59
+ (`/v1/commands/export_customer_memory`) and the existing right to be
60
+ forgotten.
61
+ - **The outcome funnel** — outcomes fold into a store-declared stage funnel
62
+ (`funnel:` on the pack) on the tick: idempotent cumulative counts, an
63
+ attribution window carried as data, a frozen baseline (`:freeze_funnel_baseline`)
64
+ and a Studio page per agent. The stage vocabulary is the deployment's; a bare
65
+ install shows no funnel at all.
66
+ - **Follow-ups** — the agent can book a future contact: `schedule()` and
67
+ `cancel_followup()` tools, a durable per-customer contact state
68
+ (`granted | revoked | unavailable`, where silence is not a refusal), a policy
69
+ engine over the tick (quiet hours, frequency ceilings, dedup, opt-out
70
+ keywords) and a Studio page per agent. A customer who opted out can never be
71
+ rescheduled; blocking happens at fire time, never at schedule time.
72
+ - **Distilled facts** — finished conversations are read back as proposed
73
+ customer facts; a human approves/rejects/dismisses each on the Studio Facts
74
+ page, with a latched dedup ledger (a dismissed tuple is never proposed again),
75
+ optimistic CAS writes and LGPD-friendly retention. Nothing is ever applied
76
+ automatically.
77
+ - **Gated skill harvest** — finished traffic can be mined for skill proposals
78
+ behind a versioned negative list and an evidence ledger (product claims must
79
+ reference ids the origin sessions actually saw); promotion requires the eval
80
+ replay **and** the store's conversion ruler not to regress, with an
81
+ append-only log and snapshot rollback. Nothing is ever applied automatically.
82
+ - **A domain-free core** — the gem payload is a spec-asserted boundary
83
+ (`lib/` + `docs/` + the four root files; `deploy/`, `packs/`, `examples/`,
84
+ `evals/`, `scripts/` and `spec/` never ship, even when tracked). The pt-BR
85
+ guardrail corpus is language-tagged removable data, and `insika doctor --domain`
86
+ inventories what a deployment declares. A bare install names no store.
87
+ - **Model-visible conformance** — every byte that reaches the provider is
88
+ reconstructable from checkpoints + traces, proven byte-for-byte by the
89
+ conformance suite (`spec/insika/conformance/`): what the chat held == the
90
+ checkpoint transcript == the model trace, across plain, tool-calling, steered,
91
+ subagent, scheduled and resuming turns.
92
+ - **Intent routing** — `AgentProfile#routes` classifies the turn's
93
+ message into one configured route with a cheap model before the ask, from a
94
+ prompt auto-generated out of the route descriptions. The route rides the
95
+ turn (`state.route`, the `:route_classified` event, the terminal event), its
96
+ provider cost is counted in the usage, and a route can `delegate` to an
97
+ existing agent (its answer becomes the parent's) or end the turn with the
98
+ stuck outcome (`stuck: true`). Deterministic `default` fallback;
99
+ classifier failure leaves the turn unrouted (additive).
100
+ - **Outcomes** — `POST /v1/outcomes` records a conversation's business
101
+ outcome (`conversion`/`escalation`/`deflected`/…, optional monetary value)
102
+ from the operator or the integration — additive, outside the response
103
+ contract, tenant-stamped. `GET /v1/outcomes` serves the last outcome
104
+ per agent + per-day series; the Studio's agent grid shows the last-outcome
105
+ pill and the agent detail shows the per-day series.
106
+ - **Customer-scoped memory + right to be forgotten** — a message
107
+ carrying a `customer` key moves the memory scope to the `[tenant:]customer`
108
+ cell: two customers under one tenant never read each other (phase 1), and
109
+ the `<request_context>` merchant label stays untouched. Facts gained an
110
+ optimistic CAS write (`replace_if_revision`, microsecond revisions).
111
+ `POST /v1/commands/forget_customer` (phase 2, LGPD) purges one customer's
112
+ memory cell, their sessions and per-session traces — nothing else's.
113
+ - **Tenant deletion + retention** —
114
+ `POST /v1/commands/delete_tenant_data` purges everything the engine holds
115
+ about one tenant (sessions, traces, every memory cell under the tenant and
116
+ its outcome records); the `retention_days` settings key turns on the tick's
117
+ daily age-based sweep (sessions + traces, terminal tasks + checkpoints,
118
+ memory cells and outcomes — OFF by default). The KV store contract gained
119
+ an additive `scopes(prefix)` enumeration.
120
+ - **Media in the message contract** — messages accept
121
+ additive content parts (`text`/`image`/`audio` with a URL). Audio is
122
+ transcribed via RubyLLM STT (model/language via `INSIKA_STT_MODEL`/
123
+ `INSIKA_STT_LANGUAGE`) and the text enters the turn marked
124
+ `source: "voice"` on the terminal event; images attach to the model's ask
125
+ (provider-billed, usage flows) and the first image URL is
126
+ `{{ctx.image_url}}` for data/HTTP tools; media URLs pass the egress guard. The
127
+ OpenAI multimodal `input` array shape works on `/v1/responses`.
128
+ - **Generated media as outputs** — the turn can produce an
129
+ image or a voice clip when BOTH gates agree: the agent opts in
130
+ (`AgentProfile#outputs` — per-kind model/voice/size config) and the request
131
+ declares the channel can receive it (`channel.capabilities`, one of
132
+ `image_output`/`audio_output`; unknown values are a 422 — the abstraction
133
+ admits only what leaks). The `generate_image`/`tts` system tools are wired
134
+ only then, the media rides the envelope additively (`output_parts` on the
135
+ terminal event and the `/v1/responses` completed frame — base64 parts, never
136
+ the answer text), image tokens merge into the turn's usage and every call
137
+ adds an honest `usage.media` counter (the speech API reports no tokens; the
138
+ part carries the model for consumer-side pricing). Generators are injectable
139
+ seams; the defaults are lazy (RubyLLM paint; a thin POST to the
140
+ OpenAI-compatible `/audio/speech` using the same provider config).
12
141
 
13
142
  ## [0.2.0] - 2026-08-13
14
143
 
15
144
  The workstreams between the first release and the one the gem actually became:
16
- multi-tenancy at the edge (WS1), calendar budgets (WS2), provider reliability
17
- (WS3), the stuck signal (WS5), operator alerts + live TTFB (WS6), and the
18
- failure-classification core (B9) — plus the two fix rounds that made them
19
- safe to ship.
145
+ multi-tenancy at the edge, calendar budgets, provider reliability, the stuck
146
+ signal, operator alerts + live TTFB, and the failure-classification core — plus
147
+ the two fix rounds that made them safe to ship.
20
148
 
21
149
  ### Added
22
150
 
23
- - **Multi-tenant at the edge (WS1)** — `INSIKA_TENANCY=multi_tenant` resolves the
151
+ - **Multi-tenant at the edge** — `INSIKA_TENANCY=multi_tenant` resolves the
24
152
  Bearer to a principal before the routes: per-tenant + operator tokens stored
25
153
  only as SHA-256 hashes, a tenant's sessions/tasks/streams living under its own
26
154
  `<tenant>:` namespace (fail-closed: another tenant's reads as `404`), and every
27
155
  authoring/config surface refused to a tenant.
28
- - **Calendar budgets (WS2)** — `AgentProfile#budget` caps the billed spend
156
+ - **Calendar budgets** — `AgentProfile#budget` caps the billed spend
29
157
  (input + output + cached + cache-creation) per calendar day/month and
30
158
  (tenant, agent): HARD (default) fails the turn with the typed
31
159
  `Insika::BudgetExceeded` + `retry_after`; `soft: true` runs the turn and warns
32
160
  once per window — with the `alert_at` (`0.8`) crossing and the real cap
33
161
  crossing as separate events.
34
- - **Reliability (WS3)** — retries with backoff, mid-turn rotation to the
162
+ - **Reliability** — retries with backoff, mid-turn rotation to the
35
163
  fallback chain, a per-`(tenant, provider/model)` circuit breaker with
36
164
  half-open trials (a failed trial reopens), and a per-attempt `timeout`
37
165
  (default 30s) counted as retryable. A `:fatal` provider error is never
38
166
  retried.
39
- - **Stuck signal (WS5)** — an agent declared stuck ends its turn with
167
+ - **Stuck signal** — an agent declared stuck ends its turn with
40
168
  `outcome: "stuck"` on the envelope and a dedicated `:turn_stuck` event — the
41
169
  deterministic point a consumer escalates on.
42
- - **Operator alerts + live TTFB (WS6)** — `budget_warning`, `breaker_open` and
170
+ - **Operator alerts + live TTFB** — `budget_warning`, `breaker_open` and
43
171
  `delivery_failed` POSTed to a per-agent `alerts.webhook` over the at-most-once
44
172
  outbox pipeline (boot-recoverable); under `INSIKA_TURN_TIMING` the first
45
173
  content chunk emits a live `:ttft` on the streaming envelope.
46
- - **Failure classification (B9)** — provider/transport failures classified by
174
+ - **Failure classification** — provider/transport failures classified by
47
175
  action (`:fatal` / `:retryable` / `:rate_limited_*`) and wrapped with the
48
- provider's `retry_after`; mechanical tool-output dedupe (C3) back-references a
176
+ provider's `retry_after`; mechanical tool-output dedupe back-references a
49
177
  byte-identical repeat only when the reference is genuinely shorter.
50
178
  - **The periodic tick** — durability no longer waits for a reboot. Serving
51
179
  workers run a tick every `INSIKA_TICK_INTERVAL` (default 60s, `0` disables)
@@ -58,7 +186,7 @@ safe to ship.
58
186
 
59
187
  ### Fixed
60
188
 
61
- - **WS2/WS3/WS6 criticals** — the budget alert marker no longer returns inside
189
+ - **Budget/reliability/alerts criticals** — the budget alert marker no longer returns inside
62
190
  the store transaction (a leaked `BEGIN IMMEDIATE` locked SQLite on the 2nd
63
191
  over-threshold turn); the monthly reset is December-safe and UTC-aligned; an
64
192
  unset reliability timeout is 30s, not 1s, and a timeout retries/rotates
@@ -66,10 +194,10 @@ safe to ship.
66
194
  webhook deliveries pass the egress guard (SSRF); `:ttft` is emitted once per
67
195
  turn; webhook channels pre-register so the boot sweep recovers pending alerts;
68
196
  the alert dispatcher subscribes typed and re-subscribes on overflow.
69
- - **WS1** — `#revoke` rides the store transaction; a `tenant_id` containing
197
+ - **Tenancy** — `#revoke` rides the store transaction; a `tenant_id` containing
70
198
  `:` is refused (the session-namespace delimiter); `POST /v1/sessions` mints a
71
199
  tenant's session under its own prefix.
72
- - **WS2/WS3 softs** — a failed turn's consumed tokens count against the budget;
200
+ - **Budget and reliability softs** — a failed turn's consumed tokens count against the budget;
73
201
  `:breaker_open` alerts only on the closed→open transition; the fallback chain
74
202
  dedupes `"model"` vs `"provider/model"` spellings.
75
203
 
@@ -143,7 +271,7 @@ The first release: `gem install insika`.
143
271
  - **The gate refuses to grade a judged baseline without a judge** — a rubric'd case
144
272
  with no verdict counts as a pass, so replaying without a judge against a baseline
145
273
  recorded with one does not measure less, it measures backwards. Found by running the
146
- panel against the real pilot: the gate reported *6/6, no regression* against a
274
+ panel against real traffic: the gate reported *6/6, no regression* against a
147
275
  baseline the same corpus had just scored *2/6*, and both candidates cleared. With the
148
276
  judge configured, the same two candidates were correctly rejected on judge-score
149
277
  drops. Third member of the same family as the missing and all-red baseline refusals.
@@ -151,7 +279,7 @@ The first release: `gem install insika`.
151
279
  input + output and excludes the cached prefix (a 27 KB pack reports `88` total against
152
280
  `26624` cached), so a ceiling built on it alone let a run send hundreds of times what
153
281
  it said. Cost now bills `total + cached` and records the cached share, which on a real
154
- panel run was 95% of the spend. A run that cannot be gated is also refused **before**
282
+ run was 95% of the spend. A run that cannot be gated is also refused **before**
155
283
  the proposal is paid for, not after.
156
284
  - **The eval baseline is a per-agent record, not only a file** — `evals/baseline.json`
157
285
  works from a checkout; the refinement gate runs inside a deployment that has none.
data/README.md CHANGED
@@ -22,6 +22,10 @@ deployment.
22
22
  - **Safe by default** — content-safety guardrails, an egress guard, confined execution, approvals, edge limits.
23
23
  - **Observable** — an event stream, per-session tool-call traces, optional OpenTelemetry.
24
24
  - **~0.4 ms of engine per turn** — p50 overhead on a neutral, key-free benchmark you can rerun yourself ([methodology](docs/BENCHMARK.md)).
25
+ - **Domain-free core** — the gem carries no store vocabulary, no persona and no fixed
26
+ conversation language: the built-in pt-BR guardrail corpus is removable data, a
27
+ deployment declares what it is, and the payload boundary is a spec the suite asserts
28
+ on ([the domain-free map](docs/domain.md)).
25
29
 
26
30
  ## Your first agent
27
31
 
@@ -37,7 +41,7 @@ require "insika"
37
41
  assistant = Insika.agent("assistant") do
38
42
  model "deepseek-v4-flash"
39
43
  provider :deepseek
40
- instructions "You are Bia, a concise and friendly assistant. Answer briefly."
44
+ instructions "You are a concise and friendly assistant. Answer briefly."
41
45
  end
42
46
 
43
47
  puts assistant.reply("hi, what can you do?") # one turn, in-process
@@ -109,6 +113,8 @@ markdown). Public and on by default when you `serve`; opt-in in production
109
113
  - [Load test](docs/LOADTEST.md) — load-testing and data topology.
110
114
  - [Evals](docs/EVALS.md) — the cases that grade an agent: rubrics, the judge panel, and the pre-merge gate.
111
115
  - [Refinement](docs/REFINEMENT.md) — read an agent's own traffic back as a ranked report of what broke.
116
+ - [Facts](docs/FACTS.md) — distill finished customer conversations into proposed facts; a human approves them into memory (nothing is ever applied automatically).
117
+ - [Harvest](docs/HARVEST.md) — mine real traffic for SKILL proposals; the negative list, the evidence-ledger grounding filter and the double gate hold every proposal to a human approval and the append-only log (nothing is ever applied automatically).
112
118
 
113
119
  All of the above is also browsable, searchable and cross-linked at
114
120
  **[guizaols.github.io/insika](https://guizaols.github.io/insika/)** — the same files,
data/bin/insika CHANGED
@@ -4,8 +4,9 @@
4
4
  # `insika` CLI — strict config + doctor --fix. The operator's
5
5
  # DX front door to the strict-config discipline:
6
6
  #
7
- # insika doctor [--fix] [--json] diagnose the deployment config; --fix applies
8
- # the safe autofixes; exits non-zero on any error
7
+ # insika doctor [--fix] [--json] [--domain] diagnose the deployment config; --fix applies
8
+ # the safe autofixes; --domain appends the domain inventory;
9
+ # exits non-zero on any error
9
10
  # insika env [--json] list the known config keys + current (masked) values
10
11
  # insika version print the engine version
11
12
  # insika help this text
@@ -27,6 +28,10 @@ module Insika
27
28
  when "doctor" then doctor(argv)
28
29
  when "env" then env(argv)
29
30
  when "refine" then refine(argv)
31
+ when "harvest" then harvest(argv)
32
+ when "harvest:negative" then harvest_negative_import(argv)
33
+ when "harvest:criterion" then harvest_criterion_check(argv)
34
+ when "soak" then exit(Insika::Soak::Runner.main(argv, stdout: $stdout, stderr: $stderr))
30
35
  when "evals:import" then evals_import(argv)
31
36
  when "evals:export" then evals_export(argv)
32
37
  when "evals:baseline" then evals_baseline(argv)
@@ -45,9 +50,15 @@ module Insika
45
50
  Usage: insika <command> [options]
46
51
 
47
52
  Commands:
48
- doctor [--fix] [--json] diagnose the deployment configuration
53
+ doctor [--fix] [--json] [--domain] diagnose the deployment configuration
49
54
  env [--json] list known config keys + current (masked) values
50
55
  refine --agent ID [...] report what broke in an agent's real traffic
56
+ harvest --agent ID [...] mine one window of finished traffic
57
+ harvest:negative import --agent ID --file F
58
+ seed the profile's negative list from a rules file
59
+ harvest:criterion check --file F
60
+ strict-load the frozen conversion criterion
61
+ soak --run | --verify FILE | --preflight the 72h soak
51
62
  evals:import [--dir D] load the golden corpus into the store (editable in Studio)
52
63
  evals:export [--dir D] write the stored cases back out as YAML
53
64
  evals:baseline <sub> the accepted state per agent: show | import | export
@@ -60,13 +71,14 @@ module Insika
60
71
  TXT
61
72
  end
62
73
 
63
- # insika doctor [--fix] [--json]
74
+ # insika doctor [--fix] [--json] [--domain]
64
75
  def doctor(argv)
65
- opts = { fix: false, json: false }
76
+ opts = { fix: false, json: false, domain: false }
66
77
  OptionParser.new do |o|
67
- o.banner = "Usage: insika doctor [--fix] [--json]"
78
+ o.banner = "Usage: insika doctor [--fix] [--json] [--domain]"
68
79
  o.on("--fix", "apply the safe autofixes, then re-diagnose") { opts[:fix] = true }
69
80
  o.on("--json", "emit the report as JSON") { opts[:json] = true }
81
+ o.on("--domain", "append the domain inventory (declared personas, corpora, funnels, evidence)") { opts[:domain] = true }
70
82
  end.parse!(argv)
71
83
 
72
84
  doc = build_doctor
@@ -74,9 +86,12 @@ module Insika
74
86
 
75
87
  if opts[:json]
76
88
  require "json"
77
- puts JSON.pretty_generate(report.to_h)
89
+ envelope = report.to_h
90
+ envelope["domain"] = doc.domain.to_h if opts[:domain]
91
+ puts JSON.pretty_generate(envelope)
78
92
  else
79
93
  puts report.to_s(color: $stdout.tty?)
94
+ puts doc.domain.to_s if opts[:domain]
80
95
  end
81
96
  exit(report.ok? ? 0 : 1)
82
97
  end
@@ -296,6 +311,153 @@ end
296
311
  }.compact, transport: :cli))
297
312
  end
298
313
 
314
+ # insika harvest --agent ID [--last-sessions N] [--since ISO8601] [--full] [--json]
315
+ #
316
+ # mines one window of the agent's finished traffic and records
317
+ # the candidates. Writes nothing but the run + candidates + markers (D2's
318
+ # fork discipline); a cron or the operator's own habit runs it.
319
+ def harvest(argv)
320
+ opts = { json: false, full: false }
321
+ OptionParser.new do |o|
322
+ o.banner = "Usage: insika harvest --agent ID [options]"
323
+ o.on("--agent ID", "agent whose traffic to mine (required)") { |v| opts[:agent] = v }
324
+ o.on("--last-sessions N", Integer, "mine the N most recent conversations") { |v| opts[:last_sessions] = v }
325
+ o.on("--since ISO8601", "mine only turns from this instant on") { |v| opts[:since] = v }
326
+ o.on("--full", "ignore the mined markers and re-read the window") { opts[:full] = true }
327
+ o.on("--json", "emit the summary as JSON") { opts[:json] = true }
328
+ end.parse!(argv)
329
+
330
+ abort "insika harvest: --agent is required" if Insika::Coercion.blank?(opts[:agent])
331
+
332
+ result = begin
333
+ run_harvest(opts)
334
+ rescue Insika::Error, ArgumentError => e
335
+ warn "insika harvest: #{e.message}"
336
+ exit 2
337
+ end
338
+ if opts[:json]
339
+ require "json"
340
+ puts JSON.pretty_generate(result)
341
+ else
342
+ puts harvest_report(result)
343
+ end
344
+ exit(1) if result[:mined] && result[:run_id] && run_failed?(result)
345
+ end
346
+
347
+ # insika harvest:negative import --agent ID --file F
348
+ #
349
+ # the seed path: reads the VERSIONED rules file (parse! — loud), and
350
+ # writes the rules into the profile's harvest.negative_list via UpdateAgent
351
+ # (the operative, hot-editable list the engine applies).
352
+ def harvest_negative_import(argv)
353
+ opts = {}
354
+ OptionParser.new do |o|
355
+ o.banner = "Usage: insika harvest:negative import --agent ID --file F"
356
+ o.on("--agent ID", "agent whose profile receives the rules (required)") { |v| opts[:agent] = v }
357
+ o.on("--file F", "the negative-list rules file (required)") { |v| opts[:file] = v }
358
+ end.parse!(argv)
359
+
360
+ abort "insika harvest:negative import: --agent is required" if Insika::Coercion.blank?(opts[:agent])
361
+ abort "insika harvest:negative import: --file is required" if Insika::Coercion.blank?(opts[:file])
362
+
363
+ rules = begin
364
+ list = Insika::Harvest::NegativeList.parse!(File.read(opts[:file]))
365
+ list.rules.map { |r| { "rule" => r.rule, "pattern" => r.pattern, "note" => r.note } }
366
+ rescue Insika::Error, Errno::ENOENT => e
367
+ warn "insika harvest:negative import: #{e.message}"
368
+ exit 2
369
+ end
370
+
371
+ backend = Insika::Wiring::Graph.backend_from_env
372
+ config_store = Insika::ConfigStore.new(store: backend)
373
+ profiles = Insika::StoredProfileSource.new(config_store: config_store)
374
+ profile = profiles.fetch(opts[:agent]) ||
375
+ (abort "insika harvest:negative import: agent '#{opts[:agent]}' not configured")
376
+ handler = Insika::Commands::UpdateAgent.new(profile_source: profiles,
377
+ event_stream: Insika::EventStream.new)
378
+ merged = (profile.harvest || {}).merge("negative_list" => rules)
379
+ handler.call(Insika::Command.build(:update_agent,
380
+ { id: opts[:agent], harvest: merged }, transport: :cli))
381
+ puts "imported #{rules.size} rule(s) into #{opts[:agent]}.harvest.negative_list"
382
+ end
383
+
384
+ # insika harvest:criterion check --file F
385
+ #
386
+ # strict-loads the frozen conversion criterion — a missing key, an unknown
387
+ # key or an absent file exits non-zero. The hook before any promotion is
388
+ # allowed.
389
+ def harvest_criterion_check(argv)
390
+ opts = {}
391
+ OptionParser.new do |o|
392
+ o.banner = "Usage: insika harvest:criterion check --file F"
393
+ o.on("--file F", "the frozen criterion (required)") { |v| opts[:file] = v }
394
+ end.parse!(argv)
395
+
396
+ abort "insika harvest:criterion check: --file is required" if Insika::Coercion.blank?(opts[:file])
397
+
398
+ criterion = Insika::Harvest::Criterion.load(opts[:file])
399
+ puts "criterion ok: #{criterion.rule.metric} / #{criterion.rule.window} " \
400
+ "threshold #{criterion.rule.threshold} min_span #{criterion.rule.min_span} " \
401
+ "(#{criterion.sha})"
402
+ 0
403
+ rescue Insika::Error => e
404
+ warn "insika harvest:criterion check: #{e.message}"
405
+ exit 1
406
+ end
407
+
408
+ def run_failed?(result)
409
+ backend = Insika::Wiring::Graph.backend_from_env
410
+ run = Insika::HarvestStore.new(store: backend).find_run(result[:run_id])
411
+ run && run.status == "failed"
412
+ end
413
+
414
+ # Builds the handler over the real durable backend — same "no app boot" rule as
415
+ # `doctor`: no DEEPSEEK, no seeding, no reactor.
416
+ def run_harvest(opts)
417
+ backend = Insika::Wiring::Graph.backend_from_env
418
+ config_store = Insika::ConfigStore.new(store: backend)
419
+ profiles = Insika::StoredProfileSource.new(config_store: config_store)
420
+ handler = Insika::Commands::RunHarvest.new(
421
+ profiles: profiles,
422
+ harvest_store: Insika::HarvestStore.new(store: backend),
423
+ session_store: Insika::SessionStore.new(store: backend),
424
+ task_store: Insika::TaskStore.new(store: backend),
425
+ skill_store: Insika::SkillStore.new(config_store: config_store),
426
+ settings_store: Insika::SettingsStore.new(config_store: config_store),
427
+ negative_list: begin
428
+ path = Insika::EnvSchema.read("INSIKA_HARVEST_NEGATIVE")
429
+ path && Insika::Harvest::NegativeList.parse(File.read(path))
430
+ rescue Errno::ENOENT
431
+ nil
432
+ end,
433
+ miner_factory: nil,
434
+ event_stream: Insika::EventStream.new
435
+ )
436
+ handler.call(Insika::Command.build(:run_harvest, {
437
+ agent: opts[:agent], since: opts[:since],
438
+ last_sessions: opts[:last_sessions], full: opts[:full]
439
+ }.compact, transport: :cli))
440
+ end
441
+
442
+ def harvest_report(result)
443
+ head = if result[:mined]
444
+ "harvest: #{result[:candidates]} candidate(s) — #{harvest_rejected(result[:rejected])}"
445
+ else
446
+ "harvest: skipped (#{result[:skipped]})"
447
+ end
448
+ if result[:cost] && result[:cost]["spent"].to_i.positive?
449
+ head += " — #{result[:cost]['spent']} tokens" \
450
+ "#{result[:cost]['cached'].to_i.positive? ? " (#{result[:cost]['cached']} cached)" : ''}"
451
+ end
452
+ head
453
+ end
454
+
455
+ def harvest_rejected(rejected)
456
+ return "nothing rejected" if rejected.nil? || rejected.values.all?(&:zero?)
457
+
458
+ rejected.reject { |_k, v| v.to_i.zero? }.map { |k, v| "#{k}: #{v}" }.join(", ")
459
+ end
460
+
299
461
  INDENT = " " * 23 # aligns the provenance line under the finding's title
300
462
 
301
463
  def refine_report(run)
@@ -337,10 +499,51 @@ end
337
499
  # a skill and the agents holding it, so neither half alone is enough. The
338
500
  # catalog brings the DISK seeds into the sweep (deploy/skills in a deployment,
339
501
  # skills/ in the minimal wiring — whichever exists here); without it a stale
340
- # `eager:` or a Natura-in-shared-body sitting in a seed pack is invisible.
502
+ # `eager:` or a store-specific phrase sitting in a shared seed pack is invisible.
341
503
  skill_store: Insika::SkillStore.new(config_store: config_store),
342
504
  skill_catalog: Insika::SkillCatalog.new(skill_roots),
343
505
  profile_source: Insika::StoredProfileSource.new(config_store: config_store),
506
+ # the shadow-parity check reads the pair store over the same
507
+ # runtime backend (pairs hold raw customer text — the check reports their
508
+ # age and the criterion's loadability before boot does).
509
+ shadow_pair_store: Insika::ShadowPairStore.new(store: backend),
510
+ # the cache-layers check — the BUILTIN set as classes (the
511
+ # check reads the layer declaration off the class, no instances needed).
512
+ context_providers: [
513
+ Insika::Context::Providers::Request, Insika::Context::Providers::Prompt,
514
+ Insika::Context::Providers::Skill, Insika::Context::Providers::SkillTrigger,
515
+ Insika::Context::Providers::ToolSearch, Insika::Context::Providers::Memory,
516
+ Insika::Context::Providers::Session
517
+ ],
518
+ # the memory-scopes check reads the cells over the same
519
+ # runtime backend; `agent_ids` excuses the agent-memory tab's cells
520
+ # (a bare cell named like an agent is a profile, not a customer —
521
+ # the profile source is right here, so the CLI names them).
522
+ memory_store: Insika::MemoryStore.new(store: backend),
523
+ agent_ids: Insika::StoredProfileSource.new(config_store: config_store).all.map(&:id),
524
+ # the outcome-funnel check reads the fold's cells and the
525
+ # outcomes over the same runtime backend — a profile with outcomes but
526
+ # no funnel warns, and a frozen baseline is confirmed before it is
527
+ # relied on.
528
+ funnel_store: Insika::FunnelStore.new(store: backend),
529
+ outcome_store: Insika::OutcomeStore.new(store: backend),
530
+ # the follow-up check reads the schedule records and the
531
+ # contact cells over the same runtime backend (nil collaborators = the
532
+ # CLI reports declarations only).
533
+ followup_store: Insika::FollowupStore.new(store: backend),
534
+ contact_store: Insika::ContactStore.new(store: backend),
535
+ # the distillation check reads the proposal store over
536
+ # the same runtime backend (nil = the CLI reports declarations only).
537
+ proposal_store: Insika::ProposalStore.new(store: backend),
538
+ # the harvest check reads the candidates over the same
539
+ # runtime backend + the frozen criterion (nil = declarations only).
540
+ harvest_store: Insika::HarvestStore.new(store: backend),
541
+ harvest_criterion: begin
542
+ path = Insika::EnvSchema.read("INSIKA_HARVEST_CRITERION")
543
+ path && Insika::Harvest::Criterion.load(path)
544
+ rescue Insika::ConfigError, Insika::ValidationError
545
+ nil
546
+ end,
344
547
  backend: backend
345
548
  )
346
549
  end