insika 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (280) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +199 -5
  3. data/README.md +8 -2
  4. data/bin/insika +231 -13
  5. data/docs/AGENTS.md +505 -6
  6. data/docs/API.md +56 -0
  7. data/docs/CHANNELS.md +100 -10
  8. data/docs/CONTEXT.md +147 -19
  9. data/docs/DEPLOY.md +34 -11
  10. data/docs/EMBEDDING.md +11 -7
  11. data/docs/EVALS.md +20 -1
  12. data/docs/FACTS.md +135 -0
  13. data/docs/HARVEST.md +117 -0
  14. data/docs/LOADTEST.md +17 -10
  15. data/docs/OBSERVABILITY.md +65 -2
  16. data/docs/REFINEMENT.md +9 -9
  17. data/docs/RELEASING.md +34 -7
  18. data/docs/RUNNING-LOCAL.md +4 -4
  19. data/docs/SECURITY.md +85 -11
  20. data/docs/SKILLS.md +189 -3
  21. data/docs/SOAK.md +127 -0
  22. data/docs/TOOLS.md +70 -2
  23. data/docs/WHY.md +1 -1
  24. data/docs/WORKFLOWS.md +2 -2
  25. data/docs/domain.md +115 -0
  26. data/docs/index.md +2 -2
  27. data/docs/onboarding/start.md +1 -1
  28. data/lib/insika/agent_profile.rb +228 -26
  29. data/lib/insika/alert_dispatcher.rb +139 -0
  30. data/lib/insika/balloon_splitter.rb +102 -0
  31. data/lib/insika/baseline_store.rb +2 -2
  32. data/lib/insika/budget_ledger.rb +166 -0
  33. data/lib/insika/cache_series_store.rb +49 -0
  34. data/lib/insika/channel_delivery.rb +132 -24
  35. data/lib/insika/channel_registry.rb +1 -1
  36. data/lib/insika/channels/relay.rb +80 -6
  37. data/lib/insika/channels/web/widget.js +2 -2
  38. data/lib/insika/channels/web.rb +9 -9
  39. data/lib/insika/channels/webhook.rb +58 -0
  40. data/lib/insika/chat_builder.rb +145 -13
  41. data/lib/insika/checkpoint_store.rb +16 -0
  42. data/lib/insika/circuit_state.rb +114 -0
  43. data/lib/insika/coercion.rb +8 -0
  44. data/lib/insika/commands/agent_payload.rb +6 -4
  45. data/lib/insika/commands/cancel_followup.rb +49 -0
  46. data/lib/insika/commands/create_agent.rb +2 -2
  47. data/lib/insika/commands/create_session.rb +1 -1
  48. data/lib/insika/commands/delete_llm_provider.rb +1 -1
  49. data/lib/insika/commands/delete_skill.rb +43 -0
  50. data/lib/insika/commands/delete_tenant_data.rb +95 -0
  51. data/lib/insika/commands/export_customer_memory.rb +48 -0
  52. data/lib/insika/commands/forget_customer.rb +117 -0
  53. data/lib/insika/commands/freeze_funnel_baseline.rb +113 -0
  54. data/lib/insika/commands/gate_harvest.rb +138 -0
  55. data/lib/insika/commands/gate_refinement.rb +12 -12
  56. data/lib/insika/commands/import_mcp_tools.rb +1 -1
  57. data/lib/insika/commands/import_tools.rb +4 -4
  58. data/lib/insika/commands/issue_tenant_token.rb +41 -0
  59. data/lib/insika/commands/judge_shadow_pairs.rb +124 -0
  60. data/lib/insika/commands/memory_forget_fact.rb +20 -4
  61. data/lib/insika/commands/memory_put_fact.rb +23 -4
  62. data/lib/insika/commands/promote_harvest.rb +130 -0
  63. data/lib/insika/commands/record_outcome.rb +46 -0
  64. data/lib/insika/commands/record_shadow_reply.rb +68 -0
  65. data/lib/insika/commands/reject_harvest.rb +38 -0
  66. data/lib/insika/commands/resolve_proposal.rb +108 -0
  67. data/lib/insika/commands/resolve_refinement.rb +1 -1
  68. data/lib/insika/commands/revoke_contact.rb +49 -0
  69. data/lib/insika/commands/revoke_token.rb +39 -0
  70. data/lib/insika/commands/rollback_harvest.rb +86 -0
  71. data/lib/insika/commands/rotate_tenant_token.rb +43 -0
  72. data/lib/insika/commands/run_distillation.rb +186 -0
  73. data/lib/insika/commands/run_harvest.rb +393 -0
  74. data/lib/insika/commands/run_refinement.rb +5 -5
  75. data/lib/insika/commands/send_message.rb +112 -15
  76. data/lib/insika/commands/session_purge.rb +67 -0
  77. data/lib/insika/commands/set_agent_tools.rb +1 -1
  78. data/lib/insika/commands/set_skill_agents.rb +60 -19
  79. data/lib/insika/commands/trigger_workflow.rb +1 -1
  80. data/lib/insika/commands/update_agent.rb +1 -1
  81. data/lib/insika/commands/write_data_tool.rb +1 -1
  82. data/lib/insika/commands/write_golden.rb +1 -1
  83. data/lib/insika/commands/write_skill.rb +19 -9
  84. data/lib/insika/config_store.rb +8 -4
  85. data/lib/insika/contact_store.rb +183 -0
  86. data/lib/insika/context/builder.rb +23 -5
  87. data/lib/insika/context/fragment.rb +31 -3
  88. data/lib/insika/context/priority.rb +6 -2
  89. data/lib/insika/context/provider.rb +17 -3
  90. data/lib/insika/context/providers/briefing.rb +96 -0
  91. data/lib/insika/context/providers/memory.rb +16 -7
  92. data/lib/insika/context/providers/prompt.rb +30 -2
  93. data/lib/insika/context/providers/request.rb +1 -1
  94. data/lib/insika/context/providers/session.rb +17 -2
  95. data/lib/insika/context/providers/skill.rb +7 -1
  96. data/lib/insika/context/providers/skill_trigger.rb +128 -0
  97. data/lib/insika/context/providers/tool_search.rb +2 -0
  98. data/lib/insika/context_trace_store.rb +128 -0
  99. data/lib/insika/delegation_store.rb +2 -2
  100. data/lib/insika/distill.rb +224 -0
  101. data/lib/insika/distill_engine.rb +169 -0
  102. data/lib/insika/doctor.rb +962 -7
  103. data/lib/insika/dsl/runtime.rb +20 -11
  104. data/lib/insika/dsl/server_boot.rb +74 -4
  105. data/lib/insika/dsl/system.rb +1 -1
  106. data/lib/insika/dsl.rb +152 -15
  107. data/lib/insika/edge_limiter.rb +167 -8
  108. data/lib/insika/egress_guard.rb +3 -3
  109. data/lib/insika/env_schema.rb +22 -12
  110. data/lib/insika/errors.rb +72 -5
  111. data/lib/insika/evals/assertions.rb +15 -14
  112. data/lib/insika/evals/baseline.rb +3 -3
  113. data/lib/insika/evals/golden.rb +8 -8
  114. data/lib/insika/evals/judge.rb +7 -7
  115. data/lib/insika/evals/pairwise.rb +21 -9
  116. data/lib/insika/evals/report.rb +2 -2
  117. data/lib/insika/evals/runner.rb +6 -6
  118. data/lib/insika/evals/transport.rb +2 -2
  119. data/lib/insika/event_stream.rb +23 -5
  120. data/lib/insika/evidence.rb +183 -0
  121. data/lib/insika/executor.rb +1092 -160
  122. data/lib/insika/followup_engine.rb +207 -0
  123. data/lib/insika/followup_policy.rb +221 -0
  124. data/lib/insika/followup_store.rb +306 -0
  125. data/lib/insika/frontmatter.rb +1 -1
  126. data/lib/insika/funnel_declaration.rb +106 -0
  127. data/lib/insika/funnel_fold.rb +179 -0
  128. data/lib/insika/funnel_store.rb +163 -0
  129. data/lib/insika/golden_store.rb +3 -3
  130. data/lib/insika/grounding/matcher.rb +69 -0
  131. data/lib/insika/grounding.rb +44 -0
  132. data/lib/insika/harvest/conversion_gate.rb +159 -0
  133. data/lib/insika/harvest/criterion.rb +98 -0
  134. data/lib/insika/harvest/gate.rb +194 -0
  135. data/lib/insika/harvest/negative_list.rb +199 -0
  136. data/lib/insika/harvest.rb +241 -0
  137. data/lib/insika/harvest_engine.rb +193 -0
  138. data/lib/insika/harvest_store.rb +548 -0
  139. data/lib/insika/http_client.rb +3 -3
  140. data/lib/insika/inbound_log.rb +1 -1
  141. data/lib/insika/llm_configurator.rb +3 -3
  142. data/lib/insika/loop_detector.rb +143 -0
  143. data/lib/insika/mcp_http_client.rb +4 -4
  144. data/lib/insika/mcp_tool_ingestor.rb +6 -6
  145. data/lib/insika/media.rb +298 -0
  146. data/lib/insika/memory_audit_store.rb +85 -0
  147. data/lib/insika/memory_store.rb +264 -23
  148. data/lib/insika/message_origin.rb +8 -3
  149. data/lib/insika/model_resolver.rb +1 -1
  150. data/lib/insika/model_selection.rb +5 -4
  151. data/lib/insika/model_visible.rb +87 -0
  152. data/lib/insika/model_visible_trace_store.rb +66 -0
  153. data/lib/insika/onboarding.rb +8 -3
  154. data/lib/insika/outbox_store.rb +44 -6
  155. data/lib/insika/outcome_store.rb +147 -0
  156. data/lib/insika/overlay_tool_registry.rb +3 -4
  157. data/lib/insika/pack.rb +3 -3
  158. data/lib/insika/pack_importer.rb +17 -15
  159. data/lib/insika/packaging.rb +163 -0
  160. data/lib/insika/parity/criterion.rb +79 -0
  161. data/lib/insika/parity/verdict.rb +318 -0
  162. data/lib/insika/pending_action_store.rb +1 -1
  163. data/lib/insika/plugin/loader.rb +2 -2
  164. data/lib/insika/policy/policy.rb +1 -1
  165. data/lib/insika/prefix_fingerprint.rb +58 -0
  166. data/lib/insika/profile_source.rb +34 -7
  167. data/lib/insika/proposal_store.rb +271 -0
  168. data/lib/insika/provider_error_classifier.rb +160 -0
  169. data/lib/insika/queue_policy.rb +6 -3
  170. data/lib/insika/recovery.rb +47 -6
  171. data/lib/insika/refinement/candidate.rb +4 -4
  172. data/lib/insika/refinement/evidence_collector.rb +6 -6
  173. data/lib/insika/refinement/gate.rb +7 -7
  174. data/lib/insika/refinement/panel.rb +7 -7
  175. data/lib/insika/refinement/proposer.rb +10 -10
  176. data/lib/insika/refinement_store.rb +12 -12
  177. data/lib/insika/reliability.rb +211 -0
  178. data/lib/insika/retention.rb +281 -0
  179. data/lib/insika/routing.rb +101 -0
  180. data/lib/insika/safety/config.rb +46 -6
  181. data/lib/insika/safety/corpus.rb +255 -0
  182. data/lib/insika/safety/detectors.rb +34 -115
  183. data/lib/insika/safety/factory.rb +18 -5
  184. data/lib/insika/safety/grounding_enforcer.rb +59 -0
  185. data/lib/insika/safety/grounding_validator.rb +49 -0
  186. data/lib/insika/safety/input_guardrail.rb +20 -5
  187. data/lib/insika/safety/moderator.rb +19 -11
  188. data/lib/insika/safety/output_filter.rb +10 -6
  189. data/lib/insika/safety/output_validator.rb +13 -7
  190. data/lib/insika/safety/safe_responses.rb +1 -1
  191. data/lib/insika/sandbox/boundary.rb +2 -2
  192. data/lib/insika/sandbox.rb +1 -1
  193. data/lib/insika/schema_guard.rb +35 -0
  194. data/lib/insika/server/app.rb +366 -54
  195. data/lib/insika/server/boot.rb +4 -4
  196. data/lib/insika/server/rack_app.rb +31 -7
  197. data/lib/insika/server/responses.rb +58 -9
  198. data/lib/insika/server/tenant_auth.rb +61 -0
  199. data/lib/insika/session_actor.rb +11 -7
  200. data/lib/insika/session_store.rb +66 -3
  201. data/lib/insika/settings_store.rb +15 -5
  202. data/lib/insika/shadow_pair_store.rb +258 -0
  203. data/lib/insika/shutdown.rb +4 -4
  204. data/lib/insika/skill_catalog.rb +131 -20
  205. data/lib/insika/skill_store.rb +70 -22
  206. data/lib/insika/soak/envelope.rb +140 -0
  207. data/lib/insika/soak/report.rb +392 -0
  208. data/lib/insika/soak/runner.rb +554 -0
  209. data/lib/insika/steer_injector.rb +1 -1
  210. data/lib/insika/store.rb +11 -2
  211. data/lib/insika/stores/memory.rb +6 -0
  212. data/lib/insika/stores/sqlite.rb +8 -0
  213. data/lib/insika/studio/app.rb +1058 -75
  214. data/lib/insika/studio/assets/dist/application.css +1 -1
  215. data/lib/insika/studio/assets/dist/application.js +27 -26
  216. data/lib/insika/studio/assets/dist/favicon.svg +6 -0
  217. data/lib/insika/studio/forms.rb +274 -22
  218. data/lib/insika/studio/nav_icons.rb +7 -2
  219. data/lib/insika/studio/views/_message.erb +2 -2
  220. data/lib/insika/studio/views/agent_detail.erb +629 -86
  221. data/lib/insika/studio/views/agents.erb +11 -7
  222. data/lib/insika/studio/views/approvals.erb +4 -1
  223. data/lib/insika/studio/views/chats.erb +4 -1
  224. data/lib/insika/studio/views/customer.erb +94 -0
  225. data/lib/insika/studio/views/customers.erb +32 -0
  226. data/lib/insika/studio/views/evals.erb +4 -1
  227. data/lib/insika/studio/views/facts.erb +133 -0
  228. data/lib/insika/studio/views/followups.erb +125 -0
  229. data/lib/insika/studio/views/funnel.erb +106 -0
  230. data/lib/insika/studio/views/harvest.erb +234 -0
  231. data/lib/insika/studio/views/home.erb +2 -1
  232. data/lib/insika/studio/views/layout.erb +1 -0
  233. data/lib/insika/studio/views/parity.erb +147 -0
  234. data/lib/insika/studio/views/playground.erb +7 -1
  235. data/lib/insika/studio/views/refinement.erb +4 -4
  236. data/lib/insika/studio/views/session.erb +133 -3
  237. data/lib/insika/studio/views/settings.erb +9 -12
  238. data/lib/insika/studio/views/skills.erb +66 -12
  239. data/lib/insika/studio/views/system_files.erb +1 -1
  240. data/lib/insika/studio/views/task.erb +13 -0
  241. data/lib/insika/studio/views/tasks.erb +4 -1
  242. data/lib/insika/studio/views/tools.erb +0 -1
  243. data/lib/insika/subagent_graph.rb +3 -3
  244. data/lib/insika/task_actor.rb +3 -3
  245. data/lib/insika/task_store.rb +22 -2
  246. data/lib/insika/telemetry/pricing.rb +3 -3
  247. data/lib/insika/telemetry/recorder.rb +1 -1
  248. data/lib/insika/telemetry.rb +2 -2
  249. data/lib/insika/testing/store_contract.rb +54 -33
  250. data/lib/insika/tick.rb +146 -0
  251. data/lib/insika/token_store.rb +168 -0
  252. data/lib/insika/tool_assembly.rb +5 -5
  253. data/lib/insika/tool_definition.rb +25 -15
  254. data/lib/insika/tool_envelope.rb +70 -1
  255. data/lib/insika/tool_manifest.rb +11 -7
  256. data/lib/insika/tool_output_compressor.rb +100 -0
  257. data/lib/insika/tool_store.rb +1 -1
  258. data/lib/insika/tool_trace_store.rb +1 -1
  259. data/lib/insika/tools/concurrency.rb +2 -2
  260. data/lib/insika/tools/data_defined_tool.rb +14 -5
  261. data/lib/insika/tools/generate_image.rb +44 -0
  262. data/lib/insika/tools/load_skill.rb +61 -3
  263. data/lib/insika/tools/schedule_followup.rb +164 -0
  264. data/lib/insika/tools/stuck_signal.rb +44 -0
  265. data/lib/insika/tools/subagent.rb +4 -4
  266. data/lib/insika/tools/subagents.rb +1 -1
  267. data/lib/insika/tools/tts.rb +47 -0
  268. data/lib/insika/tools/update_briefing.rb +126 -0
  269. data/lib/insika/turn_output.rb +2 -2
  270. data/lib/insika/turn_state.rb +54 -13
  271. data/lib/insika/turn_timing.rb +24 -4
  272. data/lib/insika/usage_ledger.rb +1 -1
  273. data/lib/insika/version.rb +1 -1
  274. data/lib/insika/vitals.rb +84 -0
  275. data/lib/insika/wiring/graph.rb +372 -34
  276. data/lib/insika/workflow.rb +1 -1
  277. data/lib/insika/workflow_registry.rb +1 -1
  278. data/lib/insika.rb +122 -16
  279. metadata +95 -2
  280. data/lib/insika/server/admin_auth.rb +0 -29
@@ -10,6 +10,14 @@ module Insika
10
10
  # `PendingActionStore` (normalizes symbol→string on write, O(n) scan on
11
11
  # read, records with a timestamp).
12
12
  #
13
+ # the record gains `origin`/`created_at`/`expires_at`, the
14
+ # (tenant, customer) pair becomes first-class at the API (`customer:`
15
+ # builds the same "memory:<tenant>:<customer>" scope the Executor derives),
16
+ # and the store enumerates its cells (the Studio drill, the doctor, the
17
+ # retention sweep). Migration is lazy: tolerant reads, materialized on the
18
+ # first write. The audit trail lives in `MemoryAuditStore` (C2), NOT here —
19
+ # this stays a dumb domain store.
20
+ #
13
21
  # NOT to be confused with `Insika::Stores::Memory` (in-memory KV backend):
14
22
  # this is the domain store; that one is one of the backends this writes to.
15
23
  class MemoryStore
@@ -18,52 +26,246 @@ module Insika
18
26
  NOTE_PREFIX = "note:"
19
27
  DEFAULT_TENANT = "_default" # no tenant in the Command
20
28
 
21
- Fact = Data.define(:key, :value, :updated_at)
29
+ # the record's provenance. A closed string set: "legacy"
30
+ # (migrated, unknown writer), "engine" (the remember tool / an
31
+ # integration write), "operator" (Studio edit), "distilled"
32
+ # reserved).
33
+ ORIGIN_LEGACY = "legacy"
34
+
35
+ # the per-SESSION cell marker. Engine-owner memory with no
36
+ # customer and no explicit tenant falls back to the session (executor
37
+ # parity) — the cell is MARKED "memory:chat:<session id>" so the drill and
38
+ # the doctor never read a conversation as a customer (a bare
39
+ # "memory:<session id>" cell is indistinguishable from a single-tenant
40
+ # customer ref). A real tenant NAMED "chat" would collide with the marker
41
+ # (its customer cells would read as session cells) — accepted, merchant
42
+ # ids live in a different namespace in practice.
43
+ SESSION_TAG = "chat"
44
+
45
+ Fact = Data.define(:key, :value, :origin, :created_at, :updated_at, :expires_at)
22
46
  Note = Data.define(:id, :text, :created_at)
23
47
 
24
- def initialize(store:)
48
+ def initialize(store:, clock: nil)
25
49
  @store = store
50
+ @clock = clock # -> Time, injectable for specs
26
51
  end
27
52
 
28
- # Upsert (last-write-wins, Store contract). -> Fact
29
- def put_fact(tenant:, key:, value:)
30
- record = { "key" => key.to_s, "value" => stringify(value), "updated_at" => timestamp }
31
- @store.set(scope_for(tenant), FACT_PREFIX + key.to_s, record)
53
+ # Upsert (last-write-wins, Store contract). `customer:` PRESENT moves the
54
+ # scope to the [tenant:]customer cell (WS8's rule — nil tenant + customer
55
+ # -> bare "memory:<customer>", NEVER _default). `origin` defaults to
56
+ # "engine" (the remember tool's write). `expires_at` (ISO8601 String) is
57
+ # normalized and validated here — an invalid value raises ValidationError
58
+ # (a silently dropped LGPD expiry date is the defect). Preserves the
59
+ # existing record's created_at; rewrites a legacy record in the full
60
+ # shape. -> Fact
61
+ def put_fact(tenant:, key:, value:, customer: nil, origin: "engine", expires_at: nil)
62
+ scope = scope_for(tenant, customer)
63
+ current = @store.get(scope, FACT_PREFIX + key.to_s)
64
+ prev = current.is_a?(Hash) ? current : {}
65
+ stamp = timestamp
66
+ record = {
67
+ "key" => key.to_s, "value" => stringify(value),
68
+ "origin" => Coercion.presence(origin) || "engine",
69
+ "created_at" => Coercion.presence(prev["created_at"]) || Coercion.presence(prev["updated_at"]) || stamp,
70
+ "updated_at" => stamp,
71
+ "expires_at" => normalize_expiry(expires_at)
72
+ }
73
+ @store.set(scope, FACT_PREFIX + key.to_s, record)
32
74
  to_fact(record)
33
75
  end
34
76
 
35
- # -> Fact | nil
36
- def get_fact(tenant:, key:)
37
- record = @store.get(scope_for(tenant), FACT_PREFIX + key.to_s)
77
+ # -> Fact | nil (tolerant read: missing origin -> "legacy", created_at ->
78
+ # updated_at, expires_at -> nil).
79
+ def get_fact(tenant:, key:, customer: nil)
80
+ record = @store.get(scope_for(tenant, customer), FACT_PREFIX + key.to_s)
38
81
  record && to_fact(record)
39
82
  end
40
83
 
41
- # -> [Fact] sorted by key (list is lexicographic).
42
- def facts(tenant:)
43
- scope = scope_for(tenant)
84
+ # -> [Fact] sorted by key. Facts whose expires_at <= now are EXCLUDED
85
+ # (D5: an expired fact is never injected nor shown, even before the
86
+ # sweep runs).
87
+ def facts(tenant:, customer: nil)
88
+ scope = scope_for(tenant, customer)
44
89
  @store.list(scope, FACT_PREFIX).filter_map do |k|
45
90
  record = @store.get(scope, k)
46
- record && to_fact(record)
91
+ next unless record
92
+
93
+ fact = to_fact(record)
94
+ next if expired?(fact)
95
+
96
+ fact
47
97
  end
48
98
  end
49
99
 
50
100
  # -> bool (did it exist?)
51
- def forget_fact(tenant:, key:)
52
- @store.delete(scope_for(tenant), FACT_PREFIX + key.to_s)
101
+ def forget_fact(tenant:, key:, customer: nil)
102
+ @store.delete(scope_for(tenant, customer), FACT_PREFIX + key.to_s)
103
+ end
104
+
105
+ # CAS write (WS8): only writes when the STORED fact's updated_at still
106
+ # equals the caller's revision — a fact a concurrent writer already moved
107
+ # is refused instead of clobbered (last-write-wins is the default; this is
108
+ # the opt-in optimistic path). -> Fact (written) | nil (lost the race — the
109
+ # caller must re-read and retry). The read-compare-write rides
110
+ # `@store.transaction` (the repo rule) — `next`, never `return`, inside.
111
+ def replace_if_revision(tenant:, key:, value:, expected_revision:,
112
+ customer: nil, origin: "engine", expires_at: nil)
113
+ @store.transaction do
114
+ current = @store.get(scope_for(tenant, customer), FACT_PREFIX + key.to_s)
115
+ next nil if current.nil? || current["updated_at"] != expected_revision
116
+
117
+ put_fact(tenant: tenant, key: key, value: value, customer: customer,
118
+ origin: origin, expires_at: expires_at)
119
+ end
120
+ end
121
+
122
+ # Purges the WHOLE scope (WS8 — forget_customer / delete_tenant_data). The
123
+ # scope string is the isolation boundary: one cell per
124
+ # (tenant-or-customer), so zeroing the cell cannot touch another's.
125
+ # The list-then-delete rides `@store.transaction` (the repo rule): an erasure
126
+ # that is half-applied is the LGPD defect, and a fact written between the
127
+ # list and the deletes would survive a purge that reported success.
128
+ # -> count of records purged.
129
+ def purge(tenant:, customer: nil)
130
+ scope = scope_for(tenant, customer)
131
+ @store.transaction do
132
+ keys = @store.list(scope)
133
+ keys.each { |k| @store.delete(scope, k) }
134
+ keys.size
135
+ end
136
+ end
137
+
138
+ # Purges a TENANT and every customer cell under it (WS8 phase 2 —
139
+ # delete_tenant_data). The tenant's own cell ("memory:<tenant>") plus each
140
+ # cell whose scope starts with "memory:<tenant>:" (the customer cells),
141
+ # plus the tenant's SESSION-marked cells ("memory:chat:<tenant>:*" — in
142
+ # multi-tenant a session id is "<tenant>:<id>", so its memory cell is
143
+ # "memory:chat:<tenant>:<id>"). The scope enumeration is the Store's — no
144
+ # session-derived list, so a customer cell whose session was already
145
+ # deleted is still purged. ONE transaction for the whole tenant (the repo
146
+ # rule): the scope enumeration and every delete see the same snapshot, so a
147
+ # customer cell born mid-purge cannot slip through a deletion that reported
148
+ # success. -> count of records purged.
149
+ def purge_tenant(tenant)
150
+ cell = scope_for(tenant)
151
+ @store.transaction do
152
+ scopes = [cell] + @store.scopes("#{cell}:") +
153
+ @store.scopes("#{SCOPE_PREFIX}:#{SESSION_TAG}:#{tenant}:")
154
+ scopes.sum do |scope|
155
+ keys = @store.list(scope)
156
+ keys.each { |k| @store.delete(scope, k) }
157
+ keys.size
158
+ end
159
+ end
160
+ end
161
+
162
+ # Age-based prune (WS8 retention). New : `scope:` limits the pass
163
+ # to ONE cell; a fact with an explicit expires_at is SKIPPED (D5 — the
164
+ # explicit override owns that fact's life). -> count removed.
165
+ def prune_older_than(time, scope: nil)
166
+ cutoff = time.utc.iso8601
167
+ removed = 0
168
+ scopes = scope ? [scope] : @store.scopes("#{SCOPE_PREFIX}:")
169
+ scopes.each do |sc|
170
+ @store.list(sc).each do |k|
171
+ rec = @store.get(sc, k)
172
+ next if rec && rec["expires_at"] # the override owns this fact's life
173
+
174
+ stamp = rec && (rec["updated_at"] || rec["created_at"])
175
+ next unless stamp && stamp.to_s < cutoff
176
+
177
+ @store.delete(sc, k)
178
+ removed += 1
179
+ end
180
+ end
181
+ removed
182
+ end
183
+
184
+ # Removes facts whose expires_at <= now (default clock), EVERY cell.
185
+ # -> count removed.
186
+ def prune_expired(now = nil)
187
+ cutoff = (now || self.now).utc
188
+ removed = 0
189
+ @store.scopes("#{SCOPE_PREFIX}:").each do |scope|
190
+ @store.list(scope).each do |k|
191
+ rec = @store.get(scope, k)
192
+ next unless rec && rec["expires_at"]
193
+
194
+ begin
195
+ next unless Time.iso8601(rec["expires_at"]) <= cutoff
196
+ rescue ArgumentError
197
+ next
198
+ end
199
+
200
+ @store.delete(scope, k)
201
+ removed += 1
202
+ end
203
+ end
204
+ removed
205
+ end
206
+
207
+ # --- enumeration (the drill / doctor / retention inputs) -----------
208
+
209
+ # -> [{scope:, tenant: String|nil, customer: String|nil}] — every "memory:*"
210
+ # scope, classified by SHAPE (D6). _default -> {tenant: nil, customer: nil}.
211
+ def cells
212
+ @store.scopes("#{SCOPE_PREFIX}:").map { |scope| self.class.parse_cell(scope) }
213
+ end
214
+
215
+ # -> [{scope:, tenant:, customer:}] — the cells that hold CUSTOMER memory:
216
+ # 2+-segment scopes always (except the SESSION-marked cells — a session is
217
+ # never a customer); 1-segment scopes whose name is not _default and not in
218
+ # `reserved` (the caller's agent ids / tenant cells). Sorted by scope.
219
+ def customer_cells(reserved: [])
220
+ excluded = Array(reserved).map(&:to_s)
221
+ cells.reject do |c|
222
+ self.class.session_cell?(c) ||
223
+ (c[:tenant].nil? && (c[:customer].nil? || excluded.include?(c[:customer])))
224
+ end.sort_by { |c| c[:scope] }
225
+ end
226
+
227
+ # Is this a classified cell the engine's per-SESSION cell
228
+ # ("memory:chat:<session id>")? The one classification both the drill and
229
+ # the doctor share — a session-derived cell must never read as a customer.
230
+ def self.session_cell?(cell) = cell[:tenant] == SESSION_TAG
231
+
232
+ # The public cell string for a pair (the commands and the Studio build URLs
233
+ # and audit keys with this — the same string the Executor derives).
234
+ def cell_for(tenant, customer = nil) = scope_for(tenant, customer)
235
+
236
+ # The shared classification (public — the Studio/commands split audit
237
+ # fields): "memory:acme:c-123" -> {scope:, tenant: "acme", customer: "c-123"};
238
+ # "memory:c-123" -> {scope:, tenant: nil, customer: "c-123"};
239
+ # "memory:_default" -> {scope:, tenant: nil, customer: nil}.
240
+ def self.parse_cell(scope)
241
+ rest = scope.to_s.sub(/\A#{SCOPE_PREFIX}:/, "")
242
+ base, tail = rest.split(":", 2)
243
+ if base == DEFAULT_TENANT && tail.nil?
244
+ { scope: scope, tenant: nil, customer: nil }
245
+ elsif tail
246
+ { scope: scope, tenant: base, customer: tail }
247
+ else
248
+ { scope: scope, tenant: nil, customer: base }
249
+ end
53
250
  end
54
251
 
55
252
  # Append. `at` (ISO8601) goes at the START of the key so `list` returns the notes in
56
253
  # chronological order; `id`/`at` injectable for deterministic tests. -> Note
57
- def add_note(tenant:, text:, id: SecureRandom.uuid, at: nil)
254
+ def add_note(tenant:, text:, id: SecureRandom.uuid, at: nil, customer: nil)
58
255
  at ||= timestamp
59
256
  record = { "id" => id.to_s, "text" => text.to_s, "created_at" => at }
60
- @store.set(scope_for(tenant), NOTE_PREFIX + "#{at}:#{id}", record)
257
+ @store.set(scope_for(tenant, customer), NOTE_PREFIX + "#{at}:#{id}", record)
61
258
  to_note(record)
62
259
  end
63
260
 
261
+ # Cheap fact count for the drill index (keys only, no payload reads). -> Integer
262
+ def fact_count(tenant:, customer: nil)
263
+ @store.list(scope_for(tenant, customer), FACT_PREFIX).size
264
+ end
265
+
64
266
  # -> [Note] MOST RECENT first, capped by `limit`.
65
- def notes(tenant:, limit: nil)
66
- scope = scope_for(tenant)
267
+ def notes(tenant:, limit: nil, customer: nil)
268
+ scope = scope_for(tenant, customer)
67
269
  keys = @store.list(scope, NOTE_PREFIX).reverse # list is chronological -> reverse = most recent first
68
270
  keys = keys.first(limit) if limit
69
271
  keys.filter_map do |k|
@@ -74,12 +276,51 @@ module Insika
74
276
 
75
277
  private
76
278
 
77
- def scope_for(tenant) = "#{SCOPE_PREFIX}:#{tenant.nil? || tenant.to_s.empty? ? DEFAULT_TENANT : tenant}"
279
+ # WS8's rule (executor.rb:1679 parity): customer present -> the
280
+ # [tenant:]customer cell; nil tenant + customer -> the bare customer cell,
281
+ # NEVER memory:_default:<customer>.
282
+ def scope_for(tenant, customer = nil)
283
+ base = Coercion.blank?(tenant) ? DEFAULT_TENANT : tenant
284
+ cust = Coercion.presence(customer)
285
+ return "#{SCOPE_PREFIX}:#{cust}" if cust && Coercion.blank?(tenant)
286
+
287
+ cust ? "#{SCOPE_PREFIX}:#{base}:#{cust}" : "#{SCOPE_PREFIX}:#{base}"
288
+ end
289
+
290
+ def to_fact(record)
291
+ Fact.new(key: record["key"], value: record["value"],
292
+ origin: Coercion.presence(record["origin"]) || ORIGIN_LEGACY,
293
+ created_at: Coercion.presence(record["created_at"]) || Coercion.presence(record["updated_at"]),
294
+ updated_at: record["updated_at"], expires_at: record["expires_at"])
295
+ end
78
296
 
79
- def to_fact(record) = Fact.new(key: record["key"], value: record["value"], updated_at: record["updated_at"])
80
297
  def to_note(record) = Note.new(id: record["id"], text: record["text"], created_at: record["created_at"])
81
298
 
82
- def timestamp = Time.now.utc.iso8601
299
+ def expired?(fact)
300
+ return false if fact.expires_at.nil?
301
+
302
+ Time.iso8601(fact.expires_at) <= now
303
+ rescue ArgumentError
304
+ false
305
+ end
306
+
307
+ def now = (@clock ? @clock.call : Time.now.utc).utc
308
+
309
+ # Microsecond precision ON PURPOSE: updated_at is the CAS revision
310
+ # (replace_if_revision) — second-precision collides for two writes in the
311
+ # same second (WS8).
312
+ def timestamp = now.iso8601(6)
313
+
314
+ # nil | canonical ISO8601(6); invalid -> ValidationError (fail-fast on an
315
+ # LGPD field — a silently dropped expiry date is the defect).
316
+ def normalize_expiry(expires_at)
317
+ return nil if expires_at.nil? || expires_at.to_s.strip.empty?
318
+
319
+ Time.iso8601(expires_at.to_s).utc.iso8601(6)
320
+ rescue ArgumentError
321
+ raise Insika::ValidationError,
322
+ "expires_at must be an ISO8601 timestamp (got #{expires_at.inspect})"
323
+ end
83
324
 
84
325
  def stringify(obj)
85
326
  case obj
@@ -90,4 +331,4 @@ module Insika
90
331
  end
91
332
  end
92
333
  end
93
- end
334
+ end
@@ -7,11 +7,11 @@ module Insika
7
7
  # two come apart constantly:
8
8
  #
9
9
  # · The engine delivers an async subagent's result to the parent as a NEW turn
10
- # (RFC-0010) — a `user` message the engine wrote.
10
+ # a `user` message the engine wrote.
11
11
  # · A guardrail short-circuits with a safe reply — an `assistant` message produced
12
12
  # with zero LLM calls.
13
13
  # · A consumer composes context blocks into the input it sends (`<memoria> …`,
14
- # `<cacau_cep_obrigatorio> …`) — a `user` message the customer never typed.
14
+ # `<store_cep_required> …`) — a `user` message the customer never typed.
15
15
  # · In an imported transcript, a human operator types after a handoff — an
16
16
  # `assistant` message no model produced.
17
17
  #
@@ -33,8 +33,13 @@ module Insika
33
33
  AGENT = "agent" # the model (the default for `assistant`)
34
34
  ENGINE = "engine" # Insika itself, or the consumer composing on its behalf
35
35
  OPERATOR = "operator" # a HUMAN on the assistant side (a handoff; set by importers)
36
+ # the FollowupEngine's synthetic turn — the engine's own kick,
37
+ # never the customer. RESERVED: a consumer cannot declare it (the
38
+ # SendMessage edge refuses the spelling — only the engine creates those
39
+ # turns, D5).
40
+ SCHEDULED = "scheduled"
36
41
 
37
- ALL = [CUSTOMER, AGENT, ENGINE, OPERATOR].freeze
42
+ ALL = [CUSTOMER, AGENT, ENGINE, OPERATOR, SCHEDULED].freeze
38
43
 
39
44
  module_function
40
45
 
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Insika
4
- # LLM config v2 resolution (FOLLOWUP §10). Turns the three config layers into a
4
+ # LLM config v2 resolution. Turns the three config layers into a
5
5
  # single ModelSelection at turn start:
6
6
  #
7
7
  # Chat (per-session pin) > Agent (profile.model) > Platform default (Settings)
@@ -14,8 +14,9 @@ module Insika
14
14
  # model / platform default is NOT pinned -> fallbacks apply.
15
15
  # params -> generation params (Hash of symbols: temperature/max_tokens/thinking)
16
16
  # fallbacks -> ordered [{ model:, provider: }] to try when NOT pinned. The
17
- # mid-turn ROTATION across this chain is a follow-up; today the
18
- # chain is resolved + surfaced (source/pinned) for telemetry.
17
+ # mid-turn ROTATION across this chain (plus the profile's own
18
+ # reliability["fallback"] refs) is WS3's Reliability coordinator;
19
+ # here the chain is resolved + surfaced (source/pinned) for it.
19
20
  ModelSelection = Data.define(:model, :provider, :source, :pinned, :params, :fallbacks) do
20
21
  def initialize(model:, provider: nil, source: :platform_default, pinned: false,
21
22
  params: {}, fallbacks: [])
@@ -70,7 +71,7 @@ module Insika
70
71
  chat.with_params(**payload) unless payload.empty?
71
72
  end
72
73
 
73
- # The resolved reasoning control (§10, 4-layer). Two axes folded into one field:
74
+ # The resolved reasoning control (4-layer). Two axes folded into one field:
74
75
  # off -> reasoning DISABLED (thinking:{type:disabled})
75
76
  # on -> reasoning ENABLED, provider-default effort
76
77
  # low|medium|high -> reasoning enabled at that effort (reasoning_effort)
@@ -105,7 +106,7 @@ module Insika
105
106
  def present?(v) = Insika::Coercion.present?(v)
106
107
  end
107
108
 
108
- # The selectable reasoning values (§10, 4-layer). Blank/absent = inherit the
109
+ # The selectable reasoning values (4-layer). Blank/absent = inherit the
109
110
  # broader layer; these are the explicit choices. Shared with the Studio forms.
110
111
  # Defined on the class OUTSIDE the Data.define block on purpose: a constant
111
112
  # assigned inside the block would land in the enclosing lexical scope (Insika),
@@ -0,0 +1,87 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Insika
4
+ # — the model-visible payload of ONE ask: the system text,
5
+ # the tool schemas, and the full message stream — exactly what the provider
6
+ # serializes. Captured AT the RubyLLM boundary (the chat object the executor
7
+ # hands to `ask`), never re-derived: re-derivation would re-run volatile
8
+ # context providers (LLM moderation, timestamps) and could drift from the
9
+ # bytes actually sent.
10
+ #
11
+ # "Model-visible means logged": the durable reconstruction of a turn is the
12
+ # checkpoint (messages — the transcript half) + this record (instructions +
13
+ # tool schemas — the half the checkpoint lacks). The conformance suite
14
+ # (spec/insika/conformance/model_visible_spec.rb) asserts the three-way byte
15
+ # identity: capturing chat == checkpoint == trace.
16
+ #
17
+ # Pure reads off the chat object with respond_to? guards — a chat lacking a
18
+ # reader contributes nil/[], never raises (the trace discipline).
19
+ ModelVisible = Data.define(:instructions, :tools, :messages) do
20
+ # -> ModelVisible. `chat` is the RubyLLM chat at the boundary (instructions,
21
+ # tools, messages are exactly the three parts the provider serializes).
22
+ def self.capture(chat)
23
+ new(
24
+ instructions: chat.respond_to?(:instructions) ? chat.instructions : nil,
25
+ tools: chat.respond_to?(:tools) ? Array(chat.tools).map { |t| tool_schema(t) } : [],
26
+ messages: chat.respond_to?(:messages) ? Array(chat.messages) : []
27
+ )
28
+ end
29
+
30
+ # -> Hash: the JSON-safe schema of ONE tool. Reads `parameters` then
31
+ # `schema` (the two reader shapes the house FakeChat and the gem's tools
32
+ # answer); a tool answering none degrades to nil, never raises.
33
+ def self.tool_schema(tool)
34
+ params = if tool.respond_to?(:parameters)
35
+ tool.parameters
36
+ elsif tool.respond_to?(:schema)
37
+ tool.schema
38
+ end
39
+ {
40
+ "name" => (tool.name.to_s if tool.respond_to?(:name)),
41
+ "description" => (tool.description.to_s if tool.respond_to?(:description)),
42
+ "parameters" => params && json_safe(params)
43
+ }.compact
44
+ end
45
+
46
+ # -> JSON-safe projection of an arbitrary schema value. The gem's tools
47
+ # answer parameters as Hash with RubyLLM::Parameter values (a class whose
48
+ # readers are name/type/description/required) — the trace must persist the
49
+ # schema, so it projects that shape instead of raising on the object (the
50
+ # record never breaks the turn). Anything unprojectable degrades to its
51
+ # string form, never raises.
52
+ def self.json_safe(value)
53
+ case value
54
+ when Hash
55
+ value.each_with_object({}) { |(k, v), acc| acc[k.to_s] = json_safe(v) }
56
+ when Array
57
+ value.map { |v| json_safe(v) }
58
+ when String, Integer, Float, TrueClass, FalseClass, NilClass
59
+ value
60
+ else
61
+ if %i[name type description required].all? { |m| value.respond_to?(m) }
62
+ { "name" => value.name.to_s, "type" => value.type.to_s,
63
+ "description" => value.description.to_s, "required" => value.required }
64
+ elsif value.respond_to?(:to_h)
65
+ json_safe(value.to_h)
66
+ else
67
+ value.to_s
68
+ end
69
+ end
70
+ end
71
+
72
+ # The store round-trip: a plain Hash (string keys) -> ModelVisible.
73
+ def self.from_h(hash)
74
+ h = hash || {}
75
+ new(
76
+ instructions: h["instructions"],
77
+ tools: Array(h["tools"]),
78
+ messages: Array(h["messages"])
79
+ )
80
+ end
81
+
82
+ # JSON-safe (string keys), the shape the store persists.
83
+ def to_h
84
+ { "instructions" => instructions, "tools" => tools, "messages" => messages }
85
+ end
86
+ end
87
+ end
@@ -0,0 +1,66 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Insika
4
+ # — the durable half of the conformance claim: what the model
5
+ # received, per (task, turn), captured at the RubyLLM boundary and persisted.
6
+ #
7
+ # One record per (task, turn, part) under the scope "model_visible_traces",
8
+ # key "model_visible:<task_id>:turn:<n>:<part>" — `part` separates the
9
+ # turn's own ask ("turn") from engine-internal asks that are model-visible
10
+ # too (the WS4 routing classifier, "routing").
11
+ #
12
+ # A TRACE, best-effort by construction: `record` rescues everything -> nil
13
+ # (a log must never break a turn — the house rule, tool_trace_store.rb).
14
+ # The record is an UPSERT by (task, turn, part): a resumed turn re-records
15
+ # its ask in place (the ContextTraceStore idiom), never duplicates.
16
+ class ModelVisibleTraceStore
17
+ SCOPE = "model_visible_traces"
18
+
19
+ def initialize(store:)
20
+ @store = store
21
+ end
22
+
23
+ # -> ModelVisible | nil (nil = the record failed — the turn proceeds).
24
+ def record(task_id:, turn:, part: "turn", payload:)
25
+ return nil if payload.nil?
26
+
27
+ @store.set(SCOPE, key(task_id, turn, part), payload.to_h)
28
+ payload
29
+ rescue StandardError
30
+ nil
31
+ end
32
+
33
+ # -> ModelVisible | nil.
34
+ def find(task_id, turn:, part: "turn")
35
+ raw = @store.get(SCOPE, key(task_id, turn, part))
36
+ raw && ModelVisible.from_h(raw)
37
+ end
38
+
39
+ # -> [ModelVisible] the task's records for `part`, ordered by turn.
40
+ def for_task(task_id, part: "turn")
41
+ @store.list(SCOPE, key_prefix(task_id)).filter_map do |k|
42
+ next unless k.end_with?(":#{part}")
43
+
44
+ n = k[/:turn:(\d+):/, 1]
45
+ n && [Integer(n), ModelVisible.from_h(@store.get(SCOPE, k))]
46
+ end.sort_by(&:first).map(&:last)
47
+ end
48
+
49
+ # Removes every record for a task (retention/LGPD). -> Integer (removed).
50
+ def purge(task_id)
51
+ keys = @store.list(SCOPE, key_prefix(task_id))
52
+ keys.each { |k| @store.delete(SCOPE, k) }
53
+ keys.size
54
+ end
55
+
56
+ private
57
+
58
+ def key(task_id, turn, part)
59
+ "model_visible:#{task_id}:turn:#{Integer(turn)}:#{part}"
60
+ end
61
+
62
+ def key_prefix(task_id)
63
+ "model_visible:#{task_id}:turn:"
64
+ end
65
+ end
66
+ end
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Insika
4
- # LLM-first onboarding surface (item 20 / §5.6). The "Flue trick": the insika
4
+ # LLM-first onboarding surface. The "Flue trick": the insika
5
5
  # serves, from itself, a `start.md` addressed to the DEVELOPER'S OWN coding agent
6
6
  # ("Read <base>/start.md then help me build my first agent") plus a machine-readable
7
7
  # `/models.json` and the public docs mirrored as raw markdown. It is `rails new`
@@ -31,7 +31,7 @@ module Insika
31
31
  MODELS_SCHEMA_VERSION = 1
32
32
 
33
33
  # The PUBLIC docs allowlist, repo-relative: slug => path. Explicit on purpose —
34
- # the gitignored internal docs (FOLLOWUP / techspec / TRANSLATION-TRACKER / …)
34
+ # the gitignored internal docs (/ TRANSLATION-TRACKER / …)
35
35
  # are NEVER here, so /docs can only ever serve OSS material. Kept in sync with the
36
36
  # tracked `.md` prose (README + docs/*.md).
37
37
  PUBLIC_DOCS = {
@@ -54,8 +54,13 @@ module Insika
54
54
  "observability" => "docs/OBSERVABILITY.md",
55
55
  "evals" => "docs/EVALS.md",
56
56
  "refinement" => "docs/REFINEMENT.md",
57
+ "facts" => "docs/FACTS.md",
58
+ "harvest" => "docs/HARVEST.md",
57
59
  "loadtest" => "docs/LOADTEST.md",
58
- "releasing" => "docs/RELEASING.md"
60
+ "releasing" => "docs/RELEASING.md",
61
+ # the /v1 compatibility contract and the domain-free map.
62
+ "api" => "docs/API.md",
63
+ "domain" => "docs/domain.md"
59
64
  }.freeze
60
65
 
61
66
  # Repo-relative path to the start.md template.