insika 0.2.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (300) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +325 -17
  3. data/README.md +52 -11
  4. data/bin/insika +895 -8
  5. data/bin/insika-router +87 -0
  6. data/docs/AGENTS.md +229 -163
  7. data/docs/API.md +56 -0
  8. data/docs/ARCHITECTURE.md +3 -2
  9. data/docs/ARTIFACTS.md +95 -0
  10. data/docs/BENCHMARK.md +2 -2
  11. data/docs/CHANNELS.md +111 -21
  12. data/docs/CONTEXT.md +132 -19
  13. data/docs/DEMO.md +80 -0
  14. data/docs/DEPLOY.md +75 -4
  15. data/docs/EMBEDDING.md +1 -1
  16. data/docs/EVALS.md +146 -2
  17. data/docs/FACTS.md +135 -0
  18. data/docs/HARVEST.md +116 -0
  19. data/docs/KNOWLEDGE.md +290 -0
  20. data/docs/LOADTEST.md +16 -10
  21. data/docs/MEDIA.md +128 -0
  22. data/docs/OBSERVABILITY.md +65 -6
  23. data/docs/OUTCOMES.md +137 -0
  24. data/docs/PLUGINS.md +51 -6
  25. data/docs/POLICY.md +216 -0
  26. data/docs/REFINEMENT.md +17 -12
  27. data/docs/RELEASING.md +27 -0
  28. data/docs/ROUTER.md +213 -0
  29. data/docs/RUNNING-LOCAL.md +6 -6
  30. data/docs/SCHEDULING.md +121 -0
  31. data/docs/SECURITY.md +79 -2
  32. data/docs/SKILLS.md +13 -4
  33. data/docs/SOAK.md +127 -0
  34. data/docs/TEMPLATES.md +134 -0
  35. data/docs/TOOLS.md +220 -27
  36. data/docs/WHY.md +1 -1
  37. data/docs/WORKFLOWS.md +2 -2
  38. data/docs/_includes/head_custom.html +5 -0
  39. data/docs/_includes/title.html +13 -0
  40. data/docs/_sass/color_schemes/insika.scss +32 -0
  41. data/docs/_sass/custom/custom.scss +199 -0
  42. data/docs/_sass/custom/setup.scss +26 -0
  43. data/docs/assets/img/favicon.svg +7 -0
  44. data/docs/assets/img/insika-mark.svg +7 -0
  45. data/docs/core-concepts.md +21 -0
  46. data/docs/domain.md +115 -0
  47. data/docs/improve.md +20 -0
  48. data/docs/index.md +9 -6
  49. data/docs/integrate.md +20 -0
  50. data/docs/operate.md +13 -6
  51. data/docs/prompts/ADD-TOOL.md +118 -0
  52. data/docs/prompts/DIAGNOSE-TURN.md +65 -0
  53. data/docs/prompts/GO-LIVE.md +138 -0
  54. data/docs/prompts/RUN-EXAMPLES.md +70 -0
  55. data/docs/reference.md +19 -0
  56. data/docs/ship.md +10 -2
  57. data/docs/start-here.md +18 -0
  58. data/lib/insika/agent_profile.rb +202 -10
  59. data/lib/insika/artifact_signing.rb +82 -0
  60. data/lib/insika/artifact_store.rb +160 -0
  61. data/lib/insika/balloon_splitter.rb +102 -0
  62. data/lib/insika/budget_ledger.rb +34 -3
  63. data/lib/insika/cache_series_store.rb +49 -0
  64. data/lib/insika/channel_delivery.rb +119 -14
  65. data/lib/insika/channels/relay.rb +77 -3
  66. data/lib/insika/channels/web.rb +2 -2
  67. data/lib/insika/chat_builder.rb +105 -2
  68. data/lib/insika/checkpoint_store.rb +16 -0
  69. data/lib/insika/commands/agent_payload.rb +3 -3
  70. data/lib/insika/commands/backfill_knowledge.rb +145 -0
  71. data/lib/insika/commands/cancel_followup.rb +49 -0
  72. data/lib/insika/commands/delete_artifact.rb +35 -0
  73. data/lib/insika/commands/delete_concept.rb +34 -0
  74. data/lib/insika/commands/delete_mcp.rb +6 -2
  75. data/lib/insika/commands/delete_tenant_data.rb +107 -0
  76. data/lib/insika/commands/export_customer_memory.rb +48 -0
  77. data/lib/insika/commands/forget_customer.rb +117 -0
  78. data/lib/insika/commands/freeze_funnel_baseline.rb +113 -0
  79. data/lib/insika/commands/gate_harvest.rb +138 -0
  80. data/lib/insika/commands/gate_refinement.rb +1 -1
  81. data/lib/insika/commands/judge_shadow_pairs.rb +124 -0
  82. data/lib/insika/commands/memory_forget_fact.rb +20 -4
  83. data/lib/insika/commands/memory_put_fact.rb +23 -4
  84. data/lib/insika/commands/promote_harvest.rb +130 -0
  85. data/lib/insika/commands/record_outcome.rb +46 -0
  86. data/lib/insika/commands/record_shadow_reply.rb +68 -0
  87. data/lib/insika/commands/refresh_mcp_tools.rb +47 -0
  88. data/lib/insika/commands/reject_harvest.rb +38 -0
  89. data/lib/insika/commands/resolve_proposal.rb +108 -0
  90. data/lib/insika/commands/restore_concept.rb +34 -0
  91. data/lib/insika/commands/revoke_contact.rb +49 -0
  92. data/lib/insika/commands/rollback_harvest.rb +86 -0
  93. data/lib/insika/commands/run_distillation.rb +186 -0
  94. data/lib/insika/commands/run_harvest.rb +393 -0
  95. data/lib/insika/commands/seed_demo_data.rb +31 -0
  96. data/lib/insika/commands/send_message.rb +103 -6
  97. data/lib/insika/commands/session_purge.rb +67 -0
  98. data/lib/insika/commands/upsert_mcp.rb +6 -3
  99. data/lib/insika/commands/write_concept.rb +57 -0
  100. data/lib/insika/contact_store.rb +183 -0
  101. data/lib/insika/context/builder.rb +21 -3
  102. data/lib/insika/context/fragment.rb +7 -3
  103. data/lib/insika/context/priority.rb +5 -0
  104. data/lib/insika/context/provider.rb +17 -3
  105. data/lib/insika/context/providers/briefing.rb +96 -0
  106. data/lib/insika/context/providers/knowledge.rb +108 -0
  107. data/lib/insika/context/providers/memory.rb +16 -7
  108. data/lib/insika/context/providers/prompt.rb +57 -23
  109. data/lib/insika/context/providers/skill.rb +2 -0
  110. data/lib/insika/context/providers/tool_search.rb +2 -0
  111. data/lib/insika/context_trace_store.rb +38 -2
  112. data/lib/insika/cron.rb +189 -0
  113. data/lib/insika/demo/agent_attrs.rb +43 -0
  114. data/lib/insika/demo/golden_cases.rb +81 -0
  115. data/lib/insika/demo/seeder.rb +336 -0
  116. data/lib/insika/distill.rb +224 -0
  117. data/lib/insika/distill_engine.rb +169 -0
  118. data/lib/insika/doctor.rb +889 -11
  119. data/lib/insika/dsl/definition.rb +3 -2
  120. data/lib/insika/dsl/runtime.rb +69 -82
  121. data/lib/insika/dsl/server_boot.rb +92 -1
  122. data/lib/insika/dsl/system.rb +10 -2
  123. data/lib/insika/dsl.rb +186 -5
  124. data/lib/insika/edge_limiter.rb +28 -7
  125. data/lib/insika/env_schema.rb +25 -3
  126. data/lib/insika/errors.rb +11 -0
  127. data/lib/insika/evals/assertions.rb +3 -2
  128. data/lib/insika/evals/golden.rb +41 -4
  129. data/lib/insika/evals/judge.rb +47 -2
  130. data/lib/insika/evals/pairwise.rb +29 -6
  131. data/lib/insika/evals/persona.rb +98 -0
  132. data/lib/insika/evals/runner.rb +9 -0
  133. data/lib/insika/evals/simulator.rb +225 -0
  134. data/lib/insika/evals/transport.rb +83 -1
  135. data/lib/insika/event_stream.rb +10 -0
  136. data/lib/insika/evidence.rb +183 -0
  137. data/lib/insika/executor.rb +869 -76
  138. data/lib/insika/followup_engine.rb +207 -0
  139. data/lib/insika/followup_policy.rb +198 -0
  140. data/lib/insika/followup_store.rb +306 -0
  141. data/lib/insika/funnel_declaration.rb +106 -0
  142. data/lib/insika/funnel_fold.rb +179 -0
  143. data/lib/insika/funnel_store.rb +163 -0
  144. data/lib/insika/golden_store.rb +17 -2
  145. data/lib/insika/grounding/matcher.rb +69 -0
  146. data/lib/insika/grounding.rb +44 -0
  147. data/lib/insika/harvest/conversion_gate.rb +159 -0
  148. data/lib/insika/harvest/criterion.rb +98 -0
  149. data/lib/insika/harvest/gate.rb +194 -0
  150. data/lib/insika/harvest/negative_list.rb +199 -0
  151. data/lib/insika/harvest.rb +241 -0
  152. data/lib/insika/harvest_engine.rb +193 -0
  153. data/lib/insika/harvest_store.rb +548 -0
  154. data/lib/insika/knowledge.rb +680 -0
  155. data/lib/insika/knowledge_store.rb +140 -0
  156. data/lib/insika/mcp_client.rb +94 -0
  157. data/lib/insika/mcp_json.rb +74 -0
  158. data/lib/insika/mcp_live_tool.rb +43 -0
  159. data/lib/insika/mcp_store.rb +98 -26
  160. data/lib/insika/mcp_tool_ingestor.rb +30 -8
  161. data/lib/insika/mcp_tool_registry.rb +100 -0
  162. data/lib/insika/media.rb +382 -0
  163. data/lib/insika/memory_audit_store.rb +85 -0
  164. data/lib/insika/memory_store.rb +264 -23
  165. data/lib/insika/message_origin.rb +7 -2
  166. data/lib/insika/middleware.rb +9 -0
  167. data/lib/insika/model_visible.rb +87 -0
  168. data/lib/insika/model_visible_trace_store.rb +66 -0
  169. data/lib/insika/onboarding.rb +22 -1
  170. data/lib/insika/outbox_store.rb +42 -4
  171. data/lib/insika/outcome_store.rb +147 -0
  172. data/lib/insika/overlay_tool_registry.rb +37 -17
  173. data/lib/insika/packaging.rb +163 -0
  174. data/lib/insika/parity/criterion.rb +79 -0
  175. data/lib/insika/parity/verdict.rb +318 -0
  176. data/lib/insika/prefix_fingerprint.rb +58 -0
  177. data/lib/insika/profile_source.rb +29 -1
  178. data/lib/insika/prompt_catalog.rb +10 -0
  179. data/lib/insika/proposal_store.rb +271 -0
  180. data/lib/insika/queue_policy.rb +4 -1
  181. data/lib/insika/refinement/proposer.rb +1 -1
  182. data/lib/insika/reliability.rb +32 -6
  183. data/lib/insika/retention.rb +316 -0
  184. data/lib/insika/router/app.rb +157 -0
  185. data/lib/insika/router/backend_pool.rb +98 -0
  186. data/lib/insika/router/hash_ring.rb +55 -0
  187. data/lib/insika/router/proxy_body.rb +34 -0
  188. data/lib/insika/router/session_key.rb +54 -0
  189. data/lib/insika/router.rb +18 -0
  190. data/lib/insika/routing.rb +101 -0
  191. data/lib/insika/safety/config.rb +44 -4
  192. data/lib/insika/safety/corpus.rb +255 -0
  193. data/lib/insika/safety/detectors.rb +32 -113
  194. data/lib/insika/safety/factory.rb +15 -2
  195. data/lib/insika/safety/grounding_enforcer.rb +59 -0
  196. data/lib/insika/safety/grounding_validator.rb +49 -0
  197. data/lib/insika/safety/input_guardrail.rb +1 -1
  198. data/lib/insika/safety/moderator.rb +1 -1
  199. data/lib/insika/safety/output_filter.rb +9 -5
  200. data/lib/insika/safety/output_validator.rb +11 -5
  201. data/lib/insika/schedule.rb +177 -0
  202. data/lib/insika/schedule_engine.rb +314 -0
  203. data/lib/insika/schedule_store.rb +208 -0
  204. data/lib/insika/schema_guard.rb +35 -0
  205. data/lib/insika/server/app.rb +247 -17
  206. data/lib/insika/server/rack_app.rb +21 -1
  207. data/lib/insika/server/responses.rb +40 -1
  208. data/lib/insika/session_actor.rb +8 -4
  209. data/lib/insika/session_store.rb +65 -2
  210. data/lib/insika/settings_store.rb +10 -0
  211. data/lib/insika/shadow_pair_store.rb +258 -0
  212. data/lib/insika/skill_catalog.rb +16 -0
  213. data/lib/insika/soak/envelope.rb +140 -0
  214. data/lib/insika/soak/report.rb +392 -0
  215. data/lib/insika/soak/runner.rb +554 -0
  216. data/lib/insika/steer_injector.rb +21 -10
  217. data/lib/insika/store.rb +10 -1
  218. data/lib/insika/stores/memory.rb +6 -0
  219. data/lib/insika/stores/sqlite.rb +8 -0
  220. data/lib/insika/studio/app.rb +1437 -54
  221. data/lib/insika/studio/assets/dist/application.css +1 -1
  222. data/lib/insika/studio/assets/dist/application.js +23 -23
  223. data/lib/insika/studio/assets/dist/favicon.svg +6 -0
  224. data/lib/insika/studio/forms.rb +308 -10
  225. data/lib/insika/studio/nav_icons.rb +19 -1
  226. data/lib/insika/studio/views/_agent_tab_cache.erb +25 -0
  227. data/lib/insika/studio/views/_agent_tab_config.erb +514 -0
  228. data/lib/insika/studio/views/_agent_tab_history.erb +24 -0
  229. data/lib/insika/studio/views/_agent_tab_loops.erb +54 -0
  230. data/lib/insika/studio/views/_agent_tab_memory.erb +51 -0
  231. data/lib/insika/studio/views/_agent_tab_outcomes.erb +31 -0
  232. data/lib/insika/studio/views/_agent_tab_prompts.erb +108 -0
  233. data/lib/insika/studio/views/_agent_tab_skills.erb +38 -0
  234. data/lib/insika/studio/views/_agents_master.erb +44 -0
  235. data/lib/insika/studio/views/_message.erb +49 -32
  236. data/lib/insika/studio/views/agent_detail.erb +62 -278
  237. data/lib/insika/studio/views/agents.erb +71 -54
  238. data/lib/insika/studio/views/approvals.erb +4 -1
  239. data/lib/insika/studio/views/artifact.erb +23 -0
  240. data/lib/insika/studio/views/artifacts.erb +59 -0
  241. data/lib/insika/studio/views/chats.erb +4 -1
  242. data/lib/insika/studio/views/customer.erb +94 -0
  243. data/lib/insika/studio/views/customers.erb +32 -0
  244. data/lib/insika/studio/views/evals.erb +6 -3
  245. data/lib/insika/studio/views/facts.erb +133 -0
  246. data/lib/insika/studio/views/followups.erb +125 -0
  247. data/lib/insika/studio/views/funnel.erb +106 -0
  248. data/lib/insika/studio/views/harvest.erb +234 -0
  249. data/lib/insika/studio/views/home.erb +106 -66
  250. data/lib/insika/studio/views/knowledge.erb +123 -0
  251. data/lib/insika/studio/views/layout.erb +15 -11
  252. data/lib/insika/studio/views/mcp.erb +174 -80
  253. data/lib/insika/studio/views/parity.erb +147 -0
  254. data/lib/insika/studio/views/playground.erb +7 -1
  255. data/lib/insika/studio/views/session.erb +233 -124
  256. data/lib/insika/studio/views/settings.erb +41 -1
  257. data/lib/insika/studio/views/skills.erb +1 -2
  258. data/lib/insika/studio/views/system_files.erb +1 -1
  259. data/lib/insika/studio/views/task.erb +13 -0
  260. data/lib/insika/studio/views/tasks.erb +4 -1
  261. data/lib/insika/studio/views/tools.erb +24 -10
  262. data/lib/insika/task_store.rb +21 -1
  263. data/lib/insika/templates/browser-agent/README.md +36 -0
  264. data/lib/insika/templates/browser-agent/agent.rb +49 -0
  265. data/lib/insika/templates/daily-digest/README.md +38 -0
  266. data/lib/insika/templates/daily-digest/agent.rb +77 -0
  267. data/lib/insika/templates/repo-explorer/README.md +36 -0
  268. data/lib/insika/templates/repo-explorer/agent.rb +45 -0
  269. data/lib/insika/templates/research-analyst/README.md +26 -0
  270. data/lib/insika/templates/research-analyst/agent.rb +58 -0
  271. data/lib/insika/templates/review-panel/README.md +20 -0
  272. data/lib/insika/templates/review-panel/agent.rb +50 -0
  273. data/lib/insika/templates/travel-planner/README.md +35 -0
  274. data/lib/insika/templates/travel-planner/agent.rb +87 -0
  275. data/lib/insika/templates.rb +112 -0
  276. data/lib/insika/testing/store_contract.rb +27 -6
  277. data/lib/insika/tick.rb +47 -11
  278. data/lib/insika/timezone.rb +45 -0
  279. data/lib/insika/tool_definition.rb +17 -7
  280. data/lib/insika/tool_envelope.rb +69 -0
  281. data/lib/insika/tool_manifest.rb +5 -1
  282. data/lib/insika/tools/data_defined_tool.rb +10 -0
  283. data/lib/insika/tools/generate_image.rb +89 -0
  284. data/lib/insika/tools/load_knowledge.rb +74 -0
  285. data/lib/insika/tools/run_persona_eval.rb +328 -0
  286. data/lib/insika/tools/save_artifact.rb +95 -0
  287. data/lib/insika/tools/schedule_followup.rb +164 -0
  288. data/lib/insika/tools/tts.rb +47 -0
  289. data/lib/insika/tools/update_briefing.rb +126 -0
  290. data/lib/insika/turn_output.rb +1 -1
  291. data/lib/insika/turn_state.rb +49 -1
  292. data/lib/insika/turn_timing.rb +22 -2
  293. data/lib/insika/version.rb +1 -1
  294. data/lib/insika/vitals.rb +84 -0
  295. data/lib/insika/wiring/graph.rb +472 -13
  296. data/lib/insika/wiring/graph_chat.rb +102 -0
  297. data/lib/insika.rb +149 -3
  298. metadata +181 -5
  299. data/docs/build.md +0 -14
  300. data/docs/understand.md +0 -10
@@ -10,6 +10,14 @@ module Insika
10
10
  # `PendingActionStore` (normalizes symbol→string on write, O(n) scan on
11
11
  # read, records with a timestamp).
12
12
  #
13
+ # the record gains `origin`/`created_at`/`expires_at`, the
14
+ # (tenant, customer) pair becomes first-class at the API (`customer:`
15
+ # builds the same "memory:<tenant>:<customer>" scope the Executor derives),
16
+ # and the store enumerates its cells (the Studio drill, the doctor, the
17
+ # retention sweep). Migration is lazy: tolerant reads, materialized on the
18
+ # first write. The audit trail lives in `MemoryAuditStore` (C2), NOT here —
19
+ # this stays a dumb domain store.
20
+ #
13
21
  # NOT to be confused with `Insika::Stores::Memory` (in-memory KV backend):
14
22
  # this is the domain store; that one is one of the backends this writes to.
15
23
  class MemoryStore
@@ -18,52 +26,246 @@ module Insika
18
26
  NOTE_PREFIX = "note:"
19
27
  DEFAULT_TENANT = "_default" # no tenant in the Command
20
28
 
21
- Fact = Data.define(:key, :value, :updated_at)
29
+ # the record's provenance. A closed string set: "legacy"
30
+ # (migrated, unknown writer), "engine" (the remember tool / an
31
+ # integration write), "operator" (Studio edit), "distilled"
32
+ # reserved).
33
+ ORIGIN_LEGACY = "legacy"
34
+
35
+ # the per-SESSION cell marker. Engine-owner memory with no
36
+ # customer and no explicit tenant falls back to the session (executor
37
+ # parity) — the cell is MARKED "memory:chat:<session id>" so the drill and
38
+ # the doctor never read a conversation as a customer (a bare
39
+ # "memory:<session id>" cell is indistinguishable from a single-tenant
40
+ # customer ref). A real tenant NAMED "chat" would collide with the marker
41
+ # (its customer cells would read as session cells) — accepted, merchant
42
+ # ids live in a different namespace in practice.
43
+ SESSION_TAG = "chat"
44
+
45
+ Fact = Data.define(:key, :value, :origin, :created_at, :updated_at, :expires_at)
22
46
  Note = Data.define(:id, :text, :created_at)
23
47
 
24
- def initialize(store:)
48
+ def initialize(store:, clock: nil)
25
49
  @store = store
50
+ @clock = clock # -> Time, injectable for specs
26
51
  end
27
52
 
28
- # Upsert (last-write-wins, Store contract). -> Fact
29
- def put_fact(tenant:, key:, value:)
30
- record = { "key" => key.to_s, "value" => stringify(value), "updated_at" => timestamp }
31
- @store.set(scope_for(tenant), FACT_PREFIX + key.to_s, record)
53
+ # Upsert (last-write-wins, Store contract). `customer:` PRESENT moves the
54
+ # scope to the [tenant:]customer cell (WS8's rule — nil tenant + customer
55
+ # -> bare "memory:<customer>", NEVER _default). `origin` defaults to
56
+ # "engine" (the remember tool's write). `expires_at` (ISO8601 String) is
57
+ # normalized and validated here — an invalid value raises ValidationError
58
+ # (a silently dropped LGPD expiry date is the defect). Preserves the
59
+ # existing record's created_at; rewrites a legacy record in the full
60
+ # shape. -> Fact
61
+ def put_fact(tenant:, key:, value:, customer: nil, origin: "engine", expires_at: nil)
62
+ scope = scope_for(tenant, customer)
63
+ current = @store.get(scope, FACT_PREFIX + key.to_s)
64
+ prev = current.is_a?(Hash) ? current : {}
65
+ stamp = timestamp
66
+ record = {
67
+ "key" => key.to_s, "value" => stringify(value),
68
+ "origin" => Coercion.presence(origin) || "engine",
69
+ "created_at" => Coercion.presence(prev["created_at"]) || Coercion.presence(prev["updated_at"]) || stamp,
70
+ "updated_at" => stamp,
71
+ "expires_at" => normalize_expiry(expires_at)
72
+ }
73
+ @store.set(scope, FACT_PREFIX + key.to_s, record)
32
74
  to_fact(record)
33
75
  end
34
76
 
35
- # -> Fact | nil
36
- def get_fact(tenant:, key:)
37
- record = @store.get(scope_for(tenant), FACT_PREFIX + key.to_s)
77
+ # -> Fact | nil (tolerant read: missing origin -> "legacy", created_at ->
78
+ # updated_at, expires_at -> nil).
79
+ def get_fact(tenant:, key:, customer: nil)
80
+ record = @store.get(scope_for(tenant, customer), FACT_PREFIX + key.to_s)
38
81
  record && to_fact(record)
39
82
  end
40
83
 
41
- # -> [Fact] sorted by key (list is lexicographic).
42
- def facts(tenant:)
43
- scope = scope_for(tenant)
84
+ # -> [Fact] sorted by key. Facts whose expires_at <= now are EXCLUDED
85
+ # (D5: an expired fact is never injected nor shown, even before the
86
+ # sweep runs).
87
+ def facts(tenant:, customer: nil)
88
+ scope = scope_for(tenant, customer)
44
89
  @store.list(scope, FACT_PREFIX).filter_map do |k|
45
90
  record = @store.get(scope, k)
46
- record && to_fact(record)
91
+ next unless record
92
+
93
+ fact = to_fact(record)
94
+ next if expired?(fact)
95
+
96
+ fact
47
97
  end
48
98
  end
49
99
 
50
100
  # -> bool (did it exist?)
51
- def forget_fact(tenant:, key:)
52
- @store.delete(scope_for(tenant), FACT_PREFIX + key.to_s)
101
+ def forget_fact(tenant:, key:, customer: nil)
102
+ @store.delete(scope_for(tenant, customer), FACT_PREFIX + key.to_s)
103
+ end
104
+
105
+ # CAS write (WS8): only writes when the STORED fact's updated_at still
106
+ # equals the caller's revision — a fact a concurrent writer already moved
107
+ # is refused instead of clobbered (last-write-wins is the default; this is
108
+ # the opt-in optimistic path). -> Fact (written) | nil (lost the race — the
109
+ # caller must re-read and retry). The read-compare-write rides
110
+ # `@store.transaction` (the repo rule) — `next`, never `return`, inside.
111
+ def replace_if_revision(tenant:, key:, value:, expected_revision:,
112
+ customer: nil, origin: "engine", expires_at: nil)
113
+ @store.transaction do
114
+ current = @store.get(scope_for(tenant, customer), FACT_PREFIX + key.to_s)
115
+ next nil if current.nil? || current["updated_at"] != expected_revision
116
+
117
+ put_fact(tenant: tenant, key: key, value: value, customer: customer,
118
+ origin: origin, expires_at: expires_at)
119
+ end
120
+ end
121
+
122
+ # Purges the WHOLE scope (WS8 — forget_customer / delete_tenant_data). The
123
+ # scope string is the isolation boundary: one cell per
124
+ # (tenant-or-customer), so zeroing the cell cannot touch another's.
125
+ # The list-then-delete rides `@store.transaction` (the repo rule): an erasure
126
+ # that is half-applied is the LGPD defect, and a fact written between the
127
+ # list and the deletes would survive a purge that reported success.
128
+ # -> count of records purged.
129
+ def purge(tenant:, customer: nil)
130
+ scope = scope_for(tenant, customer)
131
+ @store.transaction do
132
+ keys = @store.list(scope)
133
+ keys.each { |k| @store.delete(scope, k) }
134
+ keys.size
135
+ end
136
+ end
137
+
138
+ # Purges a TENANT and every customer cell under it (WS8 phase 2 —
139
+ # delete_tenant_data). The tenant's own cell ("memory:<tenant>") plus each
140
+ # cell whose scope starts with "memory:<tenant>:" (the customer cells),
141
+ # plus the tenant's SESSION-marked cells ("memory:chat:<tenant>:*" — in
142
+ # multi-tenant a session id is "<tenant>:<id>", so its memory cell is
143
+ # "memory:chat:<tenant>:<id>"). The scope enumeration is the Store's — no
144
+ # session-derived list, so a customer cell whose session was already
145
+ # deleted is still purged. ONE transaction for the whole tenant (the repo
146
+ # rule): the scope enumeration and every delete see the same snapshot, so a
147
+ # customer cell born mid-purge cannot slip through a deletion that reported
148
+ # success. -> count of records purged.
149
+ def purge_tenant(tenant)
150
+ cell = scope_for(tenant)
151
+ @store.transaction do
152
+ scopes = [cell] + @store.scopes("#{cell}:") +
153
+ @store.scopes("#{SCOPE_PREFIX}:#{SESSION_TAG}:#{tenant}:")
154
+ scopes.sum do |scope|
155
+ keys = @store.list(scope)
156
+ keys.each { |k| @store.delete(scope, k) }
157
+ keys.size
158
+ end
159
+ end
160
+ end
161
+
162
+ # Age-based prune (WS8 retention). New : `scope:` limits the pass
163
+ # to ONE cell; a fact with an explicit expires_at is SKIPPED (D5 — the
164
+ # explicit override owns that fact's life). -> count removed.
165
+ def prune_older_than(time, scope: nil)
166
+ cutoff = time.utc.iso8601
167
+ removed = 0
168
+ scopes = scope ? [scope] : @store.scopes("#{SCOPE_PREFIX}:")
169
+ scopes.each do |sc|
170
+ @store.list(sc).each do |k|
171
+ rec = @store.get(sc, k)
172
+ next if rec && rec["expires_at"] # the override owns this fact's life
173
+
174
+ stamp = rec && (rec["updated_at"] || rec["created_at"])
175
+ next unless stamp && stamp.to_s < cutoff
176
+
177
+ @store.delete(sc, k)
178
+ removed += 1
179
+ end
180
+ end
181
+ removed
182
+ end
183
+
184
+ # Removes facts whose expires_at <= now (default clock), EVERY cell.
185
+ # -> count removed.
186
+ def prune_expired(now = nil)
187
+ cutoff = (now || self.now).utc
188
+ removed = 0
189
+ @store.scopes("#{SCOPE_PREFIX}:").each do |scope|
190
+ @store.list(scope).each do |k|
191
+ rec = @store.get(scope, k)
192
+ next unless rec && rec["expires_at"]
193
+
194
+ begin
195
+ next unless Time.iso8601(rec["expires_at"]) <= cutoff
196
+ rescue ArgumentError
197
+ next
198
+ end
199
+
200
+ @store.delete(scope, k)
201
+ removed += 1
202
+ end
203
+ end
204
+ removed
205
+ end
206
+
207
+ # --- enumeration (the drill / doctor / retention inputs) -----------
208
+
209
+ # -> [{scope:, tenant: String|nil, customer: String|nil}] — every "memory:*"
210
+ # scope, classified by SHAPE (D6). _default -> {tenant: nil, customer: nil}.
211
+ def cells
212
+ @store.scopes("#{SCOPE_PREFIX}:").map { |scope| self.class.parse_cell(scope) }
213
+ end
214
+
215
+ # -> [{scope:, tenant:, customer:}] — the cells that hold CUSTOMER memory:
216
+ # 2+-segment scopes always (except the SESSION-marked cells — a session is
217
+ # never a customer); 1-segment scopes whose name is not _default and not in
218
+ # `reserved` (the caller's agent ids / tenant cells). Sorted by scope.
219
+ def customer_cells(reserved: [])
220
+ excluded = Array(reserved).map(&:to_s)
221
+ cells.reject do |c|
222
+ self.class.session_cell?(c) ||
223
+ (c[:tenant].nil? && (c[:customer].nil? || excluded.include?(c[:customer])))
224
+ end.sort_by { |c| c[:scope] }
225
+ end
226
+
227
+ # Is this a classified cell the engine's per-SESSION cell
228
+ # ("memory:chat:<session id>")? The one classification both the drill and
229
+ # the doctor share — a session-derived cell must never read as a customer.
230
+ def self.session_cell?(cell) = cell[:tenant] == SESSION_TAG
231
+
232
+ # The public cell string for a pair (the commands and the Studio build URLs
233
+ # and audit keys with this — the same string the Executor derives).
234
+ def cell_for(tenant, customer = nil) = scope_for(tenant, customer)
235
+
236
+ # The shared classification (public — the Studio/commands split audit
237
+ # fields): "memory:acme:c-123" -> {scope:, tenant: "acme", customer: "c-123"};
238
+ # "memory:c-123" -> {scope:, tenant: nil, customer: "c-123"};
239
+ # "memory:_default" -> {scope:, tenant: nil, customer: nil}.
240
+ def self.parse_cell(scope)
241
+ rest = scope.to_s.sub(/\A#{SCOPE_PREFIX}:/, "")
242
+ base, tail = rest.split(":", 2)
243
+ if base == DEFAULT_TENANT && tail.nil?
244
+ { scope: scope, tenant: nil, customer: nil }
245
+ elsif tail
246
+ { scope: scope, tenant: base, customer: tail }
247
+ else
248
+ { scope: scope, tenant: nil, customer: base }
249
+ end
53
250
  end
54
251
 
55
252
  # Append. `at` (ISO8601) goes at the START of the key so `list` returns the notes in
56
253
  # chronological order; `id`/`at` injectable for deterministic tests. -> Note
57
- def add_note(tenant:, text:, id: SecureRandom.uuid, at: nil)
254
+ def add_note(tenant:, text:, id: SecureRandom.uuid, at: nil, customer: nil)
58
255
  at ||= timestamp
59
256
  record = { "id" => id.to_s, "text" => text.to_s, "created_at" => at }
60
- @store.set(scope_for(tenant), NOTE_PREFIX + "#{at}:#{id}", record)
257
+ @store.set(scope_for(tenant, customer), NOTE_PREFIX + "#{at}:#{id}", record)
61
258
  to_note(record)
62
259
  end
63
260
 
261
+ # Cheap fact count for the drill index (keys only, no payload reads). -> Integer
262
+ def fact_count(tenant:, customer: nil)
263
+ @store.list(scope_for(tenant, customer), FACT_PREFIX).size
264
+ end
265
+
64
266
  # -> [Note] MOST RECENT first, capped by `limit`.
65
- def notes(tenant:, limit: nil)
66
- scope = scope_for(tenant)
267
+ def notes(tenant:, limit: nil, customer: nil)
268
+ scope = scope_for(tenant, customer)
67
269
  keys = @store.list(scope, NOTE_PREFIX).reverse # list is chronological -> reverse = most recent first
68
270
  keys = keys.first(limit) if limit
69
271
  keys.filter_map do |k|
@@ -74,12 +276,51 @@ module Insika
74
276
 
75
277
  private
76
278
 
77
- def scope_for(tenant) = "#{SCOPE_PREFIX}:#{tenant.nil? || tenant.to_s.empty? ? DEFAULT_TENANT : tenant}"
279
+ # WS8's rule (executor.rb:1679 parity): customer present -> the
280
+ # [tenant:]customer cell; nil tenant + customer -> the bare customer cell,
281
+ # NEVER memory:_default:<customer>.
282
+ def scope_for(tenant, customer = nil)
283
+ base = Coercion.blank?(tenant) ? DEFAULT_TENANT : tenant
284
+ cust = Coercion.presence(customer)
285
+ return "#{SCOPE_PREFIX}:#{cust}" if cust && Coercion.blank?(tenant)
286
+
287
+ cust ? "#{SCOPE_PREFIX}:#{base}:#{cust}" : "#{SCOPE_PREFIX}:#{base}"
288
+ end
289
+
290
+ def to_fact(record)
291
+ Fact.new(key: record["key"], value: record["value"],
292
+ origin: Coercion.presence(record["origin"]) || ORIGIN_LEGACY,
293
+ created_at: Coercion.presence(record["created_at"]) || Coercion.presence(record["updated_at"]),
294
+ updated_at: record["updated_at"], expires_at: record["expires_at"])
295
+ end
78
296
 
79
- def to_fact(record) = Fact.new(key: record["key"], value: record["value"], updated_at: record["updated_at"])
80
297
  def to_note(record) = Note.new(id: record["id"], text: record["text"], created_at: record["created_at"])
81
298
 
82
- def timestamp = Time.now.utc.iso8601
299
+ def expired?(fact)
300
+ return false if fact.expires_at.nil?
301
+
302
+ Time.iso8601(fact.expires_at) <= now
303
+ rescue ArgumentError
304
+ false
305
+ end
306
+
307
+ def now = (@clock ? @clock.call : Time.now.utc).utc
308
+
309
+ # Microsecond precision ON PURPOSE: updated_at is the CAS revision
310
+ # (replace_if_revision) — second-precision collides for two writes in the
311
+ # same second (WS8).
312
+ def timestamp = now.iso8601(6)
313
+
314
+ # nil | canonical ISO8601(6); invalid -> ValidationError (fail-fast on an
315
+ # LGPD field — a silently dropped expiry date is the defect).
316
+ def normalize_expiry(expires_at)
317
+ return nil if expires_at.nil? || expires_at.to_s.strip.empty?
318
+
319
+ Time.iso8601(expires_at.to_s).utc.iso8601(6)
320
+ rescue ArgumentError
321
+ raise Insika::ValidationError,
322
+ "expires_at must be an ISO8601 timestamp (got #{expires_at.inspect})"
323
+ end
83
324
 
84
325
  def stringify(obj)
85
326
  case obj
@@ -90,4 +331,4 @@ module Insika
90
331
  end
91
332
  end
92
333
  end
93
- end
334
+ end
@@ -18,7 +18,7 @@ module Insika
18
18
  # Reading a transcript without that distinction is not a rounding error. The first
19
19
  # refinement run over real traffic reported `repetition ×219` on one agent: every
20
20
  # one of them the engine reading its own injected fragment back and calling it a
21
- # customer repeating themselves (PR #133). That was filtered by a REGEX on the
21
+ # customer repeating themselves. That was filtered by a REGEX on the
22
22
  # leading tag, labelled in the code as a heuristic standing in for this field.
23
23
  #
24
24
  # ABSENT is the common case and stays valid forever: a message with no origin is
@@ -33,8 +33,13 @@ module Insika
33
33
  AGENT = "agent" # the model (the default for `assistant`)
34
34
  ENGINE = "engine" # Insika itself, or the consumer composing on its behalf
35
35
  OPERATOR = "operator" # a HUMAN on the assistant side (a handoff; set by importers)
36
+ # the FollowupEngine's synthetic turn — the engine's own kick,
37
+ # never the customer. RESERVED: a consumer cannot declare it (the
38
+ # SendMessage edge refuses the spelling — only the engine creates those
39
+ # turns, D5).
40
+ SCHEDULED = "scheduled"
36
41
 
37
- ALL = [CUSTOMER, AGENT, ENGINE, OPERATOR].freeze
42
+ ALL = [CUSTOMER, AGENT, ENGINE, OPERATOR, SCHEDULED].freeze
38
43
 
39
44
  module_function
40
45
 
@@ -26,6 +26,15 @@ module Insika
26
26
  @middlewares = middlewares
27
27
  end
28
28
 
29
+ # Appends a link at the END (innermost). This is the plugin Loader's
30
+ # registration seam ("collections that respond to <<") — it runs at boot,
31
+ # single-fiber, before the server accepts connections, so appending here
32
+ # never races a turn.
33
+ def <<(middleware)
34
+ @middlewares << middleware
35
+ self
36
+ end
37
+
29
38
  def call(state, &terminal)
30
39
  chain = @middlewares.reverse.reduce(terminal) do |nxt, mw|
31
40
  proc { |s| mw.call(s, &nxt) }
@@ -0,0 +1,87 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Insika
4
+ # — the model-visible payload of ONE ask: the system text,
5
+ # the tool schemas, and the full message stream — exactly what the provider
6
+ # serializes. Captured AT the RubyLLM boundary (the chat object the executor
7
+ # hands to `ask`), never re-derived: re-derivation would re-run volatile
8
+ # context providers (LLM moderation, timestamps) and could drift from the
9
+ # bytes actually sent.
10
+ #
11
+ # "Model-visible means logged": the durable reconstruction of a turn is the
12
+ # checkpoint (messages — the transcript half) + this record (instructions +
13
+ # tool schemas — the half the checkpoint lacks). The conformance suite
14
+ # (spec/insika/conformance/model_visible_spec.rb) asserts the three-way byte
15
+ # identity: capturing chat == checkpoint == trace.
16
+ #
17
+ # Pure reads off the chat object with respond_to? guards — a chat lacking a
18
+ # reader contributes nil/[], never raises (the trace discipline).
19
+ ModelVisible = Data.define(:instructions, :tools, :messages) do
20
+ # -> ModelVisible. `chat` is the RubyLLM chat at the boundary (instructions,
21
+ # tools, messages are exactly the three parts the provider serializes).
22
+ def self.capture(chat)
23
+ new(
24
+ instructions: chat.respond_to?(:instructions) ? chat.instructions : nil,
25
+ tools: chat.respond_to?(:tools) ? Array(chat.tools).map { |t| tool_schema(t) } : [],
26
+ messages: chat.respond_to?(:messages) ? Array(chat.messages) : []
27
+ )
28
+ end
29
+
30
+ # -> Hash: the JSON-safe schema of ONE tool. Reads `parameters` then
31
+ # `schema` (the two reader shapes the house FakeChat and the gem's tools
32
+ # answer); a tool answering none degrades to nil, never raises.
33
+ def self.tool_schema(tool)
34
+ params = if tool.respond_to?(:parameters)
35
+ tool.parameters
36
+ elsif tool.respond_to?(:schema)
37
+ tool.schema
38
+ end
39
+ {
40
+ "name" => (tool.name.to_s if tool.respond_to?(:name)),
41
+ "description" => (tool.description.to_s if tool.respond_to?(:description)),
42
+ "parameters" => params && json_safe(params)
43
+ }.compact
44
+ end
45
+
46
+ # -> JSON-safe projection of an arbitrary schema value. The gem's tools
47
+ # answer parameters as Hash with RubyLLM::Parameter values (a class whose
48
+ # readers are name/type/description/required) — the trace must persist the
49
+ # schema, so it projects that shape instead of raising on the object (the
50
+ # record never breaks the turn). Anything unprojectable degrades to its
51
+ # string form, never raises.
52
+ def self.json_safe(value)
53
+ case value
54
+ when Hash
55
+ value.each_with_object({}) { |(k, v), acc| acc[k.to_s] = json_safe(v) }
56
+ when Array
57
+ value.map { |v| json_safe(v) }
58
+ when String, Integer, Float, TrueClass, FalseClass, NilClass
59
+ value
60
+ else
61
+ if %i[name type description required].all? { |m| value.respond_to?(m) }
62
+ { "name" => value.name.to_s, "type" => value.type.to_s,
63
+ "description" => value.description.to_s, "required" => value.required }
64
+ elsif value.respond_to?(:to_h)
65
+ json_safe(value.to_h)
66
+ else
67
+ value.to_s
68
+ end
69
+ end
70
+ end
71
+
72
+ # The store round-trip: a plain Hash (string keys) -> ModelVisible.
73
+ def self.from_h(hash)
74
+ h = hash || {}
75
+ new(
76
+ instructions: h["instructions"],
77
+ tools: Array(h["tools"]),
78
+ messages: Array(h["messages"])
79
+ )
80
+ end
81
+
82
+ # JSON-safe (string keys), the shape the store persists.
83
+ def to_h
84
+ { "instructions" => instructions, "tools" => tools, "messages" => messages }
85
+ end
86
+ end
87
+ end
@@ -0,0 +1,66 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Insika
4
+ # — the durable half of the conformance claim: what the model
5
+ # received, per (task, turn), captured at the RubyLLM boundary and persisted.
6
+ #
7
+ # One record per (task, turn, part) under the scope "model_visible_traces",
8
+ # key "model_visible:<task_id>:turn:<n>:<part>" — `part` separates the
9
+ # turn's own ask ("turn") from engine-internal asks that are model-visible
10
+ # too (the WS4 routing classifier, "routing").
11
+ #
12
+ # A TRACE, best-effort by construction: `record` rescues everything -> nil
13
+ # (a log must never break a turn — the house rule, tool_trace_store.rb).
14
+ # The record is an UPSERT by (task, turn, part): a resumed turn re-records
15
+ # its ask in place (the ContextTraceStore idiom), never duplicates.
16
+ class ModelVisibleTraceStore
17
+ SCOPE = "model_visible_traces"
18
+
19
+ def initialize(store:)
20
+ @store = store
21
+ end
22
+
23
+ # -> ModelVisible | nil (nil = the record failed — the turn proceeds).
24
+ def record(task_id:, turn:, part: "turn", payload:)
25
+ return nil if payload.nil?
26
+
27
+ @store.set(SCOPE, key(task_id, turn, part), payload.to_h)
28
+ payload
29
+ rescue StandardError
30
+ nil
31
+ end
32
+
33
+ # -> ModelVisible | nil.
34
+ def find(task_id, turn:, part: "turn")
35
+ raw = @store.get(SCOPE, key(task_id, turn, part))
36
+ raw && ModelVisible.from_h(raw)
37
+ end
38
+
39
+ # -> [ModelVisible] the task's records for `part`, ordered by turn.
40
+ def for_task(task_id, part: "turn")
41
+ @store.list(SCOPE, key_prefix(task_id)).filter_map do |k|
42
+ next unless k.end_with?(":#{part}")
43
+
44
+ n = k[/:turn:(\d+):/, 1]
45
+ n && [Integer(n), ModelVisible.from_h(@store.get(SCOPE, k))]
46
+ end.sort_by(&:first).map(&:last)
47
+ end
48
+
49
+ # Removes every record for a task (retention/LGPD). -> Integer (removed).
50
+ def purge(task_id)
51
+ keys = @store.list(SCOPE, key_prefix(task_id))
52
+ keys.each { |k| @store.delete(SCOPE, k) }
53
+ keys.size
54
+ end
55
+
56
+ private
57
+
58
+ def key(task_id, turn, part)
59
+ "model_visible:#{task_id}:turn:#{Integer(turn)}:#{part}"
60
+ end
61
+
62
+ def key_prefix(task_id)
63
+ "model_visible:#{task_id}:turn:"
64
+ end
65
+ end
66
+ end
@@ -38,24 +38,45 @@ module Insika
38
38
  "readme" => "README.md",
39
39
  "why" => "docs/WHY.md",
40
40
  "agents" => "docs/AGENTS.md",
41
+ "policy" => "docs/POLICY.md",
41
42
  "tools" => "docs/TOOLS.md",
42
43
  "skills" => "docs/SKILLS.md",
43
44
  "context" => "docs/CONTEXT.md",
44
45
  "workflows" => "docs/WORKFLOWS.md",
45
46
  "channels" => "docs/CHANNELS.md",
46
47
  "plugins" => "docs/PLUGINS.md",
48
+ "templates" => "docs/TEMPLATES.md",
49
+ "media" => "docs/MEDIA.md",
47
50
  "security" => "docs/SECURITY.md",
48
51
  "architecture" => "docs/ARCHITECTURE.md",
49
52
  "running-local" => "docs/RUNNING-LOCAL.md",
50
53
  "deploy" => "docs/DEPLOY.md",
54
+ "router" => "docs/ROUTER.md",
51
55
  "embedding" => "docs/EMBEDDING.md",
52
56
  "sandbox" => "docs/SANDBOX.md",
53
57
  "benchmark" => "docs/BENCHMARK.md",
54
58
  "observability" => "docs/OBSERVABILITY.md",
59
+ "schedules" => "docs/SCHEDULING.md",
60
+ "artifacts" => "docs/ARTIFACTS.md",
61
+ "soak" => "docs/SOAK.md",
55
62
  "evals" => "docs/EVALS.md",
56
63
  "refinement" => "docs/REFINEMENT.md",
64
+ "outcomes" => "docs/OUTCOMES.md",
65
+ "facts" => "docs/FACTS.md",
66
+ "harvest" => "docs/HARVEST.md",
67
+ "knowledge" => "docs/KNOWLEDGE.md",
57
68
  "loadtest" => "docs/LOADTEST.md",
58
- "releasing" => "docs/RELEASING.md"
69
+ "releasing" => "docs/RELEASING.md",
70
+ "demo" => "docs/DEMO.md",
71
+ # the /v1 compatibility contract and the domain-free map.
72
+ "api" => "docs/API.md",
73
+ "domain" => "docs/domain.md",
74
+ # copy-paste journeys for the developer's coding agent (README "Or let your
75
+ # coding agent build it").
76
+ "run-examples" => "docs/prompts/RUN-EXAMPLES.md",
77
+ "add-tool" => "docs/prompts/ADD-TOOL.md",
78
+ "diagnose-turn" => "docs/prompts/DIAGNOSE-TURN.md",
79
+ "go-live" => "docs/prompts/GO-LIVE.md"
59
80
  }.freeze
60
81
 
61
82
  # Repo-relative path to the start.md template.