insika 0.2.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (300) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +325 -17
  3. data/README.md +52 -11
  4. data/bin/insika +895 -8
  5. data/bin/insika-router +87 -0
  6. data/docs/AGENTS.md +229 -163
  7. data/docs/API.md +56 -0
  8. data/docs/ARCHITECTURE.md +3 -2
  9. data/docs/ARTIFACTS.md +95 -0
  10. data/docs/BENCHMARK.md +2 -2
  11. data/docs/CHANNELS.md +111 -21
  12. data/docs/CONTEXT.md +132 -19
  13. data/docs/DEMO.md +80 -0
  14. data/docs/DEPLOY.md +75 -4
  15. data/docs/EMBEDDING.md +1 -1
  16. data/docs/EVALS.md +146 -2
  17. data/docs/FACTS.md +135 -0
  18. data/docs/HARVEST.md +116 -0
  19. data/docs/KNOWLEDGE.md +290 -0
  20. data/docs/LOADTEST.md +16 -10
  21. data/docs/MEDIA.md +128 -0
  22. data/docs/OBSERVABILITY.md +65 -6
  23. data/docs/OUTCOMES.md +137 -0
  24. data/docs/PLUGINS.md +51 -6
  25. data/docs/POLICY.md +216 -0
  26. data/docs/REFINEMENT.md +17 -12
  27. data/docs/RELEASING.md +27 -0
  28. data/docs/ROUTER.md +213 -0
  29. data/docs/RUNNING-LOCAL.md +6 -6
  30. data/docs/SCHEDULING.md +121 -0
  31. data/docs/SECURITY.md +79 -2
  32. data/docs/SKILLS.md +13 -4
  33. data/docs/SOAK.md +127 -0
  34. data/docs/TEMPLATES.md +134 -0
  35. data/docs/TOOLS.md +220 -27
  36. data/docs/WHY.md +1 -1
  37. data/docs/WORKFLOWS.md +2 -2
  38. data/docs/_includes/head_custom.html +5 -0
  39. data/docs/_includes/title.html +13 -0
  40. data/docs/_sass/color_schemes/insika.scss +32 -0
  41. data/docs/_sass/custom/custom.scss +199 -0
  42. data/docs/_sass/custom/setup.scss +26 -0
  43. data/docs/assets/img/favicon.svg +7 -0
  44. data/docs/assets/img/insika-mark.svg +7 -0
  45. data/docs/core-concepts.md +21 -0
  46. data/docs/domain.md +115 -0
  47. data/docs/improve.md +20 -0
  48. data/docs/index.md +9 -6
  49. data/docs/integrate.md +20 -0
  50. data/docs/operate.md +13 -6
  51. data/docs/prompts/ADD-TOOL.md +118 -0
  52. data/docs/prompts/DIAGNOSE-TURN.md +65 -0
  53. data/docs/prompts/GO-LIVE.md +138 -0
  54. data/docs/prompts/RUN-EXAMPLES.md +70 -0
  55. data/docs/reference.md +19 -0
  56. data/docs/ship.md +10 -2
  57. data/docs/start-here.md +18 -0
  58. data/lib/insika/agent_profile.rb +202 -10
  59. data/lib/insika/artifact_signing.rb +82 -0
  60. data/lib/insika/artifact_store.rb +160 -0
  61. data/lib/insika/balloon_splitter.rb +102 -0
  62. data/lib/insika/budget_ledger.rb +34 -3
  63. data/lib/insika/cache_series_store.rb +49 -0
  64. data/lib/insika/channel_delivery.rb +119 -14
  65. data/lib/insika/channels/relay.rb +77 -3
  66. data/lib/insika/channels/web.rb +2 -2
  67. data/lib/insika/chat_builder.rb +105 -2
  68. data/lib/insika/checkpoint_store.rb +16 -0
  69. data/lib/insika/commands/agent_payload.rb +3 -3
  70. data/lib/insika/commands/backfill_knowledge.rb +145 -0
  71. data/lib/insika/commands/cancel_followup.rb +49 -0
  72. data/lib/insika/commands/delete_artifact.rb +35 -0
  73. data/lib/insika/commands/delete_concept.rb +34 -0
  74. data/lib/insika/commands/delete_mcp.rb +6 -2
  75. data/lib/insika/commands/delete_tenant_data.rb +107 -0
  76. data/lib/insika/commands/export_customer_memory.rb +48 -0
  77. data/lib/insika/commands/forget_customer.rb +117 -0
  78. data/lib/insika/commands/freeze_funnel_baseline.rb +113 -0
  79. data/lib/insika/commands/gate_harvest.rb +138 -0
  80. data/lib/insika/commands/gate_refinement.rb +1 -1
  81. data/lib/insika/commands/judge_shadow_pairs.rb +124 -0
  82. data/lib/insika/commands/memory_forget_fact.rb +20 -4
  83. data/lib/insika/commands/memory_put_fact.rb +23 -4
  84. data/lib/insika/commands/promote_harvest.rb +130 -0
  85. data/lib/insika/commands/record_outcome.rb +46 -0
  86. data/lib/insika/commands/record_shadow_reply.rb +68 -0
  87. data/lib/insika/commands/refresh_mcp_tools.rb +47 -0
  88. data/lib/insika/commands/reject_harvest.rb +38 -0
  89. data/lib/insika/commands/resolve_proposal.rb +108 -0
  90. data/lib/insika/commands/restore_concept.rb +34 -0
  91. data/lib/insika/commands/revoke_contact.rb +49 -0
  92. data/lib/insika/commands/rollback_harvest.rb +86 -0
  93. data/lib/insika/commands/run_distillation.rb +186 -0
  94. data/lib/insika/commands/run_harvest.rb +393 -0
  95. data/lib/insika/commands/seed_demo_data.rb +31 -0
  96. data/lib/insika/commands/send_message.rb +103 -6
  97. data/lib/insika/commands/session_purge.rb +67 -0
  98. data/lib/insika/commands/upsert_mcp.rb +6 -3
  99. data/lib/insika/commands/write_concept.rb +57 -0
  100. data/lib/insika/contact_store.rb +183 -0
  101. data/lib/insika/context/builder.rb +21 -3
  102. data/lib/insika/context/fragment.rb +7 -3
  103. data/lib/insika/context/priority.rb +5 -0
  104. data/lib/insika/context/provider.rb +17 -3
  105. data/lib/insika/context/providers/briefing.rb +96 -0
  106. data/lib/insika/context/providers/knowledge.rb +108 -0
  107. data/lib/insika/context/providers/memory.rb +16 -7
  108. data/lib/insika/context/providers/prompt.rb +57 -23
  109. data/lib/insika/context/providers/skill.rb +2 -0
  110. data/lib/insika/context/providers/tool_search.rb +2 -0
  111. data/lib/insika/context_trace_store.rb +38 -2
  112. data/lib/insika/cron.rb +189 -0
  113. data/lib/insika/demo/agent_attrs.rb +43 -0
  114. data/lib/insika/demo/golden_cases.rb +81 -0
  115. data/lib/insika/demo/seeder.rb +336 -0
  116. data/lib/insika/distill.rb +224 -0
  117. data/lib/insika/distill_engine.rb +169 -0
  118. data/lib/insika/doctor.rb +889 -11
  119. data/lib/insika/dsl/definition.rb +3 -2
  120. data/lib/insika/dsl/runtime.rb +69 -82
  121. data/lib/insika/dsl/server_boot.rb +92 -1
  122. data/lib/insika/dsl/system.rb +10 -2
  123. data/lib/insika/dsl.rb +186 -5
  124. data/lib/insika/edge_limiter.rb +28 -7
  125. data/lib/insika/env_schema.rb +25 -3
  126. data/lib/insika/errors.rb +11 -0
  127. data/lib/insika/evals/assertions.rb +3 -2
  128. data/lib/insika/evals/golden.rb +41 -4
  129. data/lib/insika/evals/judge.rb +47 -2
  130. data/lib/insika/evals/pairwise.rb +29 -6
  131. data/lib/insika/evals/persona.rb +98 -0
  132. data/lib/insika/evals/runner.rb +9 -0
  133. data/lib/insika/evals/simulator.rb +225 -0
  134. data/lib/insika/evals/transport.rb +83 -1
  135. data/lib/insika/event_stream.rb +10 -0
  136. data/lib/insika/evidence.rb +183 -0
  137. data/lib/insika/executor.rb +869 -76
  138. data/lib/insika/followup_engine.rb +207 -0
  139. data/lib/insika/followup_policy.rb +198 -0
  140. data/lib/insika/followup_store.rb +306 -0
  141. data/lib/insika/funnel_declaration.rb +106 -0
  142. data/lib/insika/funnel_fold.rb +179 -0
  143. data/lib/insika/funnel_store.rb +163 -0
  144. data/lib/insika/golden_store.rb +17 -2
  145. data/lib/insika/grounding/matcher.rb +69 -0
  146. data/lib/insika/grounding.rb +44 -0
  147. data/lib/insika/harvest/conversion_gate.rb +159 -0
  148. data/lib/insika/harvest/criterion.rb +98 -0
  149. data/lib/insika/harvest/gate.rb +194 -0
  150. data/lib/insika/harvest/negative_list.rb +199 -0
  151. data/lib/insika/harvest.rb +241 -0
  152. data/lib/insika/harvest_engine.rb +193 -0
  153. data/lib/insika/harvest_store.rb +548 -0
  154. data/lib/insika/knowledge.rb +680 -0
  155. data/lib/insika/knowledge_store.rb +140 -0
  156. data/lib/insika/mcp_client.rb +94 -0
  157. data/lib/insika/mcp_json.rb +74 -0
  158. data/lib/insika/mcp_live_tool.rb +43 -0
  159. data/lib/insika/mcp_store.rb +98 -26
  160. data/lib/insika/mcp_tool_ingestor.rb +30 -8
  161. data/lib/insika/mcp_tool_registry.rb +100 -0
  162. data/lib/insika/media.rb +382 -0
  163. data/lib/insika/memory_audit_store.rb +85 -0
  164. data/lib/insika/memory_store.rb +264 -23
  165. data/lib/insika/message_origin.rb +7 -2
  166. data/lib/insika/middleware.rb +9 -0
  167. data/lib/insika/model_visible.rb +87 -0
  168. data/lib/insika/model_visible_trace_store.rb +66 -0
  169. data/lib/insika/onboarding.rb +22 -1
  170. data/lib/insika/outbox_store.rb +42 -4
  171. data/lib/insika/outcome_store.rb +147 -0
  172. data/lib/insika/overlay_tool_registry.rb +37 -17
  173. data/lib/insika/packaging.rb +163 -0
  174. data/lib/insika/parity/criterion.rb +79 -0
  175. data/lib/insika/parity/verdict.rb +318 -0
  176. data/lib/insika/prefix_fingerprint.rb +58 -0
  177. data/lib/insika/profile_source.rb +29 -1
  178. data/lib/insika/prompt_catalog.rb +10 -0
  179. data/lib/insika/proposal_store.rb +271 -0
  180. data/lib/insika/queue_policy.rb +4 -1
  181. data/lib/insika/refinement/proposer.rb +1 -1
  182. data/lib/insika/reliability.rb +32 -6
  183. data/lib/insika/retention.rb +316 -0
  184. data/lib/insika/router/app.rb +157 -0
  185. data/lib/insika/router/backend_pool.rb +98 -0
  186. data/lib/insika/router/hash_ring.rb +55 -0
  187. data/lib/insika/router/proxy_body.rb +34 -0
  188. data/lib/insika/router/session_key.rb +54 -0
  189. data/lib/insika/router.rb +18 -0
  190. data/lib/insika/routing.rb +101 -0
  191. data/lib/insika/safety/config.rb +44 -4
  192. data/lib/insika/safety/corpus.rb +255 -0
  193. data/lib/insika/safety/detectors.rb +32 -113
  194. data/lib/insika/safety/factory.rb +15 -2
  195. data/lib/insika/safety/grounding_enforcer.rb +59 -0
  196. data/lib/insika/safety/grounding_validator.rb +49 -0
  197. data/lib/insika/safety/input_guardrail.rb +1 -1
  198. data/lib/insika/safety/moderator.rb +1 -1
  199. data/lib/insika/safety/output_filter.rb +9 -5
  200. data/lib/insika/safety/output_validator.rb +11 -5
  201. data/lib/insika/schedule.rb +177 -0
  202. data/lib/insika/schedule_engine.rb +314 -0
  203. data/lib/insika/schedule_store.rb +208 -0
  204. data/lib/insika/schema_guard.rb +35 -0
  205. data/lib/insika/server/app.rb +247 -17
  206. data/lib/insika/server/rack_app.rb +21 -1
  207. data/lib/insika/server/responses.rb +40 -1
  208. data/lib/insika/session_actor.rb +8 -4
  209. data/lib/insika/session_store.rb +65 -2
  210. data/lib/insika/settings_store.rb +10 -0
  211. data/lib/insika/shadow_pair_store.rb +258 -0
  212. data/lib/insika/skill_catalog.rb +16 -0
  213. data/lib/insika/soak/envelope.rb +140 -0
  214. data/lib/insika/soak/report.rb +392 -0
  215. data/lib/insika/soak/runner.rb +554 -0
  216. data/lib/insika/steer_injector.rb +21 -10
  217. data/lib/insika/store.rb +10 -1
  218. data/lib/insika/stores/memory.rb +6 -0
  219. data/lib/insika/stores/sqlite.rb +8 -0
  220. data/lib/insika/studio/app.rb +1437 -54
  221. data/lib/insika/studio/assets/dist/application.css +1 -1
  222. data/lib/insika/studio/assets/dist/application.js +23 -23
  223. data/lib/insika/studio/assets/dist/favicon.svg +6 -0
  224. data/lib/insika/studio/forms.rb +308 -10
  225. data/lib/insika/studio/nav_icons.rb +19 -1
  226. data/lib/insika/studio/views/_agent_tab_cache.erb +25 -0
  227. data/lib/insika/studio/views/_agent_tab_config.erb +514 -0
  228. data/lib/insika/studio/views/_agent_tab_history.erb +24 -0
  229. data/lib/insika/studio/views/_agent_tab_loops.erb +54 -0
  230. data/lib/insika/studio/views/_agent_tab_memory.erb +51 -0
  231. data/lib/insika/studio/views/_agent_tab_outcomes.erb +31 -0
  232. data/lib/insika/studio/views/_agent_tab_prompts.erb +108 -0
  233. data/lib/insika/studio/views/_agent_tab_skills.erb +38 -0
  234. data/lib/insika/studio/views/_agents_master.erb +44 -0
  235. data/lib/insika/studio/views/_message.erb +49 -32
  236. data/lib/insika/studio/views/agent_detail.erb +62 -278
  237. data/lib/insika/studio/views/agents.erb +71 -54
  238. data/lib/insika/studio/views/approvals.erb +4 -1
  239. data/lib/insika/studio/views/artifact.erb +23 -0
  240. data/lib/insika/studio/views/artifacts.erb +59 -0
  241. data/lib/insika/studio/views/chats.erb +4 -1
  242. data/lib/insika/studio/views/customer.erb +94 -0
  243. data/lib/insika/studio/views/customers.erb +32 -0
  244. data/lib/insika/studio/views/evals.erb +6 -3
  245. data/lib/insika/studio/views/facts.erb +133 -0
  246. data/lib/insika/studio/views/followups.erb +125 -0
  247. data/lib/insika/studio/views/funnel.erb +106 -0
  248. data/lib/insika/studio/views/harvest.erb +234 -0
  249. data/lib/insika/studio/views/home.erb +106 -66
  250. data/lib/insika/studio/views/knowledge.erb +123 -0
  251. data/lib/insika/studio/views/layout.erb +15 -11
  252. data/lib/insika/studio/views/mcp.erb +174 -80
  253. data/lib/insika/studio/views/parity.erb +147 -0
  254. data/lib/insika/studio/views/playground.erb +7 -1
  255. data/lib/insika/studio/views/session.erb +233 -124
  256. data/lib/insika/studio/views/settings.erb +41 -1
  257. data/lib/insika/studio/views/skills.erb +1 -2
  258. data/lib/insika/studio/views/system_files.erb +1 -1
  259. data/lib/insika/studio/views/task.erb +13 -0
  260. data/lib/insika/studio/views/tasks.erb +4 -1
  261. data/lib/insika/studio/views/tools.erb +24 -10
  262. data/lib/insika/task_store.rb +21 -1
  263. data/lib/insika/templates/browser-agent/README.md +36 -0
  264. data/lib/insika/templates/browser-agent/agent.rb +49 -0
  265. data/lib/insika/templates/daily-digest/README.md +38 -0
  266. data/lib/insika/templates/daily-digest/agent.rb +77 -0
  267. data/lib/insika/templates/repo-explorer/README.md +36 -0
  268. data/lib/insika/templates/repo-explorer/agent.rb +45 -0
  269. data/lib/insika/templates/research-analyst/README.md +26 -0
  270. data/lib/insika/templates/research-analyst/agent.rb +58 -0
  271. data/lib/insika/templates/review-panel/README.md +20 -0
  272. data/lib/insika/templates/review-panel/agent.rb +50 -0
  273. data/lib/insika/templates/travel-planner/README.md +35 -0
  274. data/lib/insika/templates/travel-planner/agent.rb +87 -0
  275. data/lib/insika/templates.rb +112 -0
  276. data/lib/insika/testing/store_contract.rb +27 -6
  277. data/lib/insika/tick.rb +47 -11
  278. data/lib/insika/timezone.rb +45 -0
  279. data/lib/insika/tool_definition.rb +17 -7
  280. data/lib/insika/tool_envelope.rb +69 -0
  281. data/lib/insika/tool_manifest.rb +5 -1
  282. data/lib/insika/tools/data_defined_tool.rb +10 -0
  283. data/lib/insika/tools/generate_image.rb +89 -0
  284. data/lib/insika/tools/load_knowledge.rb +74 -0
  285. data/lib/insika/tools/run_persona_eval.rb +328 -0
  286. data/lib/insika/tools/save_artifact.rb +95 -0
  287. data/lib/insika/tools/schedule_followup.rb +164 -0
  288. data/lib/insika/tools/tts.rb +47 -0
  289. data/lib/insika/tools/update_briefing.rb +126 -0
  290. data/lib/insika/turn_output.rb +1 -1
  291. data/lib/insika/turn_state.rb +49 -1
  292. data/lib/insika/turn_timing.rb +22 -2
  293. data/lib/insika/version.rb +1 -1
  294. data/lib/insika/vitals.rb +84 -0
  295. data/lib/insika/wiring/graph.rb +472 -13
  296. data/lib/insika/wiring/graph_chat.rb +102 -0
  297. data/lib/insika.rb +149 -3
  298. metadata +181 -5
  299. data/docs/build.md +0 -14
  300. data/docs/understand.md +0 -10
@@ -20,7 +20,10 @@ module Insika
20
20
  capability_registry: nil, tool_catalog: nil, memory_store: nil,
21
21
  tool_trace_store: nil, settings_store: nil, content_filter_factory: nil,
22
22
  delegation_store: nil, channel_delivery: nil, llm: nil,
23
- context_trace_store: nil, reliability: nil)
23
+ context_trace_store: nil, reliability: nil, media: nil, media_output: nil,
24
+ grounding_enforcer: nil, cache_series_store: nil,
25
+ contact_store: nil, followup_store: nil, model_visible_trace_store: nil,
26
+ knowledge_store: nil)
24
27
  @context_builder = context_builder
25
28
  @policy_engine = policy_engine
26
29
  @middleware = middleware
@@ -40,6 +43,10 @@ module Insika
40
43
  # per-turn context breakdown (tokens by category + budget) for the
41
44
  # Studio session card. nil = off (no record, zero overhead — parity).
42
45
  @context_trace_store = context_trace_store
46
+ # the model-visible trace — what the provider received per
47
+ # (task, turn), captured at the chat boundary. nil = off (no record,
48
+ # zero overhead — parity).
49
+ @model_visible_trace_store = model_visible_trace_store
43
50
  # Guardrails output filter: ->(state) { OutputFilter | nil }.
44
51
  # Injected by the Safety::Factory; nil = off (parity — the stream is untouched).
45
52
  # The INPUT guardrail is a Middleware (in the stack, not here); this is the seam
@@ -64,13 +71,38 @@ module Insika
64
71
  # fallback / circuit breaker, all DATA on AgentProfile#reliability.
65
72
  # nil = the plain single ask (parity).
66
73
  @reliability = reliability
74
+ # WS9 media seam (nil = default built on first audio turn): ->(url) { text }.
75
+ @media = media
76
+ # WS9 (saída) media-generation seams: { image: ->(prompt, cfg) [part,
77
+ # usage], tts: ->(text, cfg) [part, usage] }. nil = the defaults (built
78
+ # lazily on first generation — RubyLLM + Net::HTTP behind lazy requires,
79
+ # the core stays gem-free at load). Injected by specs; a production graph
80
+ # that wants a non-RubyLLM backend injects its own lambdas.
81
+ @media_output = media_output
82
+ # the :enforce boundary step, called between stages 6 and 8.
83
+ # Defaults to a REAL enforcer (inert unless the profile's grounding.mode is
84
+ # :enforce — zero behavior change for parity) so an embedder that builds
85
+ # the Executor directly still gets the cut; `nil` stays injectable for
86
+ # stubs that want none.
87
+ @grounding_enforcer = grounding_enforcer || Insika::Safety::GroundingEnforcer.new
88
+ # the per-AGENT cache-hit series. nil = no series recorded
89
+ # (parity — the trace store still gets the per-turn entry when wired).
90
+ @cache_series_store = cache_series_store
67
91
  # LLM config v2: resolves the model at turn start (Chat > Agent >
68
92
  # platform default) + model_policy + fallback chain. settings_store nil =
69
93
  # no platform layer (pre-v2 behavior: the agent's own model is used as-is).
70
94
  @model_resolver = ModelResolver.new(settings_store: settings_store)
95
+ # the follow-up stores the ChatBuilder gates the
96
+ # schedule/cancel_followup tools on (nil = never wired — parity).
97
+ @contact_store = contact_store
98
+ @followup_store = followup_store
71
99
  # the platform layer of the queue policy (nil = per-agent and
72
100
  # defaults only, which is `followup` with no window — today's behavior).
73
101
  @settings_store = settings_store
102
+ # LEARNED concepts, extracted from a turn's transcript after it
103
+ # completes. nil = the loop is off (parity — every write path below is
104
+ # skipped, zero allocations). Gated per-agent by `profile.knowledge`.
105
+ @knowledge_store = knowledge_store
74
106
  # RubyLLM glue (stages 5-7): chat assembly delegated to ChatBuilder. Its
75
107
  # optional deps tool_catalog (Tool Search) and memory_store (cross-session
76
108
  # memory) matter only to it — nil = parity (deferred
@@ -84,7 +116,23 @@ module Insika
84
116
  # the ChatBuilder wires the spawn_subagent system tool (gated by
85
117
  # profile.subagents) and hands it this Executor as the runner. `self` is not
86
118
  # yet fully built here, but the ChatBuilder only STORES it (used per-turn).
87
- subagent_runner: self
119
+ subagent_runner: self,
120
+ # WS9 (saída): the media-generation runner, same shape — the Executor
121
+ # owns the seams + usage accounting, the builder only wires the tools
122
+ # the turn's gates allow.
123
+ media_runner: self,
124
+ # the builder wires the briefing-write system tools gated by
125
+ # @session_store + profile.briefing_fields. nil = never wired (parity).
126
+ session_store: session_store,
127
+ # the builder wires the schedule/cancel_followup system
128
+ # tools gated by a parsed policy AND both stores present. nil = never
129
+ # wired (parity).
130
+ contact_store: contact_store,
131
+ followup_store: followup_store,
132
+ # the builder wires the load_knowledge system tool gated by
133
+ # @knowledge_store + profile.knowledge["retrieve"]. nil = never
134
+ # wired (parity — the write path above stays unaffected either way).
135
+ knowledge_store: knowledge_store
88
136
  )
89
137
  # Stage-3-tail tool assembly (capability resolution, instantiation,
90
138
  # injection, dedup join, ToolEnvelope wrap) — extracted collaborator.
@@ -124,6 +172,32 @@ module Insika
124
172
  # the turn supervisor in serving mode (like the tick); nil = no alerts.
125
173
  attr_accessor :alert_dispatcher
126
174
 
175
+ # the distillation engine — the tick-duty that finds idle
176
+ # customer sessions and distills them on its own worker fiber (a child of
177
+ # the turn supervisor, like the tick). nil = distillation off (parity —
178
+ # nothing scans, nothing distills).
179
+ attr_accessor :distill_engine
180
+
181
+ # the harvest engine — the tick-duty that finds idle,
182
+ # unmined sessions and mines them on its own worker fiber (a child of the
183
+ # turn supervisor, like the tick). nil = harvest off (parity — nothing
184
+ # scans, nothing mines).
185
+ attr_accessor :harvest_engine
186
+
187
+ # Forces the long-lived turn supervisor (and its tick/alert/distill/harvest
188
+ # children — see #turn_parent) to start NOW instead of lazily on the first
189
+ # served turn. Without this, a deployment whose only agents are scheduled
190
+ # (no live chat) never fires the tick until some unrelated turn happens to
191
+ # land first — observed live as 16+ minutes of silence after a clean boot.
192
+ # Call once, right after `supervised = true`, from the composition root
193
+ # (Server::Boot / config.ru / DSL::ServerBoot). A no-op outside a live
194
+ # reactor (nothing to bind the supervisor to yet) — the lazy path in
195
+ # #turn_parent still covers that case, e.g. specs that never enter Async.
196
+ def start_supervisor!
197
+ turn_parent if @supervised && Async::Task.current?
198
+ nil
199
+ end
200
+
127
201
  # closes the TURN intake for shutdown. Armed by Insika::Shutdown
128
202
  # when the process is asked to stop: from here on a new top-level turn is left
129
203
  # `:queued` (durable — the next boot's recovery replays it) instead of
@@ -225,12 +299,17 @@ module Insika
225
299
 
226
300
  # Stage 1 (async part): creates the actor, registers it and fires the fiber.
227
301
  # Called by the turn handlers (SendMessage/ResumeTask/TriggerWorkflow).
228
- def spawn(task, profile:, resume_from: nil)
302
+ #
303
+ # `timing` is the channel clock a channel turn allocated at 202
304
+ # acceptance, already carrying `:inbound`; nil means the pipeline allocates its
305
+ # own (resume, engine-initiated, non-channel). `mark` is first-write-wins, so
306
+ # re-marking `:inbound` in the pipeline is a no-op on a threaded clock.
307
+ def spawn(task, profile:, resume_from: nil, timing: nil)
229
308
  raise Insika::ValidationError, "task already running: #{task.id}" if running?(task.id)
230
309
 
231
310
  actor = TaskActor.new(task_id: task.id, parent: turn_parent)
232
311
  @running[task.id] = actor
233
- actor.run { execute(task, profile: profile, resume_from: resume_from, actor: actor) }
312
+ actor.run { execute(task, profile: profile, resume_from: resume_from, actor: actor, timing: timing) }
234
313
  task.id
235
314
  end
236
315
 
@@ -238,7 +317,7 @@ module Insika
238
317
  # session_id is SERIALIZED in that session's SessionActor queue (one at a
239
318
  # time); without a session_id (one-shot/history) it goes straight to spawn
240
319
  # (standalone).
241
- def spawn_in_session(task, profile:, resume_from: nil)
320
+ def spawn_in_session(task, profile:, resume_from: nil, timing: nil)
242
321
  # the intake is closed. The task is already durable (queued);
243
322
  # answering with its id and spawning NOTHING is what "stops accepting new
244
323
  # turns" means — the next boot's recovery replays it. Subagent turns are NOT
@@ -252,11 +331,12 @@ module Insika
252
331
  # owner awaits the turn). One-shot/history (no session_id)
253
332
  # never serialize.
254
333
  unless @supervised && task.session_id
255
- return spawn(task, profile: profile, resume_from: resume_from)
334
+ return spawn(task, profile: profile, resume_from: resume_from, timing: timing)
256
335
  end
257
336
 
258
337
  session_actor(task.session_id).enqueue(task, profile: profile, resume_from: resume_from,
259
- policy: queue_policy(profile, task.session_id))
338
+ policy: queue_policy(profile, task.session_id),
339
+ timing: timing)
260
340
  end
261
341
 
262
342
  # the `collect` door, asked BEFORE a task is created.
@@ -296,7 +376,8 @@ module Insika
296
376
  return nil unless session_actor&.alive?
297
377
 
298
378
  # No turn running (or one still at the door): there is nothing to steer INTO.
299
- # A turn at the door belongs to `collect`, which is a different mode.
379
+ # A turn at the door is the collect door's other window — a steer agent with a
380
+ # debounce merges there instead , so no message waits on either.
300
381
  task = session_actor.current_task
301
382
  return nil if task.nil?
302
383
  # A workflow turn orchestrates RubyLLM itself and has no Insika chat to append to
@@ -404,13 +485,13 @@ module Insika
404
485
  # its completion before returning — that is what serializes the session. A
405
486
  # turn error is already mapped to a terminal state inside its own fiber (single
406
487
  # capture); here we only ensure the session loop does not die.
407
- def run_serial(task, profile:, resume_from: nil)
488
+ def run_serial(task, profile:, resume_from: nil, timing: nil)
408
489
  # a drain that started with turns already queued behind this
409
490
  # session's current one must not keep feeding the loop — without this gate
410
491
  # the drain would only converge when the whole backlog ran out.
411
492
  return defer_turn(task) if @draining
412
493
 
413
- spawn(task, profile: profile, resume_from: resume_from)
494
+ spawn(task, profile: profile, resume_from: resume_from, timing: timing)
414
495
  @running[task.id]&.wait
415
496
  rescue Async::Stop
416
497
  raise # shutdown: propagate (ends the session loop)
@@ -505,7 +586,7 @@ module Insika
505
586
  end
506
587
 
507
588
  # Stages 2..9. Runs INSIDE the task's fiber.
508
- def execute(task, profile:, actor:, resume_from: nil)
589
+ def execute(task, profile:, actor:, resume_from: nil, timing: nil)
509
590
  # Resume of a crash orphan: the interrupted attempt's Execution was left OPEN
510
591
  # (the fiber died). The TaskStore forbids opening a second one while one is
511
592
  # open -> close the orphan as :interrupted before opening the N+1 (a new
@@ -519,7 +600,7 @@ module Insika
519
600
  emit(:task_started, started_data(task, profile), task: task)
520
601
 
521
602
  actor.drain!
522
- run_pipeline(task, profile, actor, resume_from)
603
+ run_pipeline(task, profile, actor, resume_from, timing)
523
604
  # SINGLE capture at the top of the fiber: a single place maps
524
605
  # error -> terminal state -> events. Stages do no rescue of their own
525
606
  # (except tool, RubyLLM semantics). The fiber NEVER re-raises.
@@ -541,6 +622,14 @@ module Insika
541
622
  # reads circuit_open + retry_after (cooldown remaining). Worth its own
542
623
  # stage: an open breaker is a reliability decision, not an error bug.
543
624
  fail_task(task, e, stage: :reliability)
625
+ rescue Insika::RoutingError => e
626
+ # WS4: a route's delegate is missing or its turn failed — an operator
627
+ # config error, staged so the envelope names routing, never :unknown.
628
+ fail_task(task, e, stage: :routing)
629
+ rescue Insika::MediaError => e
630
+ # WS9: a voice message that could not be fetched/transcribed (or a media
631
+ # URL the egress guard refused) — heard-loud, never a silent drop.
632
+ fail_task(task, e, stage: :media)
544
633
  rescue Insika::WorkflowSchemaError => e
545
634
  # a workflow OUTPUT that violates its output_schema. Distinct
546
635
  # stage so a contract breach is not conflated with an :unknown failure. (INPUT
@@ -574,6 +663,36 @@ module Insika
574
663
  release_steered(task, profile, actor)
575
664
  end
576
665
 
666
+ # WS9 (saída), the RUNNER side the generate_image/tts system tools call
667
+ # (public like run_subagent — a tool reaches back into the Executor):
668
+ #
669
+ # -> [part, usage]: resolve the seam (injected or the lazy default) and
670
+ # run it. The default seams are built on FIRST use, when the turn already
671
+ # has a chat (ruby_llm loaded), so the load-guard holds.
672
+ def generate_media_output(kind, content, config)
673
+ seam = @media_output&.fetch(kind, nil) || Insika::Media::Output.defaults(context: @llm)[kind]
674
+ raise Insika::MediaError, "no #{kind} output seam" unless seam
675
+
676
+ seam.call(content, config)
677
+ end
678
+
679
+ # Accounts a generated part in the turn's usage: the provider's token
680
+ # counts (images — the merge keeps what the classifier already banked),
681
+ # plus an honest `media` call counter per part (the speech API reports no
682
+ # tokens; the part itself carries the model for consumer-side pricing).
683
+ def account_media_usage(state, part, usage)
684
+ usage ||= {}
685
+ tokens = {}
686
+ tokens[:input_tokens] = usage[:input_tokens].to_i if usage[:input_tokens]
687
+ tokens[:output_tokens] = usage[:output_tokens].to_i if usage[:output_tokens]
688
+ tokens[:total_tokens] = tokens[:input_tokens].to_i + tokens[:output_tokens].to_i if tokens.any?
689
+ unless tokens.empty?
690
+ tokens[:model] = part["model"] if part["model"]
691
+ state.usage = merge_usage(tokens, state.usage)
692
+ end
693
+ state.usage = (state.usage || {}).merge(media: state.usage&.fetch(:media, 0).to_i + 1)
694
+ end
695
+
577
696
  private
578
697
 
579
698
  # resolves session vars > profile.limits > settings["queue"] >
@@ -646,6 +765,12 @@ module Insika
646
765
  # the alert dispatcher (WS6) lives on the same supervisor: its consumer
647
766
  # answers every alert event for as long as the process serves.
648
767
  @alert_dispatcher&.start(parent: @supervisor)
768
+ # the distillation engine lives on the same supervisor —
769
+ # its worker fiber re-scans idle sessions off the turn path.
770
+ @distill_engine&.start(parent: @supervisor)
771
+ # the harvest engine lives on the same supervisor — its
772
+ # worker fiber re-scans idle sessions off the turn path.
773
+ @harvest_engine&.start(parent: @supervisor)
649
774
  @supervisor
650
775
  end
651
776
 
@@ -710,8 +835,23 @@ module Insika
710
835
  # Stages 2-9, with mailbox drain only at the boundaries and the
711
836
  # turn-timeout wrapping everything via Async::Task#with_timeout — NEVER
712
837
  # stdlib Timeout.timeout.
713
- def run_pipeline(task, profile, actor, resume_from)
714
- timing = TurnTiming.new if TurnTiming.enabled?
838
+ def run_pipeline(task, profile, actor, resume_from, timing = nil)
839
+ # a CHANNEL turn always allocates the clock — first_balloon_ms
840
+ # (inbound -> first outbox flush) is H-latência and must not depend on
841
+ # INSIKA_TURN_TIMING. When the flag is off the clock measures ONLY that
842
+ # window (`breakdown: false`); the full prep/ttft/gen/total stays opt-in.
843
+ #
844
+ # A channel turn may already carry its clock: `SendMessage` stamped
845
+ # `:inbound` at 202 acceptance (before the debounce window and the
846
+ # SessionActor FIFO), so first_balloon_ms includes the wait the customer
847
+ # actually feels. A turn that reached here without one (boot resume,
848
+ # engine-initiated) falls back to allocating and stamps now — `mark` is
849
+ # first-write-wins, so a threaded clock is never re-stamped.
850
+ channel_turn = !channel_transport(task).nil?
851
+ timing ||= if TurnTiming.enabled? || channel_turn
852
+ TurnTiming.new(breakdown: TurnTiming.enabled?)
853
+ end
854
+ timing&.mark(:inbound) if channel_turn
715
855
  timing&.mark(:prep_start)
716
856
  state = build_turn_state(task, profile, resume_from)
717
857
  turn_timeout = turn_timeout_for(profile)
@@ -741,7 +881,7 @@ module Insika
741
881
  # · halt_reason set -> halt-as-FAILURE (the pre-existing contract).
742
882
  # · neither -> contract violation (short-circuit with no signal).
743
883
  if !terminal_ran && state.halt_response
744
- complete_with_halt(task, profile, state)
884
+ complete_with_halt(task, profile, state, timing)
745
885
  elsif state.halt_reason
746
886
  raise Insika::Error, "turn halted: #{state.halt_reason}"
747
887
  elsif !terminal_ran
@@ -749,7 +889,15 @@ module Insika
749
889
  end
750
890
 
751
891
  state # subject of the :task pair (after_task receives it; the caller discards)
752
- end.tap { |st| emit_guardrail_flags(task, st) }
892
+ end.tap do |st|
893
+ # flush the evidence ledger HERE — after the after_task
894
+ # hooks ran, because the :flag validator increments the ungrounded
895
+ # counter in after_task. The envelope's ids (stage 6) ride the same
896
+ # flush. AFTER persist_turn (already done), so a flush failure can
897
+ # never un-commit the turn (the ledger swallows store errors).
898
+ st.evidence_ledger&.flush!
899
+ emit_guardrail_flags(task, st)
900
+ end
753
901
  end
754
902
  rescue Async::TimeoutError
755
903
  raise Insika::TimeoutError.new("turn exceeded #{turn_timeout}s", stage: :turn)
@@ -762,6 +910,7 @@ module Insika
762
910
  state = TurnState.new(task: task, profile: profile, turn: turn,
763
911
  message: extract_message(task))
764
912
  state.tenant = memory_tenant(task) # WRITE-path memory scope (`remember`); =chat
913
+ stamp_customer_session(task, profile)
765
914
  state.turn_context = build_turn_context(task, profile, state) # data-tools' ctx.*
766
915
  state.resumed = !resume_from.nil? # EdgeLimiter: an admitted turn is never re-counted
767
916
  # resolved for the RUN, not per message, so what the turn accepts cannot
@@ -769,6 +918,12 @@ module Insika
769
918
  # SESSION turn: steering needs a session to arrive through, and resolving here for a
770
919
  # one-shot would make an unrelated turn fail on a queue key it can never use.
771
920
  state.queue_policy = task.session_id ? queue_policy(profile, task.session_id) : nil
921
+ # the session evidence ledger, built per turn. A nil
922
+ # session_id (one-shot) is fine — the ledger just never flushes (grounding
923
+ # on a one-shot is per-turn by definition). The envelope appends ids, the
924
+ # validator/enforcer read the union, stage 8 flushes.
925
+ state.evidence_ledger = Insika::EvidenceLedger.new(store: @session_store,
926
+ session_id: task.session_id)
772
927
  state
773
928
  end
774
929
 
@@ -786,6 +941,17 @@ module Insika
786
941
  # -> tool assembly, each followed by a mailbox drain at the boundary. Mutates
787
942
  # `state` in place (the TurnState possibly rewritten by before_task).
788
943
  def prepare_turn(task, profile, state, actor, resume_from)
944
+ # WS9 (saída): the CHANNEL's declared output media kinds — read on EVERY
945
+ # run (including resumes — it is plain config) so the ChatBuilder gate
946
+ # can re-wire the media tools; the transcription below stays guarded.
947
+ state.channel_capabilities = Insika::Media.channel_capabilities(
948
+ rebuild_command(task).payload["channel"]
949
+ )
950
+ # WS9: content parts -> turn. Runs BEFORE stage 2's context build so the
951
+ # transcribed voice text feeds the prompt; a resumed turn was already
952
+ # transcribed (never re-pay the STT call).
953
+ run_media_stage(task, state) unless state.resumed
954
+
789
955
  # stage 2: Context. The :prompt hook pair is wrapped INSIDE the
790
956
  # ContextBuilder#call — do NOT wrap here (a double-wrap would fire the hooks
791
957
  # twice). Hooks is the SAME instance injected into the Builder and here.
@@ -824,8 +990,12 @@ module Insika
824
990
 
825
991
  # one entry per turn in the ContextTraceStore — tokens per
826
992
  # category (the demodulized provider id), the tools-schema estimate and the
827
- # budget verdict. Counts and ids ONLY, never content. nil store = off; the
828
- # store itself rescues everything (the trace never breaks the turn).
993
+ # budget verdict. the entry also carries the prefix
994
+ # fingerprints + the invalidation_reason vs the previous turn, and the
995
+ # categories gain their cache layer. Counts and ids ONLY, never content.
996
+ # nil store = off; the store itself rescues everything (the trace never
997
+ # breaks the turn). -> the sanitized entry (parked on TurnState for the
998
+ # stage-8 cache stamp).
829
999
  def record_context_trace(task, state)
830
1000
  return unless @context_trace_store && task.session_id
831
1001
 
@@ -839,6 +1009,10 @@ module Insika
839
1009
  c[:tokens] += f.tokens || 0
840
1010
  c[:fragments] += 1
841
1011
  c[:pinned] += (f.tokens || 0) if f.pinned
1012
+ # the category's cache layer (identity | :volatile —
1013
+ # stamped by the Builder at production, C3).
1014
+ layer = f.layer || :volatile
1015
+ c[:layer] ||= layer
842
1016
  # WHICH skills/tools the fragment carried and WHY — ids only, still
843
1017
  # content-free. Without this the trace proves a turn injected N tokens of
844
1018
  # skill but not which ones, and deterministic activation is unauditable
@@ -846,14 +1020,88 @@ module Insika
846
1020
  labels = Array(f.labels)
847
1021
  (c[:labels] ||= []).concat(labels) unless labels.empty?
848
1022
  end
849
- @context_trace_store.record(
850
- session_id: task.session_id,
851
- entry: { task_id: task.id, turn: state.turn, at: Time.now.utc.iso8601,
852
- cap: package.budget[:cap], used: package.budget[:used],
853
- evicted: package.budget[:evicted], categories: categories,
854
- tools: { count: state.allowed_tools.size,
855
- tokens: estimate_tools_tokens(state.allowed_tools) } }
856
- )
1023
+ # the prefix chain over the SYSTEM-placement fragments in
1024
+ # canonical (identity-first) render order + the tool-schema serialization.
1025
+ # The reason is computed against the PREVIOUS trace entry (D3): the first
1026
+ # category, in current chain order, whose bytes changed (or vanished).
1027
+ previous = previous_trace_entry(task, state.turn)
1028
+ fingerprints = Insika::PrefixFingerprint.compute(
1029
+ Array(package.fragments).select { |f| f.placement == :system },
1030
+ tool_serial: serialize_tools(state.allowed_tools))
1031
+ reason = Insika::PrefixFingerprint.invalidation_reason(
1032
+ fingerprints, previous && previous["fingerprints"])
1033
+
1034
+ entry = { task_id: task.id, turn: state.turn, at: Time.now.utc.iso8601,
1035
+ cap: package.budget[:cap], used: package.budget[:used],
1036
+ evicted: package.budget[:evicted], categories: categories,
1037
+ tools: { count: state.allowed_tools.size,
1038
+ tokens: estimate_tools_tokens(state.allowed_tools) },
1039
+ fingerprints: fingerprints,
1040
+ cache: { invalidation_reason: reason } }
1041
+ # Park the SANITIZED entry (string keys) — the stage-8 stamp merges into
1042
+ # it and re-records the same key; a raw entry would add a SECOND "cache"
1043
+ # key that sanitize would then ignore (the symbol one wins).
1044
+ state.context_trace_entry = @context_trace_store.record(session_id: task.session_id,
1045
+ entry: entry)
1046
+ end
1047
+
1048
+ # the previous turn's trace entry — the session list minus
1049
+ # THIS (task_id, turn) (an approval-resumed turn re-records over its own key
1050
+ # — never compare to self). `turn` is the turn being recorded, passed
1051
+ # explicitly. -> Hash | nil (first turn of the session).
1052
+ def previous_trace_entry(task, turn)
1053
+ @context_trace_store.for_session(task.session_id)
1054
+ .reject { |x| x["task_id"] == task.id && x["turn"] == turn }
1055
+ .last
1056
+ end
1057
+
1058
+ # the tool-schema yardstick — the SAME serialization the token
1059
+ # estimate uses (estimate_tools_tokens), so the fingerprint and the estimate
1060
+ # never disagree. The digest covers name + description + parameters.inspect,
1061
+ # approximating RubyLLM's rendering (honest in the doc: the reason's job is
1062
+ # the CONTEXT categories; the tool hash is a guard rail).
1063
+ def serialize_tools(tools)
1064
+ tools.map { |t| "#{t.name} #{t.description} #{t.parameters.inspect}" }.join(" ")
1065
+ end
1066
+
1067
+ # the stage-8 stamp — the usage (cached_tokens,
1068
+ # input_tokens) only exists after the provider answered, so a SECOND
1069
+ # UPSERT with the SAME (task_id, turn) merges the cache fields into the
1070
+ # entry parked at prepare_turn (the start-of-turn write stays: a turn that
1071
+ # dies mid-flight still shows its context on the Studio screen). The same
1072
+ # numbers append one entry to the agent's CacheSeriesStore (C6).
1073
+ def stamp_cache_hit(task, state)
1074
+ usage = state.usage || {}
1075
+ input = usage[:input_tokens].to_i
1076
+ cached = usage[:cached_tokens].to_i
1077
+ creation = usage[:cache_creation_tokens].to_i
1078
+ # A4: the billed prefix is input + cached + cache_creation. RubyLLM's
1079
+ # input_tokens is the FRESH input only — cached_tokens is disjoint, not a
1080
+ # subset — so dividing by input alone yields absurd numbers (22000/500 =
1081
+ # 4400%) and renders a full hit as "—" (fresh=0). The denominator is the
1082
+ # whole billed prompt; hit_pct is then always in [0,100].
1083
+ billed = input + cached + creation
1084
+ hit = billed.positive? ? ((cached * 100.0) / billed).round : nil
1085
+ reason = state.context_trace_entry&.dig("cache", "invalidation_reason")
1086
+
1087
+ # The trace merge needs the entry parked at prepare_turn (the UPSERT
1088
+ # replaces the same (task_id, turn)); a failed trace write leaves it nil
1089
+ # and the cache line simply never lands — never a turn failure.
1090
+ if @context_trace_store && state.context_trace_entry && task.session_id
1091
+ @context_trace_store.record(
1092
+ session_id: task.session_id,
1093
+ entry: state.context_trace_entry.merge("cache" => {
1094
+ "hit_pct" => hit, "cached_tokens" => cached, "prompt_tokens" => billed,
1095
+ "invalidation_reason" => reason }))
1096
+ end
1097
+
1098
+ # The per-agent series is INDEPENDENT of the trace store: a deployment
1099
+ # that wires the series without the trace (or whose trace write failed)
1100
+ # still records its cache-hit numbers — reason is simply nil then.
1101
+ @cache_series_store&.record(agent: state.profile.id, entry: {
1102
+ at: Time.now.utc.iso8601, turn: state.turn,
1103
+ hit_pct: hit, cached_tokens: cached, prompt_tokens: billed,
1104
+ invalidation_reason: reason })
857
1105
  end
858
1106
 
859
1107
  # Skill bodies that reached the prompt WITHOUT a tool call (`triggers:` or
@@ -904,7 +1152,13 @@ module Insika
904
1152
  # stage 5: assemble chat + check mailbox (send_message only; a workflow does
905
1153
  # not use the Insika chat — it orchestrates RubyLLM internally).
906
1154
  drain_and_maybe_suspend(task, actor)
907
- unless workflow_turn?(task)
1155
+ # WS4: intent routing, data-gated. Runs BEFORE the agent chat is assembled:
1156
+ # a route that delegates or ends :stuck completes the turn with no ask at
1157
+ # all (routed = true); a plain route is a label + event and the turn
1158
+ # proceeds. Skipped for workflows (no chat to route into) and resumed turns
1159
+ # (already admitted; re-classifying would re-pay the extra call).
1160
+ routed = !workflow_turn?(task) && !st.resumed ? attempt_route(task, profile, st) : false
1161
+ unless workflow_turn?(task) || routed
908
1162
  st.chat = create_chat(profile, st)
909
1163
  @chat_builder.assemble(st.chat, st, emit: ->(type, data) { emit(type, data, task: task) })
910
1164
  # R1: baseline = seeded-history size, before `ask` appends the turn.
@@ -915,21 +1169,57 @@ module Insika
915
1169
  st.output_filter = @content_filter_factory&.call(st)
916
1170
 
917
1171
  # stage 6: the turn's single agent interaction (send_message -> chat.ask;
918
- # trigger_workflow -> workflow.call). Returns the turn's final content.
919
- content = run_agent_stage(task, st, timing)
1172
+ # trigger_workflow -> workflow.call). A routed turn's content IS its route
1173
+ # action's answer (a delegate's reply or the stuck lead-in).
1174
+ content = routed ? st.response_content : run_agent_stage(task, st, timing)
920
1175
  st.response_content = content # after_task OutputValidator inspects this
921
1176
 
1177
+ # the model-visible record — what the provider received this
1178
+ # turn, captured at the boundary BEFORE stage 8 persists the checkpoint
1179
+ # (turn n's provider-visible stream == checkpoint(turn n+1).messages).
1180
+ # Skipped for workflows (they orchestrate RubyLLM inside the workflow
1181
+ # body — the engine cannot see those calls, stated in the conformance
1182
+ # scope) and absent-store runs (parity).
1183
+ record_model_visible(task, st) if !workflow_turn?(task) && st.chat
1184
+
1185
+ # the :enforce boundary — a CUT of the final content BEFORE
1186
+ # persistence/delivery (after_task fires too late to change what is
1187
+ # persisted). The cut text is what persists, delivers and terminates.
1188
+ if @grounding_enforcer
1189
+ content, st = @grounding_enforcer.call(task, st, content)
1190
+ st.response_content = content
1191
+ end
1192
+
922
1193
  # stage 8: Persistence (fixed order checkpoint->session->task). pure drain!
923
1194
  # (NEVER suspends at stage 8 — forbidden window): a :pause here arms the flag
924
1195
  # but is not honored (last stage); :cancel here still raises.
925
1196
  actor.drain!
926
- persist_turn(task, profile, st, content)
1197
+ persist_turn(task, profile, st, content, timing: timing)
1198
+
1199
+ # the cache-hit stamp — the usage exists only now. The
1200
+ # stamped entry is durable before anything is delivered (same slot as the
1201
+ # enforcer). Best-effort by construction (both stores rescue).
1202
+ stamp_cache_hit(task, st)
927
1203
 
928
1204
  # stage 9: Response. usage (tokens) captured at stage 6 travels in the
929
1205
  # terminal event -> /v1/responses usage + Telemetry (OTEL).
930
1206
  timing&.mark(:done)
931
1207
  data = { task_id: task.id, content: content, usage: st.usage }
1208
+ # WS4: the intent route rides the terminal additively (like outcome) — a
1209
+ # consumer aggregating by route does not need the stream.
1210
+ data[:route] = st.route.to_s if st.route
1211
+ # WS9: a turn whose message came from a VOICE note is marked — the
1212
+ # consumer's signal the person spoke (text was transcribed).
1213
+ data[:source] = :voice if st.message_source == :voice
1214
+ # WS9 (saída): media the agent GENERATED this turn (image/audio clips).
1215
+ # Additive sibling — the answer text stays text on purpose; the channel
1216
+ # consumes the parts next to it. Absent when nothing was generated.
1217
+ data[:output_parts] = st.output_parts if st.output_parts && !st.output_parts.empty?
932
1218
  data[:timing] = timing.to_h if timing # opt-in TTFB breakdown (INSIKA_TURN_TIMING)
1219
+ # best-effort persist of the same timing onto the task record —
1220
+ # the Studio task page reads it from there. A failure here must not re-fail
1221
+ # the turn (the task is already committed and the event already carries it).
1222
+ persist_turn_timing(task, timing)
933
1223
  # WS5: the agent declared it cannot proceed (signal_stuck). The turn still
934
1224
  # COMPLETES (its final message was published) — but the consumer must be able
935
1225
  # to act on that, so the contract carries it twice: a dedicated :turn_stuck
@@ -946,6 +1236,23 @@ module Insika
946
1236
  command_type(task).to_s == "trigger_workflow"
947
1237
  end
948
1238
 
1239
+ # the model-visible record of ONE ask — the chat at the
1240
+ # provider boundary (instructions + tool schemas + the message stream),
1241
+ # persisted under the checkpoint's turn number (turn n's stream ==
1242
+ # checkpoint(turn n+1).messages). Best-effort: the store rescues
1243
+ # everything, and the absent-store path is parity. `chat` defaults to the
1244
+ # turn's own chat; the routing classifier passes its own.
1245
+ def record_model_visible(task, st, chat = nil, part: "turn")
1246
+ return unless @model_visible_trace_store
1247
+
1248
+ c = chat || st.chat
1249
+ return unless c
1250
+
1251
+ @model_visible_trace_store.record(
1252
+ task_id: task.id, turn: st.turn + 1, part: part,
1253
+ payload: Insika::ModelVisible.capture(c))
1254
+ end
1255
+
949
1256
  # Stage 6: the single agent interaction. Returns the turn's final content.
950
1257
  def run_agent_stage(task, state, timing = nil)
951
1258
  if workflow_turn?(task)
@@ -978,6 +1285,13 @@ module Insika
978
1285
  response = @hooks.around(:agent, state) do |s|
979
1286
  result = @reliability ? run_reliable_ask(task, s, filter, timing)
980
1287
  : run_single_ask(task, s, filter, timing)
1288
+ # A steered message that no tool batch absorbed (a text-only turn: the
1289
+ # injector only drains at a tool boundary) is still in the mailbox when
1290
+ # RubyLLM's loop returns. ONE extra round, in the SAME turn, answers the
1291
+ # burst coherently -- without it the release would spawn a follow-up
1292
+ # whose answer has no delivery channel under request/response.
1293
+ # Read AFTER it: the extra round replaces both the response and `asked`.
1294
+ absorb_steer_overflow(task, s, timing, result)
981
1295
  output = result[:output]
982
1296
  asked = result[:asked]
983
1297
  result[:response]
@@ -985,7 +1299,13 @@ module Insika
985
1299
  # release the redactor's retained tail (a value that never completed into a
986
1300
  # match is emitted redacted-if-needed, not lost) before reading anything back.
987
1301
  output.flush
988
- state.usage = with_model_source(usage_of(response), state.model_selection) unless halted?(response)
1302
+ # MERGED, not overwritten: a WS4 routing call already banked its tokens in
1303
+ # state.usage before the ask — the classifier's cost must survive the ask
1304
+ # (it feeds the EdgeLimiter's ceiling/budget and the terminal usage).
1305
+ unless halted?(response)
1306
+ state.usage = merge_usage(with_model_source(usage_of(response), state.model_selection),
1307
+ state.usage)
1308
+ end
989
1309
 
990
1310
  # BOUNDARY BEFORE THE ANSWER GOES OUT. A cancel that arrived while the provider
991
1311
  # was working used to be observed at stage 8 — AFTER `:content` had already been
@@ -1075,6 +1395,231 @@ module Insika
1075
1395
 
1076
1396
  def ref_of(selection) = model_ref(selection)
1077
1397
 
1398
+ # --- WS9 media ------------------------------------------------------
1399
+ #
1400
+ # Content parts on the command -> a turn: audio parts are transcribed (the
1401
+ # text enters the message marked `source: :voice` — the consumer's signal
1402
+ # the person SPOKE), image and document parts become the ask's attachments
1403
+ # (the model sees them; the provider bills them — usage flows) and the
1404
+ # first URL of each kind is deposited as `ctx.image_url` / `ctx.document_url`
1405
+ # for data/HTTP tools. The engine transports media, never meaning: no
1406
+ # speech/vision logic beyond the call itself.
1407
+ def run_media_stage(task, state)
1408
+ # a consumer that pre-transcribed voice text labels it `source: voice`;
1409
+ # the marker rides the turn even when there are no audio PARTS left.
1410
+ state.message_source = :voice if rebuild_command(task).payload["source"].to_s == "voice"
1411
+
1412
+ parts = Insika::Media.parts(rebuild_command(task).payload["parts"])
1413
+ return if parts.empty?
1414
+
1415
+ voice = Insika::Media.audio_parts(parts)
1416
+ if voice.any?
1417
+ text = voice.map { |p| media_transcribe(p.url, state) }.reject(&:empty?).join(" ")
1418
+ state.message = [state.message.to_s, text].reject(&:empty?).join("\n")
1419
+ state.message_source = :voice
1420
+ end
1421
+
1422
+ images = Insika::Media.image_parts(parts)
1423
+ if images.any?
1424
+ state.image_attachments = images.map { |p| media_attachment(p.url) }
1425
+ state.media_attachments = state.image_attachments
1426
+ # First image URL for data tools (`{{ctx.image_url}}`) — photo analysis
1427
+ # outside the prompt. The model still sees the attachment; the tool
1428
+ # gets the original URL (its own egress applies when it fetches).
1429
+ state.turn_context = (state.turn_context || {}).merge(image_url: images.first.url)
1430
+ end
1431
+
1432
+ # Documents (a prescription, a recipe, an invoice) ride the SAME
1433
+ # attachments array as images — RubyLLM's `ask(with:)` takes both, and
1434
+ # the attachment content-sniffs PDF magic bytes when the URL has no
1435
+ # extension — capped separately (MAX_DOCUMENT_BYTES) since a document
1436
+ # is not a photo.
1437
+ documents = Insika::Media.document_parts(parts)
1438
+ if documents.any?
1439
+ doc_attachments = documents.map { |p| media_attachment(p.url, max_bytes: Insika::Media::MAX_DOCUMENT_BYTES) }
1440
+ state.media_attachments = Array(state.media_attachments) + doc_attachments
1441
+ state.turn_context = (state.turn_context || {}).merge(document_url: documents.first.url)
1442
+ end
1443
+
1444
+ # A media-only turn (a voice note with no caption) is legitimate — the
1445
+ # surfaces admit it — but it must leave this stage with something to ask
1446
+ # about. Empty text AND no attachment means the parts carried nothing the
1447
+ # engine could use (a transcription that came back blank): fail loudly at
1448
+ # :media rather than ask the provider about nothing.
1449
+ return unless state.message.to_s.strip.empty? && state.media_attachments.nil?
1450
+
1451
+ raise Insika::MediaError, "the message parts produced no text and no attachment"
1452
+ end
1453
+
1454
+ # The STT seam: the injected transcriber (specs), else the default
1455
+ # (fetch + RubyLLM::Transcription — lazy require). A failed transcription
1456
+ # fails the turn loudly (MediaError -> :media): a voice message that was
1457
+ # not heard must not become a hallucinated one. The default is rebuilt
1458
+ # PER CALL (never memoized) because its vocabulary `prompt:` is resolved
1459
+ # from THIS turn's profile — a memoized seam would freeze the first
1460
+ # agent's prompt (or its absence) for every agent sharing the executor.
1461
+ def media_transcribe(url, state)
1462
+ transcriber = @media || Insika::Media.default_transcriber(
1463
+ stt_model: Insika::EnvSchema.read("INSIKA_STT_MODEL"),
1464
+ stt_language: Insika::EnvSchema.read("INSIKA_STT_LANGUAGE"),
1465
+ stt_prompt: resolved_stt_prompt(state.profile)
1466
+ )
1467
+ transcriber.call(url)
1468
+ end
1469
+
1470
+ # Resolution order: the agent profile's own vocabulary hint
1471
+ # (it knows its catalog) beats the deployment-wide default, which beats
1472
+ # nothing (no prompt: kwarg at all).
1473
+ def resolved_stt_prompt(profile)
1474
+ Insika::Coercion.presence(profile&.stt_prompt) || Insika::EnvSchema.read("INSIKA_STT_PROMPT")
1475
+ end
1476
+
1477
+ # An image/document part -> the ask's attachment (Insika::Media.url_attachment
1478
+ # — egress-guarded, size-capped fetch; the caller picks the ceiling).
1479
+ def media_attachment(url, max_bytes: Insika::Media::MAX_IMAGE_BYTES)
1480
+ Insika::Media.url_attachment(url, max_bytes: max_bytes)
1481
+ end
1482
+
1483
+ # --- WS4 intent routing --------------------------------------------
1484
+ #
1485
+ # The turn's message is classified into one of the profile's routes with a
1486
+ # CHEAP model (data-gated: no `routes` on the profile = byte-identical turn).
1487
+ # -> true when the route took over the turn (delegated / stuck — no ask
1488
+ # happens); false when the turn proceeds normally. Classification happens on
1489
+ # a fresh chat carrying ONLY the auto-generated route prompt (no identity,
1490
+ # no tools — it is a router, not the agent); its tokens ride the turn's
1491
+ # usage, so the trace, the token ceiling and the budget all see the cost.
1492
+ def attempt_route(task, profile, state)
1493
+ meta = Insika::Routing.normalize(profile.routes)
1494
+ return false unless meta
1495
+ return false unless route_model(meta, profile)
1496
+
1497
+ selection = route_selection(meta, profile)
1498
+ classification = classify_route(selection, meta, state.message, task, state)
1499
+ return false if classification.nil? # the classifier call failed — routing is additive
1500
+
1501
+ route = classification[:route]
1502
+ state.route = route
1503
+ # MERGED, not assigned: a WS9 transcription may already have banked its
1504
+ # tokens in state.usage (the classifier call must not erase them).
1505
+ state.usage = merge_usage(with_model_source(usage_of(classification[:response]), selection),
1506
+ state.usage)
1507
+ emit(:route_classified,
1508
+ { task_id: task.id, agent: profile.id.to_s, route: route.to_s,
1509
+ model: selection.model, usage: state.usage },
1510
+ task: task)
1511
+
1512
+ entry = meta[:entries].find { |e| e.name == route.to_s }
1513
+ apply_route_action(task, profile, state, entry)
1514
+ end
1515
+
1516
+ # The classifier call, its answer parsed back into a route.
1517
+ # -> { route:, response: } | nil (nil = the call failed — the turn proceeds
1518
+ # unrouted rather than paying a wrong label or dying for an additive step).
1519
+ def classify_route(selection, meta, message, task, st)
1520
+ response = route_ask(selection, Insika::Routing.classifier_prompt(meta), message, task, st)
1521
+ { route: Insika::Routing.parse(route_response_text(response), meta), response: response }
1522
+ rescue StandardError
1523
+ nil
1524
+ end
1525
+
1526
+ # The cheap classifier's model: routes["model"] wins, the agent's own model
1527
+ # otherwise. No model anywhere = no routing.
1528
+ def route_model(meta, profile)
1529
+ ref = meta[:model].to_s
1530
+ ref = profile.model.to_s if ref.empty?
1531
+ !ref.empty?
1532
+ end
1533
+
1534
+ def route_selection(meta, profile)
1535
+ ref = meta[:model].to_s
1536
+ ref = profile.model.to_s if ref.empty?
1537
+ parsed = parse_model_ref(ref) || {}
1538
+ provider = parsed[:provider].nil? ? profile.provider : parsed[:provider].to_sym
1539
+ Insika::ModelSelection.new(model: parsed[:model], provider: provider, source: :routing)
1540
+ end
1541
+
1542
+ # A fresh chat for the routing model with ONLY the generated prompt. RubyLLM
1543
+ # required lazily, exactly like create_chat (the load-guard holds).
1544
+ # the classifier is model-visible, so it is logged — the ONE
1545
+ # engine-internal ask the conformance spec adds a record for (part
1546
+ # "routing", same turn number as the answer ask).
1547
+ def route_ask(selection, prompt, message, task, st)
1548
+ require "ruby_llm"
1549
+ chat = (@llm || RubyLLM).chat(model: selection.model, provider: selection.provider,
1550
+ assume_model_exists: selection.assume_model_exists?)
1551
+ chat.with_instructions(prompt) if chat.respond_to?(:with_instructions)
1552
+ response = chat.ask(message.to_s)
1553
+ record_model_visible(task, st, chat, part: "routing")
1554
+ response
1555
+ end
1556
+
1557
+ def route_response_text(response)
1558
+ response.respond_to?(:content) ? response.content.to_s : response.to_s
1559
+ end
1560
+
1561
+ # The route's config decides the turn's fate: nothing (a label), a DELEGATE
1562
+ # (an existing agent answers; its reply IS the turn's), or STUCK (WS5 — the
1563
+ # turn ends with the stuck outcome; the consumer interprets it).
1564
+ def apply_route_action(task, profile, state, entry)
1565
+ return false unless entry
1566
+
1567
+ if entry.delegate && !entry.delegate.empty?
1568
+ delegate_route(profile, state, entry.delegate)
1569
+ true
1570
+ elsif entry.stuck
1571
+ message = entry.message.empty? ? entry.description : entry.message
1572
+ state.stuck_outcome = { reason: "route:#{state.route}", message: message }
1573
+ state.response_content = message
1574
+ true
1575
+ else
1576
+ false
1577
+ end
1578
+ end
1579
+
1580
+ # WS4 delegate action: the route names an existing agent — the turn is
1581
+ # handed to it (the sync subagent machinery) and the child's answer IS the
1582
+ # parent's answer. A missing agent or a failed child fails the turn: never
1583
+ # fabricate the customer's reply.
1584
+ #
1585
+ # The depth comes from the PARENT's turn context, +1, and is capped here —
1586
+ # this path does not go through `plan_subagent` (a route has no subagents
1587
+ # allowlist to check against), so a hardcoded depth of 1 made an A -> B -> A
1588
+ # route pair a loop with no floor: every hop reclassifies (a paid ask) and
1589
+ # spawns another child, forever.
1590
+ def delegate_route(profile, state, agent_id)
1591
+ child = @profiles[agent_id.to_s]
1592
+ raise Insika::RoutingError, "route delegate agent '#{agent_id}' not configured" if child.nil?
1593
+
1594
+ depth = (state.turn_context&.dig(:delegation_depth) || 0) + 1
1595
+ cap = SubagentGraph.depth_cap
1596
+ if depth > cap
1597
+ raise Insika::RoutingError,
1598
+ "routed delegate '#{agent_id}' at depth #{depth} exceeds cap #{cap}"
1599
+ end
1600
+
1601
+ result = spawn_and_await_child(child, state.message, depth, state)
1602
+ if result[:error]
1603
+ raise Insika::RoutingError, "routed delegate '#{agent_id}' failed: #{result[:error]}"
1604
+ end
1605
+
1606
+ state.response_content = result[:text].to_s
1607
+ end
1608
+
1609
+ # The WS4 classifier's tokens, summed over the ask's (a call that reported no
1610
+ # usage contributes nothing). The turn's own model/source win for attribution
1611
+ # — the routing model's identity lives on the :route_classified event.
1612
+ def merge_usage(main, extra)
1613
+ return main || extra if main.nil? || extra.nil?
1614
+
1615
+ Insika::Routing::TOKEN_FIELDS.each_with_object(main.dup) do |k, acc|
1616
+ next if extra[k].nil?
1617
+ next unless main.key?(k) || extra[k].to_i.positive?
1618
+
1619
+ acc[k] = main[k].to_i + extra[k].to_i
1620
+ end
1621
+ end
1622
+
1078
1623
  # "provider/model" for any selection duck (ModelSelection | { model:, provider: }).
1079
1624
  def model_ref(selection)
1080
1625
  model = selection.respond_to?(:model) ? selection.model.to_s : selection[:model].to_s
@@ -1122,7 +1667,39 @@ module Insika
1122
1667
  def wire_chat_output(task, state, output)
1123
1668
  chat = state.chat
1124
1669
  chat.after_message { |message| output.message_ended(message) } if chat.respond_to?(:after_message)
1125
- install_steer_injector(task, state)
1670
+ state.steer_injector = install_steer_injector(task, state)
1671
+ end
1672
+
1673
+ # The extra round for a steered message that never met a tool boundary (a
1674
+ # text-only turn closes no batch, so `SteerInjector` never got its cue). The
1675
+ # burst is appended to the history and the model answers ALL of it in one go,
1676
+ # inside the SAME turn — without this the release would spawn a follow-up whose
1677
+ # answer has no delivery channel under request/response.
1678
+ #
1679
+ # Skipped when the turn HALTED: there is no next model step by construction.
1680
+ # `release_steered` (execute's ensure) stays the fallback there, as it is for a
1681
+ # failed or cancelled turn, and for a transport that cannot `complete`.
1682
+ #
1683
+ # At most one extra round: if the model ignores the burst, the turn ends anyway.
1684
+ def absorb_steer_overflow(task, state, timing, result)
1685
+ injector = state.steer_injector
1686
+ return unless injector && state.chat.respond_to?(:complete)
1687
+ return unless state.actor&.user_messages_posted&.positive?
1688
+ return if halted?(result[:response])
1689
+ return if injector.absorb_pending!.zero?
1690
+
1691
+ # TWO provider round trips now. Bank the first one's tokens BEFORE the response
1692
+ # is replaced — stage 6's merge only sees whatever `result[:response]` ends up
1693
+ # being, so without this the extra round silently erases the first round's cost
1694
+ # from the terminal usage and from the EdgeLimiter's budget.
1695
+ state.usage = merge_usage(with_model_source(usage_of(result[:response]), state.model_selection),
1696
+ state.usage)
1697
+ # COMPLETE, not `ask(nil)`: the messages are already in the history, and an
1698
+ # ask with no text appends an EMPTY user message after them — which is both a
1699
+ # duplicate turn-opener and something providers refuse outright.
1700
+ extra = state.chat.complete(&turn_chunk_handler(task, state, result[:output], timing))
1701
+ result[:response] = extra
1702
+ result[:asked] = extra
1126
1703
  end
1127
1704
 
1128
1705
  # The ask itself, chunk-by-chunk (WS3 attempts and the plain path share it).
@@ -1132,9 +1709,27 @@ module Insika
1132
1709
  # :ttft on EVERY content chunk (3 chunks = 3 insika.ttft frames); the spec
1133
1710
  # passed because FakeChat emits a single chunk.
1134
1711
  def ask_on(task, state, chat, output, timing)
1712
+ each_chunk = turn_chunk_handler(task, state, output, timing)
1713
+ # WS9: image parts ride the ask as attachments (only then — a chat whose
1714
+ # ask has no `with:` keeps working, and the plain path is byte-identical).
1715
+ # An image with no caption asks with NIL, not "": an empty text part is a
1716
+ # thing some providers refuse, and nil is how RubyLLM says "attachments
1717
+ # only".
1718
+ if state.media_attachments
1719
+ text = state.message.to_s.empty? ? nil : state.message
1720
+ chat.ask(text, with: state.media_attachments, &each_chunk)
1721
+ else
1722
+ chat.ask(state.message, &each_chunk)
1723
+ end
1724
+ end
1725
+
1726
+ # The per-round chunk sink. Built FRESH per round: `ttft_sent` is that round's
1727
+ # own bookkeeping, and the steer overflow round is a second round on the same
1728
+ # chat.
1729
+ def turn_chunk_handler(task, state, output, timing)
1135
1730
  public_thinking = state.profile.stream_public?(:thinking)
1136
1731
  ttft_sent = false
1137
- chat.ask(state.message) do |chunk|
1732
+ lambda do |chunk|
1138
1733
  emit_thinking(chunk, task, public: public_thinking)
1139
1734
  next unless chunk.content
1140
1735
 
@@ -1148,16 +1743,15 @@ module Insika
1148
1743
  end
1149
1744
 
1150
1745
  # The provider's TTFB as a live event (data: ttft_ms) — only under
1151
- # INSIKA_TURN_TIMING, so absent by default (parity).
1746
+ # INSIKA_TURN_TIMING, so absent by default (parity). Rides the SINGLE
1747
+ # emitter: a hand-built meta here lacked `tenant`, and a tenant-scoped
1748
+ # /v1/events subscription is fail-closed on it — the tenant's own TTFB was
1749
+ # invisible to the tenant.
1152
1750
  def emit_ttft(task, timing)
1153
1751
  ttft = timing.to_h[:ttft_ms]
1154
1752
  return if ttft.nil?
1155
1753
 
1156
- @event_stream.emit(Insika::Event.new(
1157
- type: :ttft, data: { ttft_ms: ttft },
1158
- meta: { task_id: task.id, session_id: task.session_id,
1159
- at: Time.now.utc.iso8601 }
1160
- ))
1754
+ emit(:ttft, { ttft_ms: ttft }, task: task)
1161
1755
  rescue StandardError
1162
1756
  nil
1163
1757
  end
@@ -1321,9 +1915,11 @@ module Insika
1321
1915
  vars["history"] = hist if hist
1322
1916
  # The single type is Insika::ContextRequest (Data); the explicit `history`
1323
1917
  # travels in vars["history"] (Session provider convention), not in a field
1324
- # of its own.
1918
+ # of its own. `memory_scope` is the WS8 customer cell (nil = the providers
1919
+ # fall back to tenant || session, today's behavior).
1325
1920
  ContextRequest.new(profile: profile, message: state.message, session: session,
1326
- checkpoint: resume_from, tenant: command_tenant(task), vars: vars)
1921
+ checkpoint: resume_from, tenant: command_tenant(task), vars: vars,
1922
+ memory_scope: memory_tenant(task))
1327
1923
  end
1328
1924
 
1329
1925
  # task_started payload. Carries the EXPLICIT command tenant so
@@ -1357,8 +1953,57 @@ module Insika
1357
1953
  # Symmetric to the READ path (Memory provider). One-shot with no tenant -> nil
1358
1954
  # (_default). It is NOT the <request_context> tenant (that follows
1359
1955
  # command_tenant, prompt parity) — only the memory read/write scope.
1956
+ #
1957
+ # WS8: a request carrying a CUSTOMER moves the scope to the customer cell —
1958
+ # "[tenant:]customer" when a tenant is present, the bare customer otherwise
1959
+ # (never _default — a tagged customer must never land in the shared cell).
1960
+ # Per-customer memory is the 360 view; per-tenant was the leak.
1961
+ #
1962
+ # the SESSION fallback is MARKED ("chat:<session id>" -> cell
1963
+ # "memory:chat:<session id>"), never a bare cell — a bare "memory:<id>" is
1964
+ # indistinguishable from a single-tenant customer ref, and the Studio drill
1965
+ # must not list conversations as customers with a Forget button.
1360
1966
  def memory_tenant(task)
1361
- command_tenant(task) || task.session_id
1967
+ customer = command_customer(task)
1968
+ return command_tenant(task) || session_scope(task.session_id) if customer.nil?
1969
+
1970
+ [command_tenant(task), customer].compact.join(":")
1971
+ end
1972
+
1973
+ # The marked per-session scope : "chat:<session id>" -> cell
1974
+ # "memory:chat:<session id>". nil for a one-shot turn (no session) — the
1975
+ # MemoryStore applies _default.
1976
+ def session_scope(session_id)
1977
+ return nil if session_id.nil?
1978
+
1979
+ "#{MemoryStore::SESSION_TAG}:#{session_id}"
1980
+ end
1981
+
1982
+ # WS8 + : stamp the customer (WS8 — the `forget_customer`
1983
+ # purge finds the customer's sessions through this var) AND the agent (the
1984
+ # distillation engine resolves each session's pack through it) on the
1985
+ # session ONCE (idempotent). A session that does not exist yet (no
1986
+ # session_id on the turn) is skipped; a look-up failure never breaks the
1987
+ # turn.
1988
+ def stamp_customer_session(task, profile)
1989
+ customer = command_customer(task)
1990
+ return if customer.nil? || task.session_id.nil?
1991
+
1992
+ session = @session_store&.find(task.session_id)
1993
+ return if session.nil? || !Coercion.presence(session.vars["customer"]).nil?
1994
+
1995
+ @session_store.update_vars(task.session_id,
1996
+ "customer" => customer, "agent" => profile.id)
1997
+ rescue Insika::NotFoundError, ArgumentError
1998
+ nil
1999
+ end
2000
+
2001
+ # The optional customer_key on the command payload (WS8): a String identifying
2002
+ # the person the conversation belongs to — the memory scope's customer half
2003
+ # and the handle `forget_customer` purges by. nil = untagged conversation
2004
+ # (memory stays per-tenant/per-chat, byte-identical to before).
2005
+ def command_customer(task)
2006
+ Coercion.presence(rebuild_command(task).payload["customer"])
1362
2007
  end
1363
2008
 
1364
2009
  # Turn context: the ids the data-tools resolve via
@@ -1373,11 +2018,21 @@ module Insika
1373
2018
  {
1374
2019
  chat_id: task.session_id,
1375
2020
  agent_id: profile.id,
1376
- tenant: state.tenant, # already = command_tenant || session_id (memory_tenant)
2021
+ # the DATA-TOOL header tenant stays the merchant (or the chat), even when
2022
+ # the memory scope carries a customer — the backend identifies the store,
2023
+ # not the shopper (WS8 keeps the two scopes separate).
2024
+ tenant: command_tenant(task) || task.session_id,
2025
+ # the DECLARED tenant alone (nil in single-tenant). Distinct from
2026
+ # `tenant` for the ownership-binding tools (save_artifact): a report
2027
+ # belongs to the AGENT's tenant — the deployment's tenant in
2028
+ # single-tenant, never the chat that happened to run it.
2029
+ command_tenant: command_tenant(task),
1377
2030
  store_id: profile.store_id,
1378
- # current delegation depth (0 for a top-level turn). Carried in
1379
- # the child command's payload by run_subagent; read here so the child's OWN
1380
- # spawn_subagent tool sees depth+1 and the runtime cap holds down the chain.
2031
+ # the current task id — the save_artifact binding (which run produced
2032
+ # this report), same turn-origin discipline as the rest of ctx.*.
2033
+ task_id: task.id,
2034
+ # current delegation depth (0 for a top-level turn). Set by run_subagent
2035
+ # for children.
1381
2036
  delegation_depth: delegation_depth(task)
1382
2037
  }
1383
2038
  end
@@ -1701,7 +2356,7 @@ module Insika
1701
2356
  # order mirrors a real turn so both the /v1/responses consumer (which reads the
1702
2357
  # text off :content deltas) and the Studio viewer render it: audit -> safe text
1703
2358
  # -> persist -> terminal.
1704
- def complete_with_halt(task, profile, state)
2359
+ def complete_with_halt(task, profile, state, timing = nil)
1705
2360
  content = state.halt_response.to_s
1706
2361
  state.response_content = content
1707
2362
 
@@ -1722,23 +2377,44 @@ module Insika
1722
2377
  # reading the engine's own canned text (the `safe_reply` finding exists exactly
1723
2378
  # because that text is otherwise indistinguishable in the transcript).
1724
2379
  persist_turn(task, profile, state, content, reply_origin: MessageOrigin::ENGINE,
1725
- session: state.guardrail_block&.[](:source) != "edge")
1726
- emit(:task_completed, { task_id: task.id, content: content, usage: state.usage }, task: task)
2380
+ session: state.guardrail_block&.[](:source) != "edge", timing: timing)
2381
+ data = { task_id: task.id, content: content, usage: state.usage }
2382
+ data[:timing] = timing.to_h if timing # a channel halt still measured
2383
+ persist_turn_timing(task, timing)
2384
+ emit(:task_completed, data, task: task)
1727
2385
  end
1728
2386
 
1729
- # Emits one :guardrail_flagged per flag the OutputValidator appended in
1730
- # after_task (audit only — the turn already completed). Reads a plain Array off
1731
- # the state, keeping the Executor decoupled from Safety.
1732
- def emit_guardrail_flags(task, state)
1733
- return unless state.respond_to?(:guardrail_flags)
2387
+ # Best-effort write of the turn's timing onto the task record .
2388
+ # The record gains `timing` once, when the turn completes; a store failure
2389
+ # here is swallowed — the turn is already committed and the event already
2390
+ # carries the number.
2391
+ def persist_turn_timing(task, timing)
2392
+ return unless timing
1734
2393
 
1735
- Array(state.guardrail_flags).each do |flag|
1736
- emit(:guardrail_flagged, {
1737
- task_id: task.id, category: flag[:category], source: flag[:source], detail: flag[:detail]
1738
- }, task: task)
1739
- end
2394
+ hash = timing.to_h
2395
+ return if hash.empty?
2396
+
2397
+ @task_store.record_timing(task.id, hash)
2398
+ rescue Insika::Error
2399
+ nil
1740
2400
  end
1741
2401
 
2402
+ # Emits one :guardrail_flagged per flag the OutputValidator appended in
2403
+ # after_task (audit only — the turn already completed). Reads a plain Array off
2404
+ # the state, keeping the Executor decoupled from Safety. an
2405
+ # :enforce cut rides the SAME event with `action: "cut"` so the audit can
2406
+ # distinguish a cut from a flag.
2407
+ def emit_guardrail_flags(task, state)
2408
+ return unless state.respond_to?(:guardrail_flags)
2409
+
2410
+ Array(state.guardrail_flags).each do |flag|
2411
+ data = { task_id: task.id, category: flag[:category], source: flag[:source],
2412
+ detail: flag[:detail] }
2413
+ data[:action] = flag[:action] if flag[:action]
2414
+ emit(:guardrail_flagged, data, task: task)
2415
+ end
2416
+ end
2417
+
1742
2418
  # Stage 8: FIXED order checkpoint -> session -> task. If it crashes
1743
2419
  # between writes, the worst case is a new checkpoint with the task :running ->
1744
2420
  # Recovery re-executes the already-saved turn (safe thanks to the side-effect
@@ -1755,7 +2431,7 @@ module Insika
1755
2431
  # So a long session legitimately has a Checkpoint SHORTER than the Session:
1756
2432
  # that is not drift to reconcile — it is the point. Do NOT "fix" the checkpoint
1757
2433
  # to carry the full history (it would defeat the budget) nor evict the session.
1758
- def persist_turn(task, profile, state, content, session: true, reply_origin: nil)
2434
+ def persist_turn(task, profile, state, content, session: true, reply_origin: nil, timing: nil)
1759
2435
  new_messages = turn_transcript(state, content, origin: command_origin(task), reply_origin: reply_origin)
1760
2436
  transcript = flatten_history(state.context.history) + new_messages
1761
2437
 
@@ -1793,7 +2469,13 @@ module Insika
1793
2469
  # if this turn CAME IN through a Shape B channel, its answer
1794
2470
  # has to travel out of band. Same terminal hook, next door to the delegation
1795
2471
  # one, for the same reason: it fires for a fresh turn and a recovered one.
1796
- finalize_channel_delivery(task, content)
2472
+ finalize_channel_delivery(task, content, state, timing)
2473
+
2474
+ # extracts durable concepts from this turn, off the critical
2475
+ # path — the user already has the answer above. Same terminal hook,
2476
+ # next door to the other two, for the same reason: it fires for a fresh
2477
+ # turn and a recovered one.
2478
+ finalize_knowledge_extraction(task, profile, new_messages)
1797
2479
  end
1798
2480
 
1799
2481
  # Records the answer in the outbox and dispatches it. The discriminator is the
@@ -1810,31 +2492,133 @@ module Insika
1810
2492
  #
1811
2493
  # Best-effort: the turn is already committed and durable, and a delivery problem
1812
2494
  # must never re-fail it.
1813
- def finalize_channel_delivery(task, content)
2495
+ #
2496
+ # a PROGRESSIVE channel gets the answer split into balloons —
2497
+ # N outbox rows, dispatched in index order (dispatch_chain). `:at_end` is the
2498
+ # single whole-answer row, byte-identical to today.
2499
+ def finalize_channel_delivery(task, content, state, timing = nil)
1814
2500
  return unless @channel_delivery
1815
2501
 
1816
2502
  channel_id = channel_transport(task)
1817
2503
  return unless channel_id
1818
2504
 
1819
- delivery = @channel_delivery.record(task: task, channel_id: channel_id, content: content)
1820
- return unless delivery
2505
+ # the hoarded evidence attachments ride the channel delivery
2506
+ # (additive outbox payload key — the channel contract widens, nothing breaks).
2507
+ attachments = state.respond_to?(:evidence_attachments) ? state.evidence_attachments : nil
2508
+ deliveries = @channel_delivery.record_balloons(
2509
+ task: task, channel_id: channel_id, content: content,
2510
+ progressive: @channel_delivery.progressive?(channel_id),
2511
+ attachments: attachments
2512
+ )
2513
+ return if deliveries.empty?
1821
2514
 
1822
- dispatch_delivery(delivery.id)
2515
+ timing&.mark(:first_balloon) # C5: inbound -> first outbox row, first-write-wins
2516
+ dispatch_chain(deliveries.map(&:id))
1823
2517
  rescue Insika::Error
1824
2518
  nil
1825
2519
  end
1826
2520
 
1827
- # The POST goes out on the SUPERVISOR, never on the turn's fiber: a bounded
1828
- # retry against a third party would otherwise hold the session's FIFO — the
1829
- # customer's next message would wait on their previous answer's delivery.
1830
- # Non-serving (boot sweep, specs) delivers inline, where waiting is what the
1831
- # caller wants.
1832
- def dispatch_delivery(delivery_id)
1833
- return @channel_delivery.deliver(delivery_id) unless @supervised
2521
+ # Turns whose combined transcript slice is this trivially short skip
2522
+ # extraction entirely ("ok thanks" exchanges) — no new config surface,
2523
+ # just avoids a wasted utility-model call.
2524
+ KNOWLEDGE_MIN_CHARS = 200
2525
+
2526
+ # No-op without a knowledge store, without the profile's opt-in
2527
+ # (`knowledge.extract`), without a usable model, or for a trivially short
2528
+ # turn. Otherwise dispatches the extraction off the critical path — the
2529
+ # SAME `dispatch_chain` shape: inline when non-supervised (tests, CLI,
2530
+ # boot sweep), a child of the turn supervisor when serving (survives the
2531
+ # request's own disconnect). Best-effort: any failure is swallowed here,
2532
+ # never re-fails an already-committed turn.
2533
+ def finalize_knowledge_extraction(task, profile, new_messages)
2534
+ return unless @knowledge_store
2535
+
2536
+ config = Coercion.deep_stringify(profile.knowledge)
2537
+ return unless config && Coercion.truthy?(config["extract"])
2538
+ return if knowledge_transcript(new_messages).length < KNOWLEDGE_MIN_CHARS
2539
+
2540
+ extractor = Knowledge::ExtractorFactory.build(config, utility_model: utility_model)
2541
+ return unless extractor
2542
+
2543
+ # Same resolved model as the extractor — a deployment names one
2544
+ # knowledge model, not two. nil consolidator (no model resolvable) is
2545
+ # still meaningful: write_concept's conservative default.
2546
+ consolidator = Knowledge::ConsolidatorFactory.build(config, utility_model: utility_model)
2547
+
2548
+ run = lambda { run_knowledge_extraction(task, profile, config, new_messages, extractor, consolidator) }
2549
+ return run.call unless @supervised
2550
+
2551
+ turn_parent.async do |t|
2552
+ t.annotate("knowledge:#{task.id}")
2553
+ run.call
2554
+ end
2555
+ end
2556
+
2557
+ def run_knowledge_extraction(task, profile, config, new_messages, extractor, consolidator)
2558
+ prompt = knowledge_prompt(config, new_messages)
2559
+ result = extractor.extract(prompt: prompt)
2560
+ result[:concepts].each do |concept|
2561
+ outcome = Knowledge.write_concept(
2562
+ store: @knowledge_store, agent_id: profile.id, concept: concept, session_id: task.session_id,
2563
+ tenant: task_tenant(task), consolidator: consolidator
2564
+ )
2565
+ emit_knowledge_event(outcome, profile, task)
2566
+ end
2567
+ rescue StandardError
2568
+ nil # best-effort: extraction never re-fails an already-committed turn.
2569
+ end
2570
+
2571
+ # :new/:related taught the agent something; :contradicting needs a
2572
+ # human; :same is a silent reinforcement (no event — a popular concept
2573
+ # would otherwise spam the stream every time it's confirmed).
2574
+ def emit_knowledge_event(outcome, profile, task)
2575
+ case outcome[:verdict]
2576
+ when :new, :related
2577
+ emit(:knowledge_learned, { name: outcome[:name], type: outcome[:type], agent: profile.id }, task: task)
2578
+ when :contradicting
2579
+ emit(:knowledge_conflict, { name: outcome[:name], agent: profile.id }, task: task)
2580
+ end
2581
+ end
2582
+
2583
+ def knowledge_prompt(config, new_messages)
2584
+ base = Coercion.presence(config["prompt"]) || Knowledge::DEFAULT_PROMPT
2585
+ <<~PROMPT
2586
+ #{base.rstrip}
2587
+
2588
+ ## The conversation
2589
+
2590
+ #{knowledge_transcript(new_messages)}
2591
+ PROMPT
2592
+ end
2593
+
2594
+ # Redacted (RFC's PII rule applies to what reaches the model too, not
2595
+ # just what gets persisted).
2596
+ def knowledge_transcript(new_messages)
2597
+ redacted, = Insika::Safety::Detectors.redact(
2598
+ new_messages.each_with_index.map { |m, i| "[#{i}] #{m['role'] || m[:role]}: #{m['content'] || m[:content]}" }
2599
+ .join("\n")
2600
+ )
2601
+ redacted
2602
+ end
2603
+
2604
+ def utility_model
2605
+ return nil unless @settings_store
2606
+
2607
+ @settings_store.get["utility_model"]
2608
+ end
2609
+
2610
+ # ONE supervisor fiber for the whole chain. Sequential deliver
2611
+ # calls, so balloon N+1 cannot overtake balloon N on the wire. Still off the
2612
+ # session's FIFO — the customer's next message does not wait on this turn's
2613
+ # outbound. Non-serving (boot sweep, specs) delivers inline, where waiting is
2614
+ # what the caller wants.
2615
+ def dispatch_chain(ids)
2616
+ run = lambda { ids.each { |id| @channel_delivery.deliver(id) } }
2617
+ return run.call unless @supervised
1834
2618
 
1835
2619
  turn_parent.async do |t|
1836
- t.annotate("outbox:#{delivery_id}")
1837
- @channel_delivery.deliver(delivery_id)
2620
+ t.annotate("outbox:#{ids.first}")
2621
+ run.call
1838
2622
  end
1839
2623
  end
1840
2624
 
@@ -1936,16 +2720,25 @@ module Insika
1936
2720
 
1937
2721
  # Stage 6 (factory): the ONLY point that touches the gem. lazy require,
1938
2722
  # confined — not covered by unit (factory line). It also loads the system
1939
- # builtins (load_skill/tool_search/remember) that the ChatBuilder assembles at
1940
- # stage 5 — lazy, so the core installs without ruby_llm.
2723
+ # builtins (load_skill/load_knowledge/tool_search/remember) that the
2724
+ # ChatBuilder assembles at stage 5 — lazy, so the core installs without
2725
+ # ruby_llm.
1941
2726
  def create_chat(profile, state)
1942
2727
  require "ruby_llm"
1943
2728
  require_relative "tools/load_skill"
2729
+ require_relative "tools/load_knowledge"
1944
2730
  require_relative "tools/tool_search"
1945
2731
  require_relative "tools/remember"
1946
2732
  require_relative "tools/subagent"
1947
2733
  require_relative "tools/subagents"
1948
2734
  require_relative "tools/stuck_signal"
2735
+ require_relative "tools/generate_image"
2736
+ require_relative "tools/tts"
2737
+ require_relative "tools/update_briefing"
2738
+ # the schedule/cancel_followup builtins — lazy, same
2739
+ # boundary (the ChatBuilder wires them only when a profile declares
2740
+ # followup AND the stores are present).
2741
+ require_relative "tools/schedule_followup"
1949
2742
  # v2 resolution: Chat pin > Agent model > platform default, model_policy
1950
2743
  # enforced, fallback chain resolved. Kept on the state for telemetry (usage).
1951
2744
  selection = @model_resolver.resolve(profile: profile, session: state.session)