insika 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (185) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +145 -17
  3. data/README.md +7 -1
  4. data/bin/insika +211 -8
  5. data/docs/AGENTS.md +377 -2
  6. data/docs/API.md +56 -0
  7. data/docs/CHANNELS.md +99 -9
  8. data/docs/CONTEXT.md +127 -16
  9. data/docs/DEPLOY.md +5 -2
  10. data/docs/EVALS.md +19 -0
  11. data/docs/FACTS.md +135 -0
  12. data/docs/HARVEST.md +117 -0
  13. data/docs/LOADTEST.md +14 -8
  14. data/docs/OBSERVABILITY.md +54 -0
  15. data/docs/REFINEMENT.md +3 -3
  16. data/docs/RELEASING.md +27 -0
  17. data/docs/RUNNING-LOCAL.md +3 -3
  18. data/docs/SECURITY.md +61 -0
  19. data/docs/SKILLS.md +2 -2
  20. data/docs/SOAK.md +127 -0
  21. data/docs/TOOLS.md +70 -2
  22. data/docs/domain.md +115 -0
  23. data/docs/index.md +1 -1
  24. data/lib/insika/agent_profile.rb +145 -10
  25. data/lib/insika/balloon_splitter.rb +102 -0
  26. data/lib/insika/budget_ledger.rb +34 -3
  27. data/lib/insika/cache_series_store.rb +49 -0
  28. data/lib/insika/channel_delivery.rb +118 -13
  29. data/lib/insika/channels/relay.rb +77 -3
  30. data/lib/insika/channels/web.rb +2 -2
  31. data/lib/insika/chat_builder.rb +84 -1
  32. data/lib/insika/checkpoint_store.rb +16 -0
  33. data/lib/insika/commands/agent_payload.rb +3 -3
  34. data/lib/insika/commands/cancel_followup.rb +49 -0
  35. data/lib/insika/commands/delete_tenant_data.rb +95 -0
  36. data/lib/insika/commands/export_customer_memory.rb +48 -0
  37. data/lib/insika/commands/forget_customer.rb +117 -0
  38. data/lib/insika/commands/freeze_funnel_baseline.rb +113 -0
  39. data/lib/insika/commands/gate_harvest.rb +138 -0
  40. data/lib/insika/commands/gate_refinement.rb +1 -1
  41. data/lib/insika/commands/judge_shadow_pairs.rb +124 -0
  42. data/lib/insika/commands/memory_forget_fact.rb +20 -4
  43. data/lib/insika/commands/memory_put_fact.rb +23 -4
  44. data/lib/insika/commands/promote_harvest.rb +130 -0
  45. data/lib/insika/commands/record_outcome.rb +46 -0
  46. data/lib/insika/commands/record_shadow_reply.rb +68 -0
  47. data/lib/insika/commands/reject_harvest.rb +38 -0
  48. data/lib/insika/commands/resolve_proposal.rb +108 -0
  49. data/lib/insika/commands/revoke_contact.rb +49 -0
  50. data/lib/insika/commands/rollback_harvest.rb +86 -0
  51. data/lib/insika/commands/run_distillation.rb +186 -0
  52. data/lib/insika/commands/run_harvest.rb +393 -0
  53. data/lib/insika/commands/send_message.rb +103 -6
  54. data/lib/insika/commands/session_purge.rb +67 -0
  55. data/lib/insika/contact_store.rb +183 -0
  56. data/lib/insika/context/builder.rb +21 -3
  57. data/lib/insika/context/fragment.rb +7 -3
  58. data/lib/insika/context/priority.rb +3 -0
  59. data/lib/insika/context/provider.rb +17 -3
  60. data/lib/insika/context/providers/briefing.rb +96 -0
  61. data/lib/insika/context/providers/memory.rb +16 -7
  62. data/lib/insika/context/providers/prompt.rb +30 -2
  63. data/lib/insika/context/providers/skill.rb +2 -0
  64. data/lib/insika/context/providers/tool_search.rb +2 -0
  65. data/lib/insika/context_trace_store.rb +38 -2
  66. data/lib/insika/distill.rb +224 -0
  67. data/lib/insika/distill_engine.rb +169 -0
  68. data/lib/insika/doctor.rb +716 -6
  69. data/lib/insika/dsl/runtime.rb +9 -3
  70. data/lib/insika/dsl/server_boot.rb +70 -1
  71. data/lib/insika/dsl.rb +83 -3
  72. data/lib/insika/edge_limiter.rb +28 -7
  73. data/lib/insika/env_schema.rb +9 -2
  74. data/lib/insika/errors.rb +11 -0
  75. data/lib/insika/evals/assertions.rb +3 -2
  76. data/lib/insika/evals/pairwise.rb +18 -6
  77. data/lib/insika/evidence.rb +183 -0
  78. data/lib/insika/executor.rb +687 -70
  79. data/lib/insika/followup_engine.rb +207 -0
  80. data/lib/insika/followup_policy.rb +221 -0
  81. data/lib/insika/followup_store.rb +306 -0
  82. data/lib/insika/funnel_declaration.rb +106 -0
  83. data/lib/insika/funnel_fold.rb +179 -0
  84. data/lib/insika/funnel_store.rb +163 -0
  85. data/lib/insika/golden_store.rb +1 -1
  86. data/lib/insika/grounding/matcher.rb +69 -0
  87. data/lib/insika/grounding.rb +44 -0
  88. data/lib/insika/harvest/conversion_gate.rb +159 -0
  89. data/lib/insika/harvest/criterion.rb +98 -0
  90. data/lib/insika/harvest/gate.rb +194 -0
  91. data/lib/insika/harvest/negative_list.rb +199 -0
  92. data/lib/insika/harvest.rb +241 -0
  93. data/lib/insika/harvest_engine.rb +193 -0
  94. data/lib/insika/harvest_store.rb +548 -0
  95. data/lib/insika/media.rb +298 -0
  96. data/lib/insika/memory_audit_store.rb +85 -0
  97. data/lib/insika/memory_store.rb +264 -23
  98. data/lib/insika/message_origin.rb +6 -1
  99. data/lib/insika/model_visible.rb +87 -0
  100. data/lib/insika/model_visible_trace_store.rb +66 -0
  101. data/lib/insika/onboarding.rb +6 -1
  102. data/lib/insika/outbox_store.rb +42 -4
  103. data/lib/insika/outcome_store.rb +147 -0
  104. data/lib/insika/packaging.rb +163 -0
  105. data/lib/insika/parity/criterion.rb +79 -0
  106. data/lib/insika/parity/verdict.rb +318 -0
  107. data/lib/insika/prefix_fingerprint.rb +58 -0
  108. data/lib/insika/profile_source.rb +22 -1
  109. data/lib/insika/proposal_store.rb +271 -0
  110. data/lib/insika/queue_policy.rb +4 -1
  111. data/lib/insika/refinement/proposer.rb +1 -1
  112. data/lib/insika/reliability.rb +32 -6
  113. data/lib/insika/retention.rb +281 -0
  114. data/lib/insika/routing.rb +101 -0
  115. data/lib/insika/safety/config.rb +44 -4
  116. data/lib/insika/safety/corpus.rb +255 -0
  117. data/lib/insika/safety/detectors.rb +32 -113
  118. data/lib/insika/safety/factory.rb +15 -2
  119. data/lib/insika/safety/grounding_enforcer.rb +59 -0
  120. data/lib/insika/safety/grounding_validator.rb +49 -0
  121. data/lib/insika/safety/input_guardrail.rb +1 -1
  122. data/lib/insika/safety/moderator.rb +1 -1
  123. data/lib/insika/safety/output_filter.rb +9 -5
  124. data/lib/insika/safety/output_validator.rb +11 -5
  125. data/lib/insika/schema_guard.rb +35 -0
  126. data/lib/insika/server/app.rb +144 -4
  127. data/lib/insika/server/rack_app.rb +16 -0
  128. data/lib/insika/server/responses.rb +40 -1
  129. data/lib/insika/session_actor.rb +8 -4
  130. data/lib/insika/session_store.rb +65 -2
  131. data/lib/insika/settings_store.rb +10 -0
  132. data/lib/insika/shadow_pair_store.rb +258 -0
  133. data/lib/insika/skill_catalog.rb +4 -0
  134. data/lib/insika/soak/envelope.rb +140 -0
  135. data/lib/insika/soak/report.rb +392 -0
  136. data/lib/insika/soak/runner.rb +554 -0
  137. data/lib/insika/store.rb +10 -1
  138. data/lib/insika/stores/memory.rb +6 -0
  139. data/lib/insika/stores/sqlite.rb +8 -0
  140. data/lib/insika/studio/app.rb +875 -14
  141. data/lib/insika/studio/assets/dist/application.css +1 -1
  142. data/lib/insika/studio/assets/dist/application.js +22 -22
  143. data/lib/insika/studio/assets/dist/favicon.svg +6 -0
  144. data/lib/insika/studio/forms.rb +262 -5
  145. data/lib/insika/studio/nav_icons.rb +6 -1
  146. data/lib/insika/studio/views/agent_detail.erb +627 -84
  147. data/lib/insika/studio/views/agents.erb +10 -6
  148. data/lib/insika/studio/views/approvals.erb +4 -1
  149. data/lib/insika/studio/views/chats.erb +4 -1
  150. data/lib/insika/studio/views/customer.erb +94 -0
  151. data/lib/insika/studio/views/customers.erb +32 -0
  152. data/lib/insika/studio/views/evals.erb +4 -1
  153. data/lib/insika/studio/views/facts.erb +133 -0
  154. data/lib/insika/studio/views/followups.erb +125 -0
  155. data/lib/insika/studio/views/funnel.erb +106 -0
  156. data/lib/insika/studio/views/harvest.erb +234 -0
  157. data/lib/insika/studio/views/home.erb +2 -1
  158. data/lib/insika/studio/views/layout.erb +1 -0
  159. data/lib/insika/studio/views/parity.erb +147 -0
  160. data/lib/insika/studio/views/playground.erb +7 -1
  161. data/lib/insika/studio/views/session.erb +56 -1
  162. data/lib/insika/studio/views/settings.erb +2 -0
  163. data/lib/insika/studio/views/skills.erb +0 -1
  164. data/lib/insika/studio/views/system_files.erb +1 -1
  165. data/lib/insika/studio/views/task.erb +13 -0
  166. data/lib/insika/studio/views/tasks.erb +4 -1
  167. data/lib/insika/studio/views/tools.erb +0 -1
  168. data/lib/insika/task_store.rb +21 -1
  169. data/lib/insika/testing/store_contract.rb +27 -6
  170. data/lib/insika/tick.rb +25 -1
  171. data/lib/insika/tool_definition.rb +17 -7
  172. data/lib/insika/tool_envelope.rb +69 -0
  173. data/lib/insika/tool_manifest.rb +5 -1
  174. data/lib/insika/tools/data_defined_tool.rb +10 -0
  175. data/lib/insika/tools/generate_image.rb +44 -0
  176. data/lib/insika/tools/schedule_followup.rb +164 -0
  177. data/lib/insika/tools/tts.rb +47 -0
  178. data/lib/insika/tools/update_briefing.rb +126 -0
  179. data/lib/insika/turn_state.rb +38 -1
  180. data/lib/insika/turn_timing.rb +22 -2
  181. data/lib/insika/version.rb +1 -1
  182. data/lib/insika/vitals.rb +84 -0
  183. data/lib/insika/wiring/graph.rb +300 -13
  184. data/lib/insika.rb +92 -3
  185. metadata +77 -1
@@ -20,7 +20,9 @@ module Insika
20
20
  capability_registry: nil, tool_catalog: nil, memory_store: nil,
21
21
  tool_trace_store: nil, settings_store: nil, content_filter_factory: nil,
22
22
  delegation_store: nil, channel_delivery: nil, llm: nil,
23
- context_trace_store: nil, reliability: nil)
23
+ context_trace_store: nil, reliability: nil, media: nil, media_output: nil,
24
+ grounding_enforcer: nil, cache_series_store: nil,
25
+ contact_store: nil, followup_store: nil, model_visible_trace_store: nil)
24
26
  @context_builder = context_builder
25
27
  @policy_engine = policy_engine
26
28
  @middleware = middleware
@@ -40,6 +42,10 @@ module Insika
40
42
  # per-turn context breakdown (tokens by category + budget) for the
41
43
  # Studio session card. nil = off (no record, zero overhead — parity).
42
44
  @context_trace_store = context_trace_store
45
+ # the model-visible trace — what the provider received per
46
+ # (task, turn), captured at the chat boundary. nil = off (no record,
47
+ # zero overhead — parity).
48
+ @model_visible_trace_store = model_visible_trace_store
43
49
  # Guardrails output filter: ->(state) { OutputFilter | nil }.
44
50
  # Injected by the Safety::Factory; nil = off (parity — the stream is untouched).
45
51
  # The INPUT guardrail is a Middleware (in the stack, not here); this is the seam
@@ -64,10 +70,31 @@ module Insika
64
70
  # fallback / circuit breaker, all DATA on AgentProfile#reliability.
65
71
  # nil = the plain single ask (parity).
66
72
  @reliability = reliability
73
+ # WS9 media seam (nil = default built on first audio turn): ->(url) { text }.
74
+ @media = media
75
+ # WS9 (saída) media-generation seams: { image: ->(prompt, cfg) [part,
76
+ # usage], tts: ->(text, cfg) [part, usage] }. nil = the defaults (built
77
+ # lazily on first generation — RubyLLM + Net::HTTP behind lazy requires,
78
+ # the core stays gem-free at load). Injected by specs; a production graph
79
+ # that wants a non-RubyLLM backend injects its own lambdas.
80
+ @media_output = media_output
81
+ # the :enforce boundary step, called between stages 6 and 8.
82
+ # Defaults to a REAL enforcer (inert unless the profile's grounding.mode is
83
+ # :enforce — zero behavior change for parity) so an embedder that builds
84
+ # the Executor directly still gets the cut; `nil` stays injectable for
85
+ # stubs that want none.
86
+ @grounding_enforcer = grounding_enforcer || Insika::Safety::GroundingEnforcer.new
87
+ # the per-AGENT cache-hit series. nil = no series recorded
88
+ # (parity — the trace store still gets the per-turn entry when wired).
89
+ @cache_series_store = cache_series_store
67
90
  # LLM config v2: resolves the model at turn start (Chat > Agent >
68
91
  # platform default) + model_policy + fallback chain. settings_store nil =
69
92
  # no platform layer (pre-v2 behavior: the agent's own model is used as-is).
70
93
  @model_resolver = ModelResolver.new(settings_store: settings_store)
94
+ # the follow-up stores the ChatBuilder gates the
95
+ # schedule/cancel_followup tools on (nil = never wired — parity).
96
+ @contact_store = contact_store
97
+ @followup_store = followup_store
71
98
  # the platform layer of the queue policy (nil = per-agent and
72
99
  # defaults only, which is `followup` with no window — today's behavior).
73
100
  @settings_store = settings_store
@@ -84,7 +111,19 @@ module Insika
84
111
  # the ChatBuilder wires the spawn_subagent system tool (gated by
85
112
  # profile.subagents) and hands it this Executor as the runner. `self` is not
86
113
  # yet fully built here, but the ChatBuilder only STORES it (used per-turn).
87
- subagent_runner: self
114
+ subagent_runner: self,
115
+ # WS9 (saída): the media-generation runner, same shape — the Executor
116
+ # owns the seams + usage accounting, the builder only wires the tools
117
+ # the turn's gates allow.
118
+ media_runner: self,
119
+ # the builder wires the briefing-write system tools gated by
120
+ # @session_store + profile.briefing_fields. nil = never wired (parity).
121
+ session_store: session_store,
122
+ # the builder wires the schedule/cancel_followup system
123
+ # tools gated by a parsed policy AND both stores present. nil = never
124
+ # wired (parity).
125
+ contact_store: contact_store,
126
+ followup_store: followup_store
88
127
  )
89
128
  # Stage-3-tail tool assembly (capability resolution, instantiation,
90
129
  # injection, dedup join, ToolEnvelope wrap) — extracted collaborator.
@@ -124,6 +163,18 @@ module Insika
124
163
  # the turn supervisor in serving mode (like the tick); nil = no alerts.
125
164
  attr_accessor :alert_dispatcher
126
165
 
166
+ # the distillation engine — the tick-duty that finds idle
167
+ # customer sessions and distills them on its own worker fiber (a child of
168
+ # the turn supervisor, like the tick). nil = distillation off (parity —
169
+ # nothing scans, nothing distills).
170
+ attr_accessor :distill_engine
171
+
172
+ # the harvest engine — the tick-duty that finds idle,
173
+ # unmined sessions and mines them on its own worker fiber (a child of the
174
+ # turn supervisor, like the tick). nil = harvest off (parity — nothing
175
+ # scans, nothing mines).
176
+ attr_accessor :harvest_engine
177
+
127
178
  # closes the TURN intake for shutdown. Armed by Insika::Shutdown
128
179
  # when the process is asked to stop: from here on a new top-level turn is left
129
180
  # `:queued` (durable — the next boot's recovery replays it) instead of
@@ -225,12 +276,17 @@ module Insika
225
276
 
226
277
  # Stage 1 (async part): creates the actor, registers it and fires the fiber.
227
278
  # Called by the turn handlers (SendMessage/ResumeTask/TriggerWorkflow).
228
- def spawn(task, profile:, resume_from: nil)
279
+ #
280
+ # `timing` is the channel clock a channel turn allocated at 202
281
+ # acceptance, already carrying `:inbound`; nil means the pipeline allocates its
282
+ # own (resume, engine-initiated, non-channel). `mark` is first-write-wins, so
283
+ # re-marking `:inbound` in the pipeline is a no-op on a threaded clock.
284
+ def spawn(task, profile:, resume_from: nil, timing: nil)
229
285
  raise Insika::ValidationError, "task already running: #{task.id}" if running?(task.id)
230
286
 
231
287
  actor = TaskActor.new(task_id: task.id, parent: turn_parent)
232
288
  @running[task.id] = actor
233
- actor.run { execute(task, profile: profile, resume_from: resume_from, actor: actor) }
289
+ actor.run { execute(task, profile: profile, resume_from: resume_from, actor: actor, timing: timing) }
234
290
  task.id
235
291
  end
236
292
 
@@ -238,7 +294,7 @@ module Insika
238
294
  # session_id is SERIALIZED in that session's SessionActor queue (one at a
239
295
  # time); without a session_id (one-shot/history) it goes straight to spawn
240
296
  # (standalone).
241
- def spawn_in_session(task, profile:, resume_from: nil)
297
+ def spawn_in_session(task, profile:, resume_from: nil, timing: nil)
242
298
  # the intake is closed. The task is already durable (queued);
243
299
  # answering with its id and spawning NOTHING is what "stops accepting new
244
300
  # turns" means — the next boot's recovery replays it. Subagent turns are NOT
@@ -252,11 +308,12 @@ module Insika
252
308
  # owner awaits the turn). One-shot/history (no session_id)
253
309
  # never serialize.
254
310
  unless @supervised && task.session_id
255
- return spawn(task, profile: profile, resume_from: resume_from)
311
+ return spawn(task, profile: profile, resume_from: resume_from, timing: timing)
256
312
  end
257
313
 
258
314
  session_actor(task.session_id).enqueue(task, profile: profile, resume_from: resume_from,
259
- policy: queue_policy(profile, task.session_id))
315
+ policy: queue_policy(profile, task.session_id),
316
+ timing: timing)
260
317
  end
261
318
 
262
319
  # the `collect` door, asked BEFORE a task is created.
@@ -296,7 +353,8 @@ module Insika
296
353
  return nil unless session_actor&.alive?
297
354
 
298
355
  # No turn running (or one still at the door): there is nothing to steer INTO.
299
- # A turn at the door belongs to `collect`, which is a different mode.
356
+ # A turn at the door is the collect door's other window — a steer agent with a
357
+ # debounce merges there instead , so no message waits on either.
300
358
  task = session_actor.current_task
301
359
  return nil if task.nil?
302
360
  # A workflow turn orchestrates RubyLLM itself and has no Insika chat to append to
@@ -404,13 +462,13 @@ module Insika
404
462
  # its completion before returning — that is what serializes the session. A
405
463
  # turn error is already mapped to a terminal state inside its own fiber (single
406
464
  # capture); here we only ensure the session loop does not die.
407
- def run_serial(task, profile:, resume_from: nil)
465
+ def run_serial(task, profile:, resume_from: nil, timing: nil)
408
466
  # a drain that started with turns already queued behind this
409
467
  # session's current one must not keep feeding the loop — without this gate
410
468
  # the drain would only converge when the whole backlog ran out.
411
469
  return defer_turn(task) if @draining
412
470
 
413
- spawn(task, profile: profile, resume_from: resume_from)
471
+ spawn(task, profile: profile, resume_from: resume_from, timing: timing)
414
472
  @running[task.id]&.wait
415
473
  rescue Async::Stop
416
474
  raise # shutdown: propagate (ends the session loop)
@@ -505,7 +563,7 @@ module Insika
505
563
  end
506
564
 
507
565
  # Stages 2..9. Runs INSIDE the task's fiber.
508
- def execute(task, profile:, actor:, resume_from: nil)
566
+ def execute(task, profile:, actor:, resume_from: nil, timing: nil)
509
567
  # Resume of a crash orphan: the interrupted attempt's Execution was left OPEN
510
568
  # (the fiber died). The TaskStore forbids opening a second one while one is
511
569
  # open -> close the orphan as :interrupted before opening the N+1 (a new
@@ -519,7 +577,7 @@ module Insika
519
577
  emit(:task_started, started_data(task, profile), task: task)
520
578
 
521
579
  actor.drain!
522
- run_pipeline(task, profile, actor, resume_from)
580
+ run_pipeline(task, profile, actor, resume_from, timing)
523
581
  # SINGLE capture at the top of the fiber: a single place maps
524
582
  # error -> terminal state -> events. Stages do no rescue of their own
525
583
  # (except tool, RubyLLM semantics). The fiber NEVER re-raises.
@@ -541,6 +599,14 @@ module Insika
541
599
  # reads circuit_open + retry_after (cooldown remaining). Worth its own
542
600
  # stage: an open breaker is a reliability decision, not an error bug.
543
601
  fail_task(task, e, stage: :reliability)
602
+ rescue Insika::RoutingError => e
603
+ # WS4: a route's delegate is missing or its turn failed — an operator
604
+ # config error, staged so the envelope names routing, never :unknown.
605
+ fail_task(task, e, stage: :routing)
606
+ rescue Insika::MediaError => e
607
+ # WS9: a voice message that could not be fetched/transcribed (or a media
608
+ # URL the egress guard refused) — heard-loud, never a silent drop.
609
+ fail_task(task, e, stage: :media)
544
610
  rescue Insika::WorkflowSchemaError => e
545
611
  # a workflow OUTPUT that violates its output_schema. Distinct
546
612
  # stage so a contract breach is not conflated with an :unknown failure. (INPUT
@@ -574,6 +640,36 @@ module Insika
574
640
  release_steered(task, profile, actor)
575
641
  end
576
642
 
643
+ # WS9 (saída), the RUNNER side the generate_image/tts system tools call
644
+ # (public like run_subagent — a tool reaches back into the Executor):
645
+ #
646
+ # -> [part, usage]: resolve the seam (injected or the lazy default) and
647
+ # run it. The default seams are built on FIRST use, when the turn already
648
+ # has a chat (ruby_llm loaded), so the load-guard holds.
649
+ def generate_media_output(kind, content, config)
650
+ seam = @media_output&.fetch(kind, nil) || Insika::Media::Output.defaults(context: @llm)[kind]
651
+ raise Insika::MediaError, "no #{kind} output seam" unless seam
652
+
653
+ seam.call(content, config)
654
+ end
655
+
656
+ # Accounts a generated part in the turn's usage: the provider's token
657
+ # counts (images — the merge keeps what the classifier already banked),
658
+ # plus an honest `media` call counter per part (the speech API reports no
659
+ # tokens; the part itself carries the model for consumer-side pricing).
660
+ def account_media_usage(state, part, usage)
661
+ usage ||= {}
662
+ tokens = {}
663
+ tokens[:input_tokens] = usage[:input_tokens].to_i if usage[:input_tokens]
664
+ tokens[:output_tokens] = usage[:output_tokens].to_i if usage[:output_tokens]
665
+ tokens[:total_tokens] = tokens[:input_tokens].to_i + tokens[:output_tokens].to_i if tokens.any?
666
+ unless tokens.empty?
667
+ tokens[:model] = part["model"] if part["model"]
668
+ state.usage = merge_usage(tokens, state.usage)
669
+ end
670
+ state.usage = (state.usage || {}).merge(media: state.usage&.fetch(:media, 0).to_i + 1)
671
+ end
672
+
577
673
  private
578
674
 
579
675
  # resolves session vars > profile.limits > settings["queue"] >
@@ -646,6 +742,12 @@ module Insika
646
742
  # the alert dispatcher (WS6) lives on the same supervisor: its consumer
647
743
  # answers every alert event for as long as the process serves.
648
744
  @alert_dispatcher&.start(parent: @supervisor)
745
+ # the distillation engine lives on the same supervisor —
746
+ # its worker fiber re-scans idle sessions off the turn path.
747
+ @distill_engine&.start(parent: @supervisor)
748
+ # the harvest engine lives on the same supervisor — its
749
+ # worker fiber re-scans idle sessions off the turn path.
750
+ @harvest_engine&.start(parent: @supervisor)
649
751
  @supervisor
650
752
  end
651
753
 
@@ -710,8 +812,23 @@ module Insika
710
812
  # Stages 2-9, with mailbox drain only at the boundaries and the
711
813
  # turn-timeout wrapping everything via Async::Task#with_timeout — NEVER
712
814
  # stdlib Timeout.timeout.
713
- def run_pipeline(task, profile, actor, resume_from)
714
- timing = TurnTiming.new if TurnTiming.enabled?
815
+ def run_pipeline(task, profile, actor, resume_from, timing = nil)
816
+ # a CHANNEL turn always allocates the clock — first_balloon_ms
817
+ # (inbound -> first outbox flush) is H-latência and must not depend on
818
+ # INSIKA_TURN_TIMING. When the flag is off the clock measures ONLY that
819
+ # window (`breakdown: false`); the full prep/ttft/gen/total stays opt-in.
820
+ #
821
+ # A channel turn may already carry its clock: `SendMessage` stamped
822
+ # `:inbound` at 202 acceptance (before the debounce window and the
823
+ # SessionActor FIFO), so first_balloon_ms includes the wait the customer
824
+ # actually feels. A turn that reached here without one (boot resume,
825
+ # engine-initiated) falls back to allocating and stamps now — `mark` is
826
+ # first-write-wins, so a threaded clock is never re-stamped.
827
+ channel_turn = !channel_transport(task).nil?
828
+ timing ||= if TurnTiming.enabled? || channel_turn
829
+ TurnTiming.new(breakdown: TurnTiming.enabled?)
830
+ end
831
+ timing&.mark(:inbound) if channel_turn
715
832
  timing&.mark(:prep_start)
716
833
  state = build_turn_state(task, profile, resume_from)
717
834
  turn_timeout = turn_timeout_for(profile)
@@ -741,7 +858,7 @@ module Insika
741
858
  # · halt_reason set -> halt-as-FAILURE (the pre-existing contract).
742
859
  # · neither -> contract violation (short-circuit with no signal).
743
860
  if !terminal_ran && state.halt_response
744
- complete_with_halt(task, profile, state)
861
+ complete_with_halt(task, profile, state, timing)
745
862
  elsif state.halt_reason
746
863
  raise Insika::Error, "turn halted: #{state.halt_reason}"
747
864
  elsif !terminal_ran
@@ -749,7 +866,15 @@ module Insika
749
866
  end
750
867
 
751
868
  state # subject of the :task pair (after_task receives it; the caller discards)
752
- end.tap { |st| emit_guardrail_flags(task, st) }
869
+ end.tap do |st|
870
+ # flush the evidence ledger HERE — after the after_task
871
+ # hooks ran, because the :flag validator increments the ungrounded
872
+ # counter in after_task. The envelope's ids (stage 6) ride the same
873
+ # flush. AFTER persist_turn (already done), so a flush failure can
874
+ # never un-commit the turn (the ledger swallows store errors).
875
+ st.evidence_ledger&.flush!
876
+ emit_guardrail_flags(task, st)
877
+ end
753
878
  end
754
879
  rescue Async::TimeoutError
755
880
  raise Insika::TimeoutError.new("turn exceeded #{turn_timeout}s", stage: :turn)
@@ -762,6 +887,7 @@ module Insika
762
887
  state = TurnState.new(task: task, profile: profile, turn: turn,
763
888
  message: extract_message(task))
764
889
  state.tenant = memory_tenant(task) # WRITE-path memory scope (`remember`); =chat
890
+ stamp_customer_session(task, profile)
765
891
  state.turn_context = build_turn_context(task, profile, state) # data-tools' ctx.*
766
892
  state.resumed = !resume_from.nil? # EdgeLimiter: an admitted turn is never re-counted
767
893
  # resolved for the RUN, not per message, so what the turn accepts cannot
@@ -769,6 +895,12 @@ module Insika
769
895
  # SESSION turn: steering needs a session to arrive through, and resolving here for a
770
896
  # one-shot would make an unrelated turn fail on a queue key it can never use.
771
897
  state.queue_policy = task.session_id ? queue_policy(profile, task.session_id) : nil
898
+ # the session evidence ledger, built per turn. A nil
899
+ # session_id (one-shot) is fine — the ledger just never flushes (grounding
900
+ # on a one-shot is per-turn by definition). The envelope appends ids, the
901
+ # validator/enforcer read the union, stage 8 flushes.
902
+ state.evidence_ledger = Insika::EvidenceLedger.new(store: @session_store,
903
+ session_id: task.session_id)
772
904
  state
773
905
  end
774
906
 
@@ -786,6 +918,17 @@ module Insika
786
918
  # -> tool assembly, each followed by a mailbox drain at the boundary. Mutates
787
919
  # `state` in place (the TurnState possibly rewritten by before_task).
788
920
  def prepare_turn(task, profile, state, actor, resume_from)
921
+ # WS9 (saída): the CHANNEL's declared output media kinds — read on EVERY
922
+ # run (including resumes — it is plain config) so the ChatBuilder gate
923
+ # can re-wire the media tools; the transcription below stays guarded.
924
+ state.channel_capabilities = Insika::Media.channel_capabilities(
925
+ rebuild_command(task).payload["channel"]
926
+ )
927
+ # WS9: content parts -> turn. Runs BEFORE stage 2's context build so the
928
+ # transcribed voice text feeds the prompt; a resumed turn was already
929
+ # transcribed (never re-pay the STT call).
930
+ run_media_stage(task, state) unless state.resumed
931
+
789
932
  # stage 2: Context. The :prompt hook pair is wrapped INSIDE the
790
933
  # ContextBuilder#call — do NOT wrap here (a double-wrap would fire the hooks
791
934
  # twice). Hooks is the SAME instance injected into the Builder and here.
@@ -824,8 +967,12 @@ module Insika
824
967
 
825
968
  # one entry per turn in the ContextTraceStore — tokens per
826
969
  # category (the demodulized provider id), the tools-schema estimate and the
827
- # budget verdict. Counts and ids ONLY, never content. nil store = off; the
828
- # store itself rescues everything (the trace never breaks the turn).
970
+ # budget verdict. the entry also carries the prefix
971
+ # fingerprints + the invalidation_reason vs the previous turn, and the
972
+ # categories gain their cache layer. Counts and ids ONLY, never content.
973
+ # nil store = off; the store itself rescues everything (the trace never
974
+ # breaks the turn). -> the sanitized entry (parked on TurnState for the
975
+ # stage-8 cache stamp).
829
976
  def record_context_trace(task, state)
830
977
  return unless @context_trace_store && task.session_id
831
978
 
@@ -839,6 +986,10 @@ module Insika
839
986
  c[:tokens] += f.tokens || 0
840
987
  c[:fragments] += 1
841
988
  c[:pinned] += (f.tokens || 0) if f.pinned
989
+ # the category's cache layer (identity | :volatile —
990
+ # stamped by the Builder at production, C3).
991
+ layer = f.layer || :volatile
992
+ c[:layer] ||= layer
842
993
  # WHICH skills/tools the fragment carried and WHY — ids only, still
843
994
  # content-free. Without this the trace proves a turn injected N tokens of
844
995
  # skill but not which ones, and deterministic activation is unauditable
@@ -846,14 +997,88 @@ module Insika
846
997
  labels = Array(f.labels)
847
998
  (c[:labels] ||= []).concat(labels) unless labels.empty?
848
999
  end
849
- @context_trace_store.record(
850
- session_id: task.session_id,
851
- entry: { task_id: task.id, turn: state.turn, at: Time.now.utc.iso8601,
852
- cap: package.budget[:cap], used: package.budget[:used],
853
- evicted: package.budget[:evicted], categories: categories,
854
- tools: { count: state.allowed_tools.size,
855
- tokens: estimate_tools_tokens(state.allowed_tools) } }
856
- )
1000
+ # the prefix chain over the SYSTEM-placement fragments in
1001
+ # canonical (identity-first) render order + the tool-schema serialization.
1002
+ # The reason is computed against the PREVIOUS trace entry (D3): the first
1003
+ # category, in current chain order, whose bytes changed (or vanished).
1004
+ previous = previous_trace_entry(task, state.turn)
1005
+ fingerprints = Insika::PrefixFingerprint.compute(
1006
+ Array(package.fragments).select { |f| f.placement == :system },
1007
+ tool_serial: serialize_tools(state.allowed_tools))
1008
+ reason = Insika::PrefixFingerprint.invalidation_reason(
1009
+ fingerprints, previous && previous["fingerprints"])
1010
+
1011
+ entry = { task_id: task.id, turn: state.turn, at: Time.now.utc.iso8601,
1012
+ cap: package.budget[:cap], used: package.budget[:used],
1013
+ evicted: package.budget[:evicted], categories: categories,
1014
+ tools: { count: state.allowed_tools.size,
1015
+ tokens: estimate_tools_tokens(state.allowed_tools) },
1016
+ fingerprints: fingerprints,
1017
+ cache: { invalidation_reason: reason } }
1018
+ # Park the SANITIZED entry (string keys) — the stage-8 stamp merges into
1019
+ # it and re-records the same key; a raw entry would add a SECOND "cache"
1020
+ # key that sanitize would then ignore (the symbol one wins).
1021
+ state.context_trace_entry = @context_trace_store.record(session_id: task.session_id,
1022
+ entry: entry)
1023
+ end
1024
+
1025
+ # the previous turn's trace entry — the session list minus
1026
+ # THIS (task_id, turn) (an approval-resumed turn re-records over its own key
1027
+ # — never compare to self). `turn` is the turn being recorded, passed
1028
+ # explicitly. -> Hash | nil (first turn of the session).
1029
+ def previous_trace_entry(task, turn)
1030
+ @context_trace_store.for_session(task.session_id)
1031
+ .reject { |x| x["task_id"] == task.id && x["turn"] == turn }
1032
+ .last
1033
+ end
1034
+
1035
+ # the tool-schema yardstick — the SAME serialization the token
1036
+ # estimate uses (estimate_tools_tokens), so the fingerprint and the estimate
1037
+ # never disagree. The digest covers name + description + parameters.inspect,
1038
+ # approximating RubyLLM's rendering (honest in the doc: the reason's job is
1039
+ # the CONTEXT categories; the tool hash is a guard rail).
1040
+ def serialize_tools(tools)
1041
+ tools.map { |t| "#{t.name} #{t.description} #{t.parameters.inspect}" }.join(" ")
1042
+ end
1043
+
1044
+ # the stage-8 stamp — the usage (cached_tokens,
1045
+ # input_tokens) only exists after the provider answered, so a SECOND
1046
+ # UPSERT with the SAME (task_id, turn) merges the cache fields into the
1047
+ # entry parked at prepare_turn (the start-of-turn write stays: a turn that
1048
+ # dies mid-flight still shows its context on the Studio screen). The same
1049
+ # numbers append one entry to the agent's CacheSeriesStore (C6).
1050
+ def stamp_cache_hit(task, state)
1051
+ usage = state.usage || {}
1052
+ input = usage[:input_tokens].to_i
1053
+ cached = usage[:cached_tokens].to_i
1054
+ creation = usage[:cache_creation_tokens].to_i
1055
+ # A4: the billed prefix is input + cached + cache_creation. RubyLLM's
1056
+ # input_tokens is the FRESH input only — cached_tokens is disjoint, not a
1057
+ # subset — so dividing by input alone yields absurd numbers (22000/500 =
1058
+ # 4400%) and renders a full hit as "—" (fresh=0). The denominator is the
1059
+ # whole billed prompt; hit_pct is then always in [0,100].
1060
+ billed = input + cached + creation
1061
+ hit = billed.positive? ? ((cached * 100.0) / billed).round : nil
1062
+ reason = state.context_trace_entry&.dig("cache", "invalidation_reason")
1063
+
1064
+ # The trace merge needs the entry parked at prepare_turn (the UPSERT
1065
+ # replaces the same (task_id, turn)); a failed trace write leaves it nil
1066
+ # and the cache line simply never lands — never a turn failure.
1067
+ if @context_trace_store && state.context_trace_entry && task.session_id
1068
+ @context_trace_store.record(
1069
+ session_id: task.session_id,
1070
+ entry: state.context_trace_entry.merge("cache" => {
1071
+ "hit_pct" => hit, "cached_tokens" => cached, "prompt_tokens" => billed,
1072
+ "invalidation_reason" => reason }))
1073
+ end
1074
+
1075
+ # The per-agent series is INDEPENDENT of the trace store: a deployment
1076
+ # that wires the series without the trace (or whose trace write failed)
1077
+ # still records its cache-hit numbers — reason is simply nil then.
1078
+ @cache_series_store&.record(agent: state.profile.id, entry: {
1079
+ at: Time.now.utc.iso8601, turn: state.turn,
1080
+ hit_pct: hit, cached_tokens: cached, prompt_tokens: billed,
1081
+ invalidation_reason: reason })
857
1082
  end
858
1083
 
859
1084
  # Skill bodies that reached the prompt WITHOUT a tool call (`triggers:` or
@@ -904,7 +1129,13 @@ module Insika
904
1129
  # stage 5: assemble chat + check mailbox (send_message only; a workflow does
905
1130
  # not use the Insika chat — it orchestrates RubyLLM internally).
906
1131
  drain_and_maybe_suspend(task, actor)
907
- unless workflow_turn?(task)
1132
+ # WS4: intent routing, data-gated. Runs BEFORE the agent chat is assembled:
1133
+ # a route that delegates or ends :stuck completes the turn with no ask at
1134
+ # all (routed = true); a plain route is a label + event and the turn
1135
+ # proceeds. Skipped for workflows (no chat to route into) and resumed turns
1136
+ # (already admitted; re-classifying would re-pay the extra call).
1137
+ routed = !workflow_turn?(task) && !st.resumed ? attempt_route(task, profile, st) : false
1138
+ unless workflow_turn?(task) || routed
908
1139
  st.chat = create_chat(profile, st)
909
1140
  @chat_builder.assemble(st.chat, st, emit: ->(type, data) { emit(type, data, task: task) })
910
1141
  # R1: baseline = seeded-history size, before `ask` appends the turn.
@@ -915,21 +1146,57 @@ module Insika
915
1146
  st.output_filter = @content_filter_factory&.call(st)
916
1147
 
917
1148
  # stage 6: the turn's single agent interaction (send_message -> chat.ask;
918
- # trigger_workflow -> workflow.call). Returns the turn's final content.
919
- content = run_agent_stage(task, st, timing)
1149
+ # trigger_workflow -> workflow.call). A routed turn's content IS its route
1150
+ # action's answer (a delegate's reply or the stuck lead-in).
1151
+ content = routed ? st.response_content : run_agent_stage(task, st, timing)
920
1152
  st.response_content = content # after_task OutputValidator inspects this
921
1153
 
1154
+ # the model-visible record — what the provider received this
1155
+ # turn, captured at the boundary BEFORE stage 8 persists the checkpoint
1156
+ # (turn n's provider-visible stream == checkpoint(turn n+1).messages).
1157
+ # Skipped for workflows (they orchestrate RubyLLM inside the workflow
1158
+ # body — the engine cannot see those calls, stated in the conformance
1159
+ # scope) and absent-store runs (parity).
1160
+ record_model_visible(task, st) if !workflow_turn?(task) && st.chat
1161
+
1162
+ # the :enforce boundary — a CUT of the final content BEFORE
1163
+ # persistence/delivery (after_task fires too late to change what is
1164
+ # persisted). The cut text is what persists, delivers and terminates.
1165
+ if @grounding_enforcer
1166
+ content, st = @grounding_enforcer.call(task, st, content)
1167
+ st.response_content = content
1168
+ end
1169
+
922
1170
  # stage 8: Persistence (fixed order checkpoint->session->task). pure drain!
923
1171
  # (NEVER suspends at stage 8 — forbidden window): a :pause here arms the flag
924
1172
  # but is not honored (last stage); :cancel here still raises.
925
1173
  actor.drain!
926
- persist_turn(task, profile, st, content)
1174
+ persist_turn(task, profile, st, content, timing: timing)
1175
+
1176
+ # the cache-hit stamp — the usage exists only now. The
1177
+ # stamped entry is durable before anything is delivered (same slot as the
1178
+ # enforcer). Best-effort by construction (both stores rescue).
1179
+ stamp_cache_hit(task, st)
927
1180
 
928
1181
  # stage 9: Response. usage (tokens) captured at stage 6 travels in the
929
1182
  # terminal event -> /v1/responses usage + Telemetry (OTEL).
930
1183
  timing&.mark(:done)
931
1184
  data = { task_id: task.id, content: content, usage: st.usage }
1185
+ # WS4: the intent route rides the terminal additively (like outcome) — a
1186
+ # consumer aggregating by route does not need the stream.
1187
+ data[:route] = st.route.to_s if st.route
1188
+ # WS9: a turn whose message came from a VOICE note is marked — the
1189
+ # consumer's signal the person spoke (text was transcribed).
1190
+ data[:source] = :voice if st.message_source == :voice
1191
+ # WS9 (saída): media the agent GENERATED this turn (image/audio clips).
1192
+ # Additive sibling — the answer text stays text on purpose; the channel
1193
+ # consumes the parts next to it. Absent when nothing was generated.
1194
+ data[:output_parts] = st.output_parts if st.output_parts && !st.output_parts.empty?
932
1195
  data[:timing] = timing.to_h if timing # opt-in TTFB breakdown (INSIKA_TURN_TIMING)
1196
+ # best-effort persist of the same timing onto the task record —
1197
+ # the Studio task page reads it from there. A failure here must not re-fail
1198
+ # the turn (the task is already committed and the event already carries it).
1199
+ persist_turn_timing(task, timing)
933
1200
  # WS5: the agent declared it cannot proceed (signal_stuck). The turn still
934
1201
  # COMPLETES (its final message was published) — but the consumer must be able
935
1202
  # to act on that, so the contract carries it twice: a dedicated :turn_stuck
@@ -946,6 +1213,23 @@ module Insika
946
1213
  command_type(task).to_s == "trigger_workflow"
947
1214
  end
948
1215
 
1216
+ # the model-visible record of ONE ask — the chat at the
1217
+ # provider boundary (instructions + tool schemas + the message stream),
1218
+ # persisted under the checkpoint's turn number (turn n's stream ==
1219
+ # checkpoint(turn n+1).messages). Best-effort: the store rescues
1220
+ # everything, and the absent-store path is parity. `chat` defaults to the
1221
+ # turn's own chat; the routing classifier passes its own.
1222
+ def record_model_visible(task, st, chat = nil, part: "turn")
1223
+ return unless @model_visible_trace_store
1224
+
1225
+ c = chat || st.chat
1226
+ return unless c
1227
+
1228
+ @model_visible_trace_store.record(
1229
+ task_id: task.id, turn: st.turn + 1, part: part,
1230
+ payload: Insika::ModelVisible.capture(c))
1231
+ end
1232
+
949
1233
  # Stage 6: the single agent interaction. Returns the turn's final content.
950
1234
  def run_agent_stage(task, state, timing = nil)
951
1235
  if workflow_turn?(task)
@@ -985,7 +1269,13 @@ module Insika
985
1269
  # release the redactor's retained tail (a value that never completed into a
986
1270
  # match is emitted redacted-if-needed, not lost) before reading anything back.
987
1271
  output.flush
988
- state.usage = with_model_source(usage_of(response), state.model_selection) unless halted?(response)
1272
+ # MERGED, not overwritten: a WS4 routing call already banked its tokens in
1273
+ # state.usage before the ask — the classifier's cost must survive the ask
1274
+ # (it feeds the EdgeLimiter's ceiling/budget and the terminal usage).
1275
+ unless halted?(response)
1276
+ state.usage = merge_usage(with_model_source(usage_of(response), state.model_selection),
1277
+ state.usage)
1278
+ end
989
1279
 
990
1280
  # BOUNDARY BEFORE THE ANSWER GOES OUT. A cancel that arrived while the provider
991
1281
  # was working used to be observed at stage 8 — AFTER `:content` had already been
@@ -1075,6 +1365,228 @@ module Insika
1075
1365
 
1076
1366
  def ref_of(selection) = model_ref(selection)
1077
1367
 
1368
+ # --- WS9 media ------------------------------------------------------
1369
+ #
1370
+ # Content parts on the command -> a turn: audio parts are transcribed (the
1371
+ # text enters the message marked `source: :voice` — the consumer's signal
1372
+ # the person SPOKE), image parts become the ask's attachments (the model
1373
+ # sees them; the provider bills them — usage flows) and the first URL is
1374
+ # deposited as `ctx.image_url` for data/HTTP tools. The engine transports
1375
+ # media, never meaning: no speech/vision logic beyond the call itself.
1376
+ def run_media_stage(task, state)
1377
+ # a consumer that pre-transcribed voice text labels it `source: voice`;
1378
+ # the marker rides the turn even when there are no audio PARTS left.
1379
+ state.message_source = :voice if rebuild_command(task).payload["source"].to_s == "voice"
1380
+
1381
+ parts = Insika::Media.parts(rebuild_command(task).payload["parts"])
1382
+ return if parts.empty?
1383
+
1384
+ voice = Insika::Media.audio_parts(parts)
1385
+ if voice.any?
1386
+ text = voice.map { |p| media_transcribe(p.url) }.reject(&:empty?).join(" ")
1387
+ state.message = [state.message.to_s, text].reject(&:empty?).join("\n")
1388
+ state.message_source = :voice
1389
+ end
1390
+
1391
+ images = Insika::Media.image_parts(parts)
1392
+ if images.any?
1393
+ state.media_attachments = images.map { |p| media_attachment(p.url) }
1394
+ # First image URL for data tools (`{{ctx.image_url}}`) — photo analysis
1395
+ # outside the prompt. The model still sees the attachment; the tool
1396
+ # gets the original URL (its own egress applies when it fetches).
1397
+ state.turn_context = (state.turn_context || {}).merge(image_url: images.first.url)
1398
+ end
1399
+
1400
+ # A media-only turn (a voice note with no caption) is legitimate — the
1401
+ # surfaces admit it — but it must leave this stage with something to ask
1402
+ # about. Empty text AND no attachment means the parts carried nothing the
1403
+ # engine could use (a transcription that came back blank): fail loudly at
1404
+ # :media rather than ask the provider about nothing.
1405
+ return unless state.message.to_s.strip.empty? && state.media_attachments.nil?
1406
+
1407
+ raise Insika::MediaError, "the message parts produced no text and no attachment"
1408
+ end
1409
+
1410
+ # The STT seam: the injected transcriber (specs), else the default
1411
+ # (fetch + RubyLLM::Transcription — lazy require). A failed transcription
1412
+ # fails the turn loudly (MediaError -> :media): a voice message that was
1413
+ # not heard must not become a hallucinated one.
1414
+ def media_transcribe(url)
1415
+ transcriber = @media || (@default_transcriber ||= Insika::Media.default_transcriber(
1416
+ stt_model: Insika::EnvSchema.read("INSIKA_STT_MODEL"),
1417
+ stt_language: Insika::EnvSchema.read("INSIKA_STT_LANGUAGE")
1418
+ ))
1419
+ transcriber.call(url)
1420
+ end
1421
+
1422
+ # An image part -> the ask's attachment. RubyLLM required lazily (load-guard).
1423
+ #
1424
+ # The bytes come through OUR fetch (`Media.fetch_binary`), which is
1425
+ # egress-guarded — the URL is CONSUMER input, so a private/metadata target
1426
+ # fails the turn loudly at :media — and SIZE-CAPPED. Handing the raw URL to
1427
+ # `RubyLLM::Attachment` instead left the fetch to the gem, whose
1428
+ # `fetch_content` reads the whole response with no ceiling: a hostile URL
1429
+ # answering an endless body grows this process until it dies. An io-like
1430
+ # source (StringIO) is the branch of Attachment that takes bytes we already
1431
+ # hold; the provider then gets base64 rather than the URL, which every
1432
+ # vision provider accepts.
1433
+ def media_attachment(url)
1434
+ require "ruby_llm"
1435
+ require "stringio"
1436
+
1437
+ bytes = Insika::Media.fetch_binary(url, max_bytes: Insika::Media::MAX_IMAGE_BYTES)
1438
+ RubyLLM::Attachment.new(StringIO.new(bytes), filename: media_filename(url))
1439
+ end
1440
+
1441
+ # The URL's basename, for the attachment's mime sniff (".png" -> image/png;
1442
+ # a URL with no filename falls back to the content sniff RubyLLM does).
1443
+ def media_filename(url)
1444
+ name = File.basename(URI.parse(url).path.to_s)
1445
+ name.empty? ? nil : name
1446
+ rescue URI::InvalidURIError
1447
+ nil
1448
+ end
1449
+
1450
+ # --- WS4 intent routing --------------------------------------------
1451
+ #
1452
+ # The turn's message is classified into one of the profile's routes with a
1453
+ # CHEAP model (data-gated: no `routes` on the profile = byte-identical turn).
1454
+ # -> true when the route took over the turn (delegated / stuck — no ask
1455
+ # happens); false when the turn proceeds normally. Classification happens on
1456
+ # a fresh chat carrying ONLY the auto-generated route prompt (no identity,
1457
+ # no tools — it is a router, not the agent); its tokens ride the turn's
1458
+ # usage, so the trace, the token ceiling and the budget all see the cost.
1459
+ def attempt_route(task, profile, state)
1460
+ meta = Insika::Routing.normalize(profile.routes)
1461
+ return false unless meta
1462
+ return false unless route_model(meta, profile)
1463
+
1464
+ selection = route_selection(meta, profile)
1465
+ classification = classify_route(selection, meta, state.message, task, state)
1466
+ return false if classification.nil? # the classifier call failed — routing is additive
1467
+
1468
+ route = classification[:route]
1469
+ state.route = route
1470
+ # MERGED, not assigned: a WS9 transcription may already have banked its
1471
+ # tokens in state.usage (the classifier call must not erase them).
1472
+ state.usage = merge_usage(with_model_source(usage_of(classification[:response]), selection),
1473
+ state.usage)
1474
+ emit(:route_classified,
1475
+ { task_id: task.id, agent: profile.id.to_s, route: route.to_s,
1476
+ model: selection.model, usage: state.usage },
1477
+ task: task)
1478
+
1479
+ entry = meta[:entries].find { |e| e.name == route.to_s }
1480
+ apply_route_action(task, profile, state, entry)
1481
+ end
1482
+
1483
+ # The classifier call, its answer parsed back into a route.
1484
+ # -> { route:, response: } | nil (nil = the call failed — the turn proceeds
1485
+ # unrouted rather than paying a wrong label or dying for an additive step).
1486
+ def classify_route(selection, meta, message, task, st)
1487
+ response = route_ask(selection, Insika::Routing.classifier_prompt(meta), message, task, st)
1488
+ { route: Insika::Routing.parse(route_response_text(response), meta), response: response }
1489
+ rescue StandardError
1490
+ nil
1491
+ end
1492
+
1493
+ # The cheap classifier's model: routes["model"] wins, the agent's own model
1494
+ # otherwise. No model anywhere = no routing.
1495
+ def route_model(meta, profile)
1496
+ ref = meta[:model].to_s
1497
+ ref = profile.model.to_s if ref.empty?
1498
+ !ref.empty?
1499
+ end
1500
+
1501
+ def route_selection(meta, profile)
1502
+ ref = meta[:model].to_s
1503
+ ref = profile.model.to_s if ref.empty?
1504
+ parsed = parse_model_ref(ref) || {}
1505
+ provider = parsed[:provider].nil? ? profile.provider : parsed[:provider].to_sym
1506
+ Insika::ModelSelection.new(model: parsed[:model], provider: provider, source: :routing)
1507
+ end
1508
+
1509
+ # A fresh chat for the routing model with ONLY the generated prompt. RubyLLM
1510
+ # required lazily, exactly like create_chat (the load-guard holds).
1511
+ # the classifier is model-visible, so it is logged — the ONE
1512
+ # engine-internal ask the conformance spec adds a record for (part
1513
+ # "routing", same turn number as the answer ask).
1514
+ def route_ask(selection, prompt, message, task, st)
1515
+ require "ruby_llm"
1516
+ chat = (@llm || RubyLLM).chat(model: selection.model, provider: selection.provider,
1517
+ assume_model_exists: selection.assume_model_exists?)
1518
+ chat.with_instructions(prompt) if chat.respond_to?(:with_instructions)
1519
+ response = chat.ask(message.to_s)
1520
+ record_model_visible(task, st, chat, part: "routing")
1521
+ response
1522
+ end
1523
+
1524
+ def route_response_text(response)
1525
+ response.respond_to?(:content) ? response.content.to_s : response.to_s
1526
+ end
1527
+
1528
+ # The route's config decides the turn's fate: nothing (a label), a DELEGATE
1529
+ # (an existing agent answers; its reply IS the turn's), or STUCK (WS5 — the
1530
+ # turn ends with the stuck outcome; the consumer interprets it).
1531
+ def apply_route_action(task, profile, state, entry)
1532
+ return false unless entry
1533
+
1534
+ if entry.delegate && !entry.delegate.empty?
1535
+ delegate_route(profile, state, entry.delegate)
1536
+ true
1537
+ elsif entry.stuck
1538
+ message = entry.message.empty? ? entry.description : entry.message
1539
+ state.stuck_outcome = { reason: "route:#{state.route}", message: message }
1540
+ state.response_content = message
1541
+ true
1542
+ else
1543
+ false
1544
+ end
1545
+ end
1546
+
1547
+ # WS4 delegate action: the route names an existing agent — the turn is
1548
+ # handed to it (the sync subagent machinery) and the child's answer IS the
1549
+ # parent's answer. A missing agent or a failed child fails the turn: never
1550
+ # fabricate the customer's reply.
1551
+ #
1552
+ # The depth comes from the PARENT's turn context, +1, and is capped here —
1553
+ # this path does not go through `plan_subagent` (a route has no subagents
1554
+ # allowlist to check against), so a hardcoded depth of 1 made an A -> B -> A
1555
+ # route pair a loop with no floor: every hop reclassifies (a paid ask) and
1556
+ # spawns another child, forever.
1557
+ def delegate_route(profile, state, agent_id)
1558
+ child = @profiles[agent_id.to_s]
1559
+ raise Insika::RoutingError, "route delegate agent '#{agent_id}' not configured" if child.nil?
1560
+
1561
+ depth = (state.turn_context&.dig(:delegation_depth) || 0) + 1
1562
+ cap = SubagentGraph.depth_cap
1563
+ if depth > cap
1564
+ raise Insika::RoutingError,
1565
+ "routed delegate '#{agent_id}' at depth #{depth} exceeds cap #{cap}"
1566
+ end
1567
+
1568
+ result = spawn_and_await_child(child, state.message, depth, state)
1569
+ if result[:error]
1570
+ raise Insika::RoutingError, "routed delegate '#{agent_id}' failed: #{result[:error]}"
1571
+ end
1572
+
1573
+ state.response_content = result[:text].to_s
1574
+ end
1575
+
1576
+ # The WS4 classifier's tokens, summed over the ask's (a call that reported no
1577
+ # usage contributes nothing). The turn's own model/source win for attribution
1578
+ # — the routing model's identity lives on the :route_classified event.
1579
+ def merge_usage(main, extra)
1580
+ return main || extra if main.nil? || extra.nil?
1581
+
1582
+ Insika::Routing::TOKEN_FIELDS.each_with_object(main.dup) do |k, acc|
1583
+ next if extra[k].nil?
1584
+ next unless main.key?(k) || extra[k].to_i.positive?
1585
+
1586
+ acc[k] = main[k].to_i + extra[k].to_i
1587
+ end
1588
+ end
1589
+
1078
1590
  # "provider/model" for any selection duck (ModelSelection | { model:, provider: }).
1079
1591
  def model_ref(selection)
1080
1592
  model = selection.respond_to?(:model) ? selection.model.to_s : selection[:model].to_s
@@ -1134,7 +1646,7 @@ module Insika
1134
1646
  def ask_on(task, state, chat, output, timing)
1135
1647
  public_thinking = state.profile.stream_public?(:thinking)
1136
1648
  ttft_sent = false
1137
- chat.ask(state.message) do |chunk|
1649
+ each_chunk = lambda do |chunk|
1138
1650
  emit_thinking(chunk, task, public: public_thinking)
1139
1651
  next unless chunk.content
1140
1652
 
@@ -1145,19 +1657,29 @@ module Insika
1145
1657
  end
1146
1658
  output.push(chunk.content)
1147
1659
  end
1660
+ # WS9: image parts ride the ask as attachments (only then — a chat whose
1661
+ # ask has no `with:` keeps working, and the plain path is byte-identical).
1662
+ # An image with no caption asks with NIL, not "": an empty text part is a
1663
+ # thing some providers refuse, and nil is how RubyLLM says "attachments
1664
+ # only".
1665
+ if state.media_attachments
1666
+ text = state.message.to_s.empty? ? nil : state.message
1667
+ chat.ask(text, with: state.media_attachments, &each_chunk)
1668
+ else
1669
+ chat.ask(state.message, &each_chunk)
1670
+ end
1148
1671
  end
1149
1672
 
1150
1673
  # The provider's TTFB as a live event (data: ttft_ms) — only under
1151
- # INSIKA_TURN_TIMING, so absent by default (parity).
1674
+ # INSIKA_TURN_TIMING, so absent by default (parity). Rides the SINGLE
1675
+ # emitter: a hand-built meta here lacked `tenant`, and a tenant-scoped
1676
+ # /v1/events subscription is fail-closed on it — the tenant's own TTFB was
1677
+ # invisible to the tenant.
1152
1678
  def emit_ttft(task, timing)
1153
1679
  ttft = timing.to_h[:ttft_ms]
1154
1680
  return if ttft.nil?
1155
1681
 
1156
- @event_stream.emit(Insika::Event.new(
1157
- type: :ttft, data: { ttft_ms: ttft },
1158
- meta: { task_id: task.id, session_id: task.session_id,
1159
- at: Time.now.utc.iso8601 }
1160
- ))
1682
+ emit(:ttft, { ttft_ms: ttft }, task: task)
1161
1683
  rescue StandardError
1162
1684
  nil
1163
1685
  end
@@ -1321,9 +1843,11 @@ module Insika
1321
1843
  vars["history"] = hist if hist
1322
1844
  # The single type is Insika::ContextRequest (Data); the explicit `history`
1323
1845
  # travels in vars["history"] (Session provider convention), not in a field
1324
- # of its own.
1846
+ # of its own. `memory_scope` is the WS8 customer cell (nil = the providers
1847
+ # fall back to tenant || session, today's behavior).
1325
1848
  ContextRequest.new(profile: profile, message: state.message, session: session,
1326
- checkpoint: resume_from, tenant: command_tenant(task), vars: vars)
1849
+ checkpoint: resume_from, tenant: command_tenant(task), vars: vars,
1850
+ memory_scope: memory_tenant(task))
1327
1851
  end
1328
1852
 
1329
1853
  # task_started payload. Carries the EXPLICIT command tenant so
@@ -1357,8 +1881,57 @@ module Insika
1357
1881
  # Symmetric to the READ path (Memory provider). One-shot with no tenant -> nil
1358
1882
  # (_default). It is NOT the <request_context> tenant (that follows
1359
1883
  # command_tenant, prompt parity) — only the memory read/write scope.
1884
+ #
1885
+ # WS8: a request carrying a CUSTOMER moves the scope to the customer cell —
1886
+ # "[tenant:]customer" when a tenant is present, the bare customer otherwise
1887
+ # (never _default — a tagged customer must never land in the shared cell).
1888
+ # Per-customer memory is the 360 view; per-tenant was the leak.
1889
+ #
1890
+ # the SESSION fallback is MARKED ("chat:<session id>" -> cell
1891
+ # "memory:chat:<session id>"), never a bare cell — a bare "memory:<id>" is
1892
+ # indistinguishable from a single-tenant customer ref, and the Studio drill
1893
+ # must not list conversations as customers with a Forget button.
1360
1894
  def memory_tenant(task)
1361
- command_tenant(task) || task.session_id
1895
+ customer = command_customer(task)
1896
+ return command_tenant(task) || session_scope(task.session_id) if customer.nil?
1897
+
1898
+ [command_tenant(task), customer].compact.join(":")
1899
+ end
1900
+
1901
+ # The marked per-session scope : "chat:<session id>" -> cell
1902
+ # "memory:chat:<session id>". nil for a one-shot turn (no session) — the
1903
+ # MemoryStore applies _default.
1904
+ def session_scope(session_id)
1905
+ return nil if session_id.nil?
1906
+
1907
+ "#{MemoryStore::SESSION_TAG}:#{session_id}"
1908
+ end
1909
+
1910
+ # WS8 + : stamp the customer (WS8 — the `forget_customer`
1911
+ # purge finds the customer's sessions through this var) AND the agent (the
1912
+ # distillation engine resolves each session's pack through it) on the
1913
+ # session ONCE (idempotent). A session that does not exist yet (no
1914
+ # session_id on the turn) is skipped; a look-up failure never breaks the
1915
+ # turn.
1916
+ def stamp_customer_session(task, profile)
1917
+ customer = command_customer(task)
1918
+ return if customer.nil? || task.session_id.nil?
1919
+
1920
+ session = @session_store&.find(task.session_id)
1921
+ return if session.nil? || !Coercion.presence(session.vars["customer"]).nil?
1922
+
1923
+ @session_store.update_vars(task.session_id,
1924
+ "customer" => customer, "agent" => profile.id)
1925
+ rescue Insika::NotFoundError, ArgumentError
1926
+ nil
1927
+ end
1928
+
1929
+ # The optional customer_key on the command payload (WS8): a String identifying
1930
+ # the person the conversation belongs to — the memory scope's customer half
1931
+ # and the handle `forget_customer` purges by. nil = untagged conversation
1932
+ # (memory stays per-tenant/per-chat, byte-identical to before).
1933
+ def command_customer(task)
1934
+ Coercion.presence(rebuild_command(task).payload["customer"])
1362
1935
  end
1363
1936
 
1364
1937
  # Turn context: the ids the data-tools resolve via
@@ -1373,7 +1946,10 @@ module Insika
1373
1946
  {
1374
1947
  chat_id: task.session_id,
1375
1948
  agent_id: profile.id,
1376
- tenant: state.tenant, # already = command_tenant || session_id (memory_tenant)
1949
+ # the DATA-TOOL header tenant stays the merchant (or the chat), even when
1950
+ # the memory scope carries a customer — the backend identifies the store,
1951
+ # not the shopper (WS8 keeps the two scopes separate).
1952
+ tenant: command_tenant(task) || task.session_id,
1377
1953
  store_id: profile.store_id,
1378
1954
  # current delegation depth (0 for a top-level turn). Carried in
1379
1955
  # the child command's payload by run_subagent; read here so the child's OWN
@@ -1701,7 +2277,7 @@ module Insika
1701
2277
  # order mirrors a real turn so both the /v1/responses consumer (which reads the
1702
2278
  # text off :content deltas) and the Studio viewer render it: audit -> safe text
1703
2279
  # -> persist -> terminal.
1704
- def complete_with_halt(task, profile, state)
2280
+ def complete_with_halt(task, profile, state, timing = nil)
1705
2281
  content = state.halt_response.to_s
1706
2282
  state.response_content = content
1707
2283
 
@@ -1722,23 +2298,44 @@ module Insika
1722
2298
  # reading the engine's own canned text (the `safe_reply` finding exists exactly
1723
2299
  # because that text is otherwise indistinguishable in the transcript).
1724
2300
  persist_turn(task, profile, state, content, reply_origin: MessageOrigin::ENGINE,
1725
- session: state.guardrail_block&.[](:source) != "edge")
1726
- emit(:task_completed, { task_id: task.id, content: content, usage: state.usage }, task: task)
2301
+ session: state.guardrail_block&.[](:source) != "edge", timing: timing)
2302
+ data = { task_id: task.id, content: content, usage: state.usage }
2303
+ data[:timing] = timing.to_h if timing # a channel halt still measured
2304
+ persist_turn_timing(task, timing)
2305
+ emit(:task_completed, data, task: task)
1727
2306
  end
1728
2307
 
1729
- # Emits one :guardrail_flagged per flag the OutputValidator appended in
1730
- # after_task (audit only the turn already completed). Reads a plain Array off
1731
- # the state, keeping the Executor decoupled from Safety.
1732
- def emit_guardrail_flags(task, state)
1733
- return unless state.respond_to?(:guardrail_flags)
2308
+ # Best-effort write of the turn's timing onto the task record .
2309
+ # The record gains `timing` once, when the turn completes; a store failure
2310
+ # here is swallowed the turn is already committed and the event already
2311
+ # carries the number.
2312
+ def persist_turn_timing(task, timing)
2313
+ return unless timing
1734
2314
 
1735
- Array(state.guardrail_flags).each do |flag|
1736
- emit(:guardrail_flagged, {
1737
- task_id: task.id, category: flag[:category], source: flag[:source], detail: flag[:detail]
1738
- }, task: task)
1739
- end
2315
+ hash = timing.to_h
2316
+ return if hash.empty?
2317
+
2318
+ @task_store.record_timing(task.id, hash)
2319
+ rescue Insika::Error
2320
+ nil
1740
2321
  end
1741
2322
 
2323
+ # Emits one :guardrail_flagged per flag the OutputValidator appended in
2324
+ # after_task (audit only — the turn already completed). Reads a plain Array off
2325
+ # the state, keeping the Executor decoupled from Safety. an
2326
+ # :enforce cut rides the SAME event with `action: "cut"` so the audit can
2327
+ # distinguish a cut from a flag.
2328
+ def emit_guardrail_flags(task, state)
2329
+ return unless state.respond_to?(:guardrail_flags)
2330
+
2331
+ Array(state.guardrail_flags).each do |flag|
2332
+ data = { task_id: task.id, category: flag[:category], source: flag[:source],
2333
+ detail: flag[:detail] }
2334
+ data[:action] = flag[:action] if flag[:action]
2335
+ emit(:guardrail_flagged, data, task: task)
2336
+ end
2337
+ end
2338
+
1742
2339
  # Stage 8: FIXED order checkpoint -> session -> task. If it crashes
1743
2340
  # between writes, the worst case is a new checkpoint with the task :running ->
1744
2341
  # Recovery re-executes the already-saved turn (safe thanks to the side-effect
@@ -1755,7 +2352,7 @@ module Insika
1755
2352
  # So a long session legitimately has a Checkpoint SHORTER than the Session:
1756
2353
  # that is not drift to reconcile — it is the point. Do NOT "fix" the checkpoint
1757
2354
  # to carry the full history (it would defeat the budget) nor evict the session.
1758
- def persist_turn(task, profile, state, content, session: true, reply_origin: nil)
2355
+ def persist_turn(task, profile, state, content, session: true, reply_origin: nil, timing: nil)
1759
2356
  new_messages = turn_transcript(state, content, origin: command_origin(task), reply_origin: reply_origin)
1760
2357
  transcript = flatten_history(state.context.history) + new_messages
1761
2358
 
@@ -1793,7 +2390,7 @@ module Insika
1793
2390
  # if this turn CAME IN through a Shape B channel, its answer
1794
2391
  # has to travel out of band. Same terminal hook, next door to the delegation
1795
2392
  # one, for the same reason: it fires for a fresh turn and a recovered one.
1796
- finalize_channel_delivery(task, content)
2393
+ finalize_channel_delivery(task, content, state, timing)
1797
2394
  end
1798
2395
 
1799
2396
  # Records the answer in the outbox and dispatches it. The discriminator is the
@@ -1810,31 +2407,44 @@ module Insika
1810
2407
  #
1811
2408
  # Best-effort: the turn is already committed and durable, and a delivery problem
1812
2409
  # must never re-fail it.
1813
- def finalize_channel_delivery(task, content)
2410
+ #
2411
+ # a PROGRESSIVE channel gets the answer split into balloons —
2412
+ # N outbox rows, dispatched in index order (dispatch_chain). `:at_end` is the
2413
+ # single whole-answer row, byte-identical to today.
2414
+ def finalize_channel_delivery(task, content, state, timing = nil)
1814
2415
  return unless @channel_delivery
1815
2416
 
1816
2417
  channel_id = channel_transport(task)
1817
2418
  return unless channel_id
1818
2419
 
1819
- delivery = @channel_delivery.record(task: task, channel_id: channel_id, content: content)
1820
- return unless delivery
2420
+ # the hoarded evidence attachments ride the channel delivery
2421
+ # (additive outbox payload key — the channel contract widens, nothing breaks).
2422
+ attachments = state.respond_to?(:evidence_attachments) ? state.evidence_attachments : nil
2423
+ deliveries = @channel_delivery.record_balloons(
2424
+ task: task, channel_id: channel_id, content: content,
2425
+ progressive: @channel_delivery.progressive?(channel_id),
2426
+ attachments: attachments
2427
+ )
2428
+ return if deliveries.empty?
1821
2429
 
1822
- dispatch_delivery(delivery.id)
2430
+ timing&.mark(:first_balloon) # C5: inbound -> first outbox row, first-write-wins
2431
+ dispatch_chain(deliveries.map(&:id))
1823
2432
  rescue Insika::Error
1824
2433
  nil
1825
2434
  end
1826
2435
 
1827
- # The POST goes out on the SUPERVISOR, never on the turn's fiber: a bounded
1828
- # retry against a third party would otherwise hold the session's FIFO the
1829
- # customer's next message would wait on their previous answer's delivery.
1830
- # Non-serving (boot sweep, specs) delivers inline, where waiting is what the
1831
- # caller wants.
1832
- def dispatch_delivery(delivery_id)
1833
- return @channel_delivery.deliver(delivery_id) unless @supervised
2436
+ # ONE supervisor fiber for the whole chain. Sequential deliver
2437
+ # calls, so balloon N+1 cannot overtake balloon N on the wire. Still off the
2438
+ # session's FIFO — the customer's next message does not wait on this turn's
2439
+ # outbound. Non-serving (boot sweep, specs) delivers inline, where waiting is
2440
+ # what the caller wants.
2441
+ def dispatch_chain(ids)
2442
+ run = lambda { ids.each { |id| @channel_delivery.deliver(id) } }
2443
+ return run.call unless @supervised
1834
2444
 
1835
2445
  turn_parent.async do |t|
1836
- t.annotate("outbox:#{delivery_id}")
1837
- @channel_delivery.deliver(delivery_id)
2446
+ t.annotate("outbox:#{ids.first}")
2447
+ run.call
1838
2448
  end
1839
2449
  end
1840
2450
 
@@ -1946,6 +2556,13 @@ module Insika
1946
2556
  require_relative "tools/subagent"
1947
2557
  require_relative "tools/subagents"
1948
2558
  require_relative "tools/stuck_signal"
2559
+ require_relative "tools/generate_image"
2560
+ require_relative "tools/tts"
2561
+ require_relative "tools/update_briefing"
2562
+ # the schedule/cancel_followup builtins — lazy, same
2563
+ # boundary (the ChatBuilder wires them only when a profile declares
2564
+ # followup AND the stores are present).
2565
+ require_relative "tools/schedule_followup"
1949
2566
  # v2 resolution: Chat pin > Agent model > platform default, model_policy
1950
2567
  # enforced, fallback chain resolved. Kept on the state for telemetry (usage).
1951
2568
  selection = @model_resolver.resolve(profile: profile, session: state.session)