insika 0.3.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (204) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +296 -0
  3. data/README.md +48 -12
  4. data/bin/insika +725 -0
  5. data/bin/insika-router +87 -0
  6. data/docs/AGENTS.md +116 -406
  7. data/docs/API.md +5 -5
  8. data/docs/ARCHITECTURE.md +3 -2
  9. data/docs/ARTIFACTS.md +137 -0
  10. data/docs/BENCHMARK.md +2 -2
  11. data/docs/CHANNELS.md +14 -14
  12. data/docs/CONTEXT.md +63 -19
  13. data/docs/DEMO.md +80 -0
  14. data/docs/DEPLOY.md +87 -10
  15. data/docs/EMBEDDING.md +1 -1
  16. data/docs/EVALS.md +128 -3
  17. data/docs/FACTS.md +3 -3
  18. data/docs/HARVEST.md +5 -6
  19. data/docs/KNOWLEDGE.md +290 -0
  20. data/docs/LOADTEST.md +17 -29
  21. data/docs/MEDIA.md +128 -0
  22. data/docs/OBSERVABILITY.md +46 -12
  23. data/docs/OUTCOMES.md +137 -0
  24. data/docs/PLUGINS.md +51 -6
  25. data/docs/POLICY.md +222 -0
  26. data/docs/REFINEMENT.md +14 -9
  27. data/docs/RELEASING.md +4 -4
  28. data/docs/ROUTER.md +213 -0
  29. data/docs/RUNNING-LOCAL.md +5 -5
  30. data/docs/SCHEDULING.md +121 -0
  31. data/docs/SECURITY.md +23 -7
  32. data/docs/SKILLS.md +11 -2
  33. data/docs/SOAK.md +3 -3
  34. data/docs/TEMPLATES.md +134 -0
  35. data/docs/TOOLS.md +176 -27
  36. data/docs/WHY.md +1 -1
  37. data/docs/WORKFLOWS.md +2 -2
  38. data/docs/_includes/head_custom.html +5 -0
  39. data/docs/_includes/title.html +13 -0
  40. data/docs/_sass/color_schemes/insika.scss +32 -0
  41. data/docs/_sass/custom/custom.scss +199 -0
  42. data/docs/_sass/custom/setup.scss +26 -0
  43. data/docs/assets/img/favicon.svg +7 -0
  44. data/docs/assets/img/insika-mark.svg +7 -0
  45. data/docs/core-concepts.md +21 -0
  46. data/docs/domain.md +4 -4
  47. data/docs/improve.md +20 -0
  48. data/docs/index.md +8 -5
  49. data/docs/integrate.md +20 -0
  50. data/docs/operate.md +13 -6
  51. data/docs/prompts/ADD-TOOL.md +118 -0
  52. data/docs/prompts/DIAGNOSE-TURN.md +65 -0
  53. data/docs/prompts/GO-LIVE.md +138 -0
  54. data/docs/prompts/RUN-EXAMPLES.md +70 -0
  55. data/docs/reference.md +19 -0
  56. data/docs/ship.md +10 -2
  57. data/docs/start-here.md +18 -0
  58. data/lib/insika/agent_profile.rb +99 -17
  59. data/lib/insika/artifact_signing.rb +82 -0
  60. data/lib/insika/artifact_store.rb +160 -0
  61. data/lib/insika/channel_delivery.rb +1 -1
  62. data/lib/insika/chat_builder.rb +50 -19
  63. data/lib/insika/commands/agent_payload.rb +2 -2
  64. data/lib/insika/commands/backfill_knowledge.rb +145 -0
  65. data/lib/insika/commands/delete_artifact.rb +35 -0
  66. data/lib/insika/commands/delete_concept.rb +34 -0
  67. data/lib/insika/commands/delete_mcp.rb +6 -2
  68. data/lib/insika/commands/delete_tenant_data.rb +15 -3
  69. data/lib/insika/commands/gate_refinement.rb +1 -1
  70. data/lib/insika/commands/refresh_mcp_tools.rb +47 -0
  71. data/lib/insika/commands/restore_concept.rb +34 -0
  72. data/lib/insika/commands/seed_demo_data.rb +31 -0
  73. data/lib/insika/commands/upsert_mcp.rb +6 -3
  74. data/lib/insika/commands/write_concept.rb +57 -0
  75. data/lib/insika/compaction.rb +196 -0
  76. data/lib/insika/context/builder.rb +6 -2
  77. data/lib/insika/context/fragment.rb +4 -1
  78. data/lib/insika/context/priority.rb +8 -0
  79. data/lib/insika/context/providers/briefing.rb +53 -24
  80. data/lib/insika/context/providers/knowledge.rb +108 -0
  81. data/lib/insika/context/providers/prompt.rb +30 -24
  82. data/lib/insika/context/providers/session.rb +46 -10
  83. data/lib/insika/context_trace_store.rb +11 -1
  84. data/lib/insika/cron.rb +189 -0
  85. data/lib/insika/demo/agent_attrs.rb +43 -0
  86. data/lib/insika/demo/golden_cases.rb +81 -0
  87. data/lib/insika/demo/seeder.rb +336 -0
  88. data/lib/insika/doctor.rb +280 -17
  89. data/lib/insika/dsl/definition.rb +3 -2
  90. data/lib/insika/dsl/runtime.rb +64 -79
  91. data/lib/insika/dsl/server_boot.rb +23 -1
  92. data/lib/insika/dsl/system.rb +10 -2
  93. data/lib/insika/dsl.rb +103 -2
  94. data/lib/insika/env_schema.rb +21 -7
  95. data/lib/insika/evals/golden.rb +41 -4
  96. data/lib/insika/evals/judge.rb +47 -2
  97. data/lib/insika/evals/pairwise.rb +11 -0
  98. data/lib/insika/evals/persona.rb +98 -0
  99. data/lib/insika/evals/runner.rb +9 -0
  100. data/lib/insika/evals/simulator.rb +225 -0
  101. data/lib/insika/evals/transport.rb +84 -2
  102. data/lib/insika/event_stream.rb +10 -0
  103. data/lib/insika/executor.rb +295 -55
  104. data/lib/insika/followup_policy.rb +2 -25
  105. data/lib/insika/golden_store.rb +16 -1
  106. data/lib/insika/grounding/matcher.rb +1 -1
  107. data/lib/insika/knowledge.rb +680 -0
  108. data/lib/insika/knowledge_store.rb +140 -0
  109. data/lib/insika/loop_detector.rb +5 -34
  110. data/lib/insika/mcp_client.rb +94 -0
  111. data/lib/insika/mcp_json.rb +74 -0
  112. data/lib/insika/mcp_live_tool.rb +43 -0
  113. data/lib/insika/mcp_store.rb +98 -26
  114. data/lib/insika/mcp_tool_ingestor.rb +30 -8
  115. data/lib/insika/mcp_tool_registry.rb +100 -0
  116. data/lib/insika/media.rb +115 -31
  117. data/lib/insika/message_origin.rb +1 -1
  118. data/lib/insika/middleware.rb +9 -0
  119. data/lib/insika/onboarding.rb +17 -1
  120. data/lib/insika/outcome_store.rb +1 -1
  121. data/lib/insika/overlay_tool_registry.rb +37 -17
  122. data/lib/insika/packaging.rb +2 -2
  123. data/lib/insika/profile_source.rb +15 -1
  124. data/lib/insika/prompt_catalog.rb +10 -0
  125. data/lib/insika/retention.rb +36 -1
  126. data/lib/insika/router/app.rb +157 -0
  127. data/lib/insika/router/backend_pool.rb +98 -0
  128. data/lib/insika/router/hash_ring.rb +55 -0
  129. data/lib/insika/router/proxy_body.rb +34 -0
  130. data/lib/insika/router/session_key.rb +54 -0
  131. data/lib/insika/router.rb +18 -0
  132. data/lib/insika/schedule.rb +177 -0
  133. data/lib/insika/schedule_engine.rb +314 -0
  134. data/lib/insika/schedule_store.rb +208 -0
  135. data/lib/insika/server/app.rb +105 -15
  136. data/lib/insika/server/rack_app.rb +5 -1
  137. data/lib/insika/server/responses.rb +5 -5
  138. data/lib/insika/session_store.rb +34 -4
  139. data/lib/insika/settings_store.rb +8 -1
  140. data/lib/insika/skill_catalog.rb +12 -0
  141. data/lib/insika/soak/runner.rb +4 -4
  142. data/lib/insika/steer_injector.rb +21 -10
  143. data/lib/insika/studio/app.rb +591 -47
  144. data/lib/insika/studio/assets/dist/application.css +1 -1
  145. data/lib/insika/studio/assets/dist/application.js +21 -21
  146. data/lib/insika/studio/forms.rb +57 -5
  147. data/lib/insika/studio/nav_icons.rb +14 -1
  148. data/lib/insika/studio/views/_agent_tab_cache.erb +25 -0
  149. data/lib/insika/studio/views/_agent_tab_config.erb +514 -0
  150. data/lib/insika/studio/views/_agent_tab_history.erb +24 -0
  151. data/lib/insika/studio/views/_agent_tab_loops.erb +54 -0
  152. data/lib/insika/studio/views/_agent_tab_memory.erb +51 -0
  153. data/lib/insika/studio/views/_agent_tab_outcomes.erb +31 -0
  154. data/lib/insika/studio/views/_agent_tab_prompts.erb +108 -0
  155. data/lib/insika/studio/views/_agent_tab_skills.erb +38 -0
  156. data/lib/insika/studio/views/_agents_master.erb +44 -0
  157. data/lib/insika/studio/views/_message.erb +49 -32
  158. data/lib/insika/studio/views/agent_detail.erb +61 -820
  159. data/lib/insika/studio/views/agents.erb +70 -57
  160. data/lib/insika/studio/views/artifact.erb +23 -0
  161. data/lib/insika/studio/views/artifacts.erb +59 -0
  162. data/lib/insika/studio/views/evals.erb +2 -2
  163. data/lib/insika/studio/views/facts.erb +1 -1
  164. data/lib/insika/studio/views/funnel.erb +1 -1
  165. data/lib/insika/studio/views/home.erb +106 -67
  166. data/lib/insika/studio/views/knowledge.erb +123 -0
  167. data/lib/insika/studio/views/layout.erb +14 -11
  168. data/lib/insika/studio/views/mcp.erb +174 -80
  169. data/lib/insika/studio/views/session.erb +231 -177
  170. data/lib/insika/studio/views/settings.erb +50 -1
  171. data/lib/insika/studio/views/skills.erb +1 -1
  172. data/lib/insika/studio/views/tools.erb +24 -9
  173. data/lib/insika/telemetry/recorder.rb +49 -1
  174. data/lib/insika/templates/browser-agent/README.md +36 -0
  175. data/lib/insika/templates/browser-agent/agent.rb +49 -0
  176. data/lib/insika/templates/daily-digest/README.md +47 -0
  177. data/lib/insika/templates/daily-digest/agent.rb +77 -0
  178. data/lib/insika/templates/repo-explorer/README.md +36 -0
  179. data/lib/insika/templates/repo-explorer/agent.rb +45 -0
  180. data/lib/insika/templates/research-analyst/README.md +26 -0
  181. data/lib/insika/templates/research-analyst/agent.rb +68 -0
  182. data/lib/insika/templates/review-panel/README.md +20 -0
  183. data/lib/insika/templates/review-panel/agent.rb +50 -0
  184. data/lib/insika/templates/travel-planner/README.md +35 -0
  185. data/lib/insika/templates/travel-planner/agent.rb +87 -0
  186. data/lib/insika/templates.rb +112 -0
  187. data/lib/insika/tick.rb +24 -12
  188. data/lib/insika/timezone.rb +45 -0
  189. data/lib/insika/tool_batch.rb +67 -0
  190. data/lib/insika/tool_usage_report.rb +162 -0
  191. data/lib/insika/tools/generate_image.rb +52 -7
  192. data/lib/insika/tools/load_knowledge.rb +74 -0
  193. data/lib/insika/tools/run_persona_eval.rb +328 -0
  194. data/lib/insika/tools/save_artifact.rb +95 -0
  195. data/lib/insika/turn_budget.rb +91 -0
  196. data/lib/insika/turn_output.rb +1 -1
  197. data/lib/insika/turn_state.rb +15 -4
  198. data/lib/insika/version.rb +1 -1
  199. data/lib/insika/wiring/graph.rb +184 -12
  200. data/lib/insika/wiring/graph_chat.rb +102 -0
  201. data/lib/insika.rb +64 -0
  202. metadata +109 -5
  203. data/docs/build.md +0 -14
  204. data/docs/understand.md +0 -10
@@ -22,7 +22,8 @@ module Insika
22
22
  delegation_store: nil, channel_delivery: nil, llm: nil,
23
23
  context_trace_store: nil, reliability: nil, media: nil, media_output: nil,
24
24
  grounding_enforcer: nil, cache_series_store: nil,
25
- contact_store: nil, followup_store: nil, model_visible_trace_store: nil)
25
+ contact_store: nil, followup_store: nil, model_visible_trace_store: nil,
26
+ knowledge_store: nil)
26
27
  @context_builder = context_builder
27
28
  @policy_engine = policy_engine
28
29
  @middleware = middleware
@@ -98,6 +99,10 @@ module Insika
98
99
  # the platform layer of the queue policy (nil = per-agent and
99
100
  # defaults only, which is `followup` with no window — today's behavior).
100
101
  @settings_store = settings_store
102
+ # LEARNED concepts, extracted from a turn's transcript after it
103
+ # completes. nil = the loop is off (parity — every write path below is
104
+ # skipped, zero allocations). Gated per-agent by `profile.knowledge`.
105
+ @knowledge_store = knowledge_store
101
106
  # RubyLLM glue (stages 5-7): chat assembly delegated to ChatBuilder. Its
102
107
  # optional deps tool_catalog (Tool Search) and memory_store (cross-session
103
108
  # memory) matter only to it — nil = parity (deferred
@@ -123,7 +128,11 @@ module Insika
123
128
  # tools gated by a parsed policy AND both stores present. nil = never
124
129
  # wired (parity).
125
130
  contact_store: contact_store,
126
- followup_store: followup_store
131
+ followup_store: followup_store,
132
+ # the builder wires the load_knowledge system tool gated by
133
+ # @knowledge_store + profile.knowledge["retrieve"]. nil = never
134
+ # wired (parity — the write path above stays unaffected either way).
135
+ knowledge_store: knowledge_store
127
136
  )
128
137
  # Stage-3-tail tool assembly (capability resolution, instantiation,
129
138
  # injection, dedup join, ToolEnvelope wrap) — extracted collaborator.
@@ -175,6 +184,20 @@ module Insika
175
184
  # scans, nothing mines).
176
185
  attr_accessor :harvest_engine
177
186
 
187
+ # Forces the long-lived turn supervisor (and its tick/alert/distill/harvest
188
+ # children — see #turn_parent) to start NOW instead of lazily on the first
189
+ # served turn. Without this, a deployment whose only agents are scheduled
190
+ # (no live chat) never fires the tick until some unrelated turn happens to
191
+ # land first — observed live as 16+ minutes of silence after a clean boot.
192
+ # Call once, right after `supervised = true`, from the composition root
193
+ # (Server::Boot / config.ru / DSL::ServerBoot). A no-op outside a live
194
+ # reactor (nothing to bind the supervisor to yet) — the lazy path in
195
+ # #turn_parent still covers that case, e.g. specs that never enter Async.
196
+ def start_supervisor!
197
+ turn_parent if @supervised && Async::Task.current?
198
+ nil
199
+ end
200
+
178
201
  # closes the TURN intake for shutdown. Armed by Insika::Shutdown
179
202
  # when the process is asked to stop: from here on a new top-level turn is left
180
203
  # `:queued` (durable — the next boot's recovery replays it) instead of
@@ -1015,6 +1038,11 @@ module Insika
1015
1038
  tokens: estimate_tools_tokens(state.allowed_tools) },
1016
1039
  fingerprints: fingerprints,
1017
1040
  cache: { invalidation_reason: reason } }
1041
+ # the compaction state this turn was BUILT over (RFC-0044) —
1042
+ # {upto, runs}, counts only; the summary fragment itself already shows
1043
+ # as the trace's own "compaction" category via its fragment source.
1044
+ compaction = state.session.respond_to?(:compaction) ? state.session.compaction : nil
1045
+ entry[:compaction] = { upto: compaction["upto"], runs: compaction["runs"] } if compaction
1018
1046
  # Park the SANITIZED entry (string keys) — the stage-8 stamp merges into
1019
1047
  # it and re-records the same key; a raw entry would add a SECOND "cache"
1020
1048
  # key that sanitize would then ignore (the symbol one wins).
@@ -1262,6 +1290,13 @@ module Insika
1262
1290
  response = @hooks.around(:agent, state) do |s|
1263
1291
  result = @reliability ? run_reliable_ask(task, s, filter, timing)
1264
1292
  : run_single_ask(task, s, filter, timing)
1293
+ # A steered message that no tool batch absorbed (a text-only turn: the
1294
+ # injector only drains at a tool boundary) is still in the mailbox when
1295
+ # RubyLLM's loop returns. ONE extra round, in the SAME turn, answers the
1296
+ # burst coherently -- without it the release would spawn a follow-up
1297
+ # whose answer has no delivery channel under request/response.
1298
+ # Read AFTER it: the extra round replaces both the response and `asked`.
1299
+ absorb_steer_overflow(task, s, timing, result)
1265
1300
  output = result[:output]
1266
1301
  asked = result[:asked]
1267
1302
  result[:response]
@@ -1369,10 +1404,11 @@ module Insika
1369
1404
  #
1370
1405
  # Content parts on the command -> a turn: audio parts are transcribed (the
1371
1406
  # text enters the message marked `source: :voice` — the consumer's signal
1372
- # the person SPOKE), image parts become the ask's attachments (the model
1373
- # sees them; the provider bills them — usage flows) and the first URL is
1374
- # deposited as `ctx.image_url` for data/HTTP tools. The engine transports
1375
- # media, never meaning: no speech/vision logic beyond the call itself.
1407
+ # the person SPOKE), image and document parts become the ask's attachments
1408
+ # (the model sees them; the provider bills them — usage flows) and the
1409
+ # first URL of each kind is deposited as `ctx.image_url` / `ctx.document_url`
1410
+ # for data/HTTP tools. The engine transports media, never meaning: no
1411
+ # speech/vision logic beyond the call itself.
1376
1412
  def run_media_stage(task, state)
1377
1413
  # a consumer that pre-transcribed voice text labels it `source: voice`;
1378
1414
  # the marker rides the turn even when there are no audio PARTS left.
@@ -1383,20 +1419,33 @@ module Insika
1383
1419
 
1384
1420
  voice = Insika::Media.audio_parts(parts)
1385
1421
  if voice.any?
1386
- text = voice.map { |p| media_transcribe(p.url) }.reject(&:empty?).join(" ")
1422
+ text = voice.map { |p| media_transcribe(p.url, state) }.reject(&:empty?).join(" ")
1387
1423
  state.message = [state.message.to_s, text].reject(&:empty?).join("\n")
1388
1424
  state.message_source = :voice
1389
1425
  end
1390
1426
 
1391
1427
  images = Insika::Media.image_parts(parts)
1392
1428
  if images.any?
1393
- state.media_attachments = images.map { |p| media_attachment(p.url) }
1429
+ state.image_attachments = images.map { |p| media_attachment(p.url) }
1430
+ state.media_attachments = state.image_attachments
1394
1431
  # First image URL for data tools (`{{ctx.image_url}}`) — photo analysis
1395
1432
  # outside the prompt. The model still sees the attachment; the tool
1396
1433
  # gets the original URL (its own egress applies when it fetches).
1397
1434
  state.turn_context = (state.turn_context || {}).merge(image_url: images.first.url)
1398
1435
  end
1399
1436
 
1437
+ # Documents (a prescription, a recipe, an invoice) ride the SAME
1438
+ # attachments array as images — RubyLLM's `ask(with:)` takes both, and
1439
+ # the attachment content-sniffs PDF magic bytes when the URL has no
1440
+ # extension — capped separately (MAX_DOCUMENT_BYTES) since a document
1441
+ # is not a photo.
1442
+ documents = Insika::Media.document_parts(parts)
1443
+ if documents.any?
1444
+ doc_attachments = documents.map { |p| media_attachment(p.url, max_bytes: Insika::Media::MAX_DOCUMENT_BYTES) }
1445
+ state.media_attachments = Array(state.media_attachments) + doc_attachments
1446
+ state.turn_context = (state.turn_context || {}).merge(document_url: documents.first.url)
1447
+ end
1448
+
1400
1449
  # A media-only turn (a voice note with no caption) is legitimate — the
1401
1450
  # surfaces admit it — but it must leave this stage with something to ask
1402
1451
  # about. Empty text AND no attachment means the parts carried nothing the
@@ -1410,41 +1459,30 @@ module Insika
1410
1459
  # The STT seam: the injected transcriber (specs), else the default
1411
1460
  # (fetch + RubyLLM::Transcription — lazy require). A failed transcription
1412
1461
  # fails the turn loudly (MediaError -> :media): a voice message that was
1413
- # not heard must not become a hallucinated one.
1414
- def media_transcribe(url)
1415
- transcriber = @media || (@default_transcriber ||= Insika::Media.default_transcriber(
1462
+ # not heard must not become a hallucinated one. The default is rebuilt
1463
+ # PER CALL (never memoized) because its vocabulary `prompt:` is resolved
1464
+ # from THIS turn's profile — a memoized seam would freeze the first
1465
+ # agent's prompt (or its absence) for every agent sharing the executor.
1466
+ def media_transcribe(url, state)
1467
+ transcriber = @media || Insika::Media.default_transcriber(
1416
1468
  stt_model: Insika::EnvSchema.read("INSIKA_STT_MODEL"),
1417
- stt_language: Insika::EnvSchema.read("INSIKA_STT_LANGUAGE")
1418
- ))
1469
+ stt_language: Insika::EnvSchema.read("INSIKA_STT_LANGUAGE"),
1470
+ stt_prompt: resolved_stt_prompt(state.profile)
1471
+ )
1419
1472
  transcriber.call(url)
1420
1473
  end
1421
1474
 
1422
- # An image part -> the ask's attachment. RubyLLM required lazily (load-guard).
1423
- #
1424
- # The bytes come through OUR fetch (`Media.fetch_binary`), which is
1425
- # egress-guarded — the URL is CONSUMER input, so a private/metadata target
1426
- # fails the turn loudly at :media — and SIZE-CAPPED. Handing the raw URL to
1427
- # `RubyLLM::Attachment` instead left the fetch to the gem, whose
1428
- # `fetch_content` reads the whole response with no ceiling: a hostile URL
1429
- # answering an endless body grows this process until it dies. An io-like
1430
- # source (StringIO) is the branch of Attachment that takes bytes we already
1431
- # hold; the provider then gets base64 rather than the URL, which every
1432
- # vision provider accepts.
1433
- def media_attachment(url)
1434
- require "ruby_llm"
1435
- require "stringio"
1436
-
1437
- bytes = Insika::Media.fetch_binary(url, max_bytes: Insika::Media::MAX_IMAGE_BYTES)
1438
- RubyLLM::Attachment.new(StringIO.new(bytes), filename: media_filename(url))
1475
+ # Resolution order: the agent profile's own vocabulary hint
1476
+ # (it knows its catalog) beats the deployment-wide default, which beats
1477
+ # nothing (no prompt: kwarg at all).
1478
+ def resolved_stt_prompt(profile)
1479
+ Insika::Coercion.presence(profile&.stt_prompt) || Insika::EnvSchema.read("INSIKA_STT_PROMPT")
1439
1480
  end
1440
1481
 
1441
- # The URL's basename, for the attachment's mime sniff (".png" -> image/png;
1442
- # a URL with no filename falls back to the content sniff RubyLLM does).
1443
- def media_filename(url)
1444
- name = File.basename(URI.parse(url).path.to_s)
1445
- name.empty? ? nil : name
1446
- rescue URI::InvalidURIError
1447
- nil
1482
+ # An image/document part -> the ask's attachment (Insika::Media.url_attachment
1483
+ # — egress-guarded, size-capped fetch; the caller picks the ceiling).
1484
+ def media_attachment(url, max_bytes: Insika::Media::MAX_IMAGE_BYTES)
1485
+ Insika::Media.url_attachment(url, max_bytes: max_bytes)
1448
1486
  end
1449
1487
 
1450
1488
  # --- WS4 intent routing --------------------------------------------
@@ -1634,7 +1672,39 @@ module Insika
1634
1672
  def wire_chat_output(task, state, output)
1635
1673
  chat = state.chat
1636
1674
  chat.after_message { |message| output.message_ended(message) } if chat.respond_to?(:after_message)
1637
- install_steer_injector(task, state)
1675
+ state.steer_injector = install_steer_injector(task, state)
1676
+ end
1677
+
1678
+ # The extra round for a steered message that never met a tool boundary (a
1679
+ # text-only turn closes no batch, so `SteerInjector` never got its cue). The
1680
+ # burst is appended to the history and the model answers ALL of it in one go,
1681
+ # inside the SAME turn — without this the release would spawn a follow-up whose
1682
+ # answer has no delivery channel under request/response.
1683
+ #
1684
+ # Skipped when the turn HALTED: there is no next model step by construction.
1685
+ # `release_steered` (execute's ensure) stays the fallback there, as it is for a
1686
+ # failed or cancelled turn, and for a transport that cannot `complete`.
1687
+ #
1688
+ # At most one extra round: if the model ignores the burst, the turn ends anyway.
1689
+ def absorb_steer_overflow(task, state, timing, result)
1690
+ injector = state.steer_injector
1691
+ return unless injector && state.chat.respond_to?(:complete)
1692
+ return unless state.actor&.user_messages_posted&.positive?
1693
+ return if halted?(result[:response])
1694
+ return if injector.absorb_pending!.zero?
1695
+
1696
+ # TWO provider round trips now. Bank the first one's tokens BEFORE the response
1697
+ # is replaced — stage 6's merge only sees whatever `result[:response]` ends up
1698
+ # being, so without this the extra round silently erases the first round's cost
1699
+ # from the terminal usage and from the EdgeLimiter's budget.
1700
+ state.usage = merge_usage(with_model_source(usage_of(result[:response]), state.model_selection),
1701
+ state.usage)
1702
+ # COMPLETE, not `ask(nil)`: the messages are already in the history, and an
1703
+ # ask with no text appends an EMPTY user message after them — which is both a
1704
+ # duplicate turn-opener and something providers refuse outright.
1705
+ extra = state.chat.complete(&turn_chunk_handler(task, state, result[:output], timing))
1706
+ result[:response] = extra
1707
+ result[:asked] = extra
1638
1708
  end
1639
1709
 
1640
1710
  # The ask itself, chunk-by-chunk (WS3 attempts and the plain path share it).
@@ -1644,9 +1714,27 @@ module Insika
1644
1714
  # :ttft on EVERY content chunk (3 chunks = 3 insika.ttft frames); the spec
1645
1715
  # passed because FakeChat emits a single chunk.
1646
1716
  def ask_on(task, state, chat, output, timing)
1717
+ each_chunk = turn_chunk_handler(task, state, output, timing)
1718
+ # WS9: image parts ride the ask as attachments (only then — a chat whose
1719
+ # ask has no `with:` keeps working, and the plain path is byte-identical).
1720
+ # An image with no caption asks with NIL, not "": an empty text part is a
1721
+ # thing some providers refuse, and nil is how RubyLLM says "attachments
1722
+ # only".
1723
+ if state.media_attachments
1724
+ text = state.message.to_s.empty? ? nil : state.message
1725
+ chat.ask(text, with: state.media_attachments, &each_chunk)
1726
+ else
1727
+ chat.ask(state.message, &each_chunk)
1728
+ end
1729
+ end
1730
+
1731
+ # The per-round chunk sink. Built FRESH per round: `ttft_sent` is that round's
1732
+ # own bookkeeping, and the steer overflow round is a second round on the same
1733
+ # chat.
1734
+ def turn_chunk_handler(task, state, output, timing)
1647
1735
  public_thinking = state.profile.stream_public?(:thinking)
1648
1736
  ttft_sent = false
1649
- each_chunk = lambda do |chunk|
1737
+ lambda do |chunk|
1650
1738
  emit_thinking(chunk, task, public: public_thinking)
1651
1739
  next unless chunk.content
1652
1740
 
@@ -1657,17 +1745,6 @@ module Insika
1657
1745
  end
1658
1746
  output.push(chunk.content)
1659
1747
  end
1660
- # WS9: image parts ride the ask as attachments (only then — a chat whose
1661
- # ask has no `with:` keeps working, and the plain path is byte-identical).
1662
- # An image with no caption asks with NIL, not "": an empty text part is a
1663
- # thing some providers refuse, and nil is how RubyLLM says "attachments
1664
- # only".
1665
- if state.media_attachments
1666
- text = state.message.to_s.empty? ? nil : state.message
1667
- chat.ask(text, with: state.media_attachments, &each_chunk)
1668
- else
1669
- chat.ask(state.message, &each_chunk)
1670
- end
1671
1748
  end
1672
1749
 
1673
1750
  # The provider's TTFB as a live event (data: ttft_ms) — only under
@@ -1950,10 +2027,17 @@ module Insika
1950
2027
  # the memory scope carries a customer — the backend identifies the store,
1951
2028
  # not the shopper (WS8 keeps the two scopes separate).
1952
2029
  tenant: command_tenant(task) || task.session_id,
2030
+ # the DECLARED tenant alone (nil in single-tenant). Distinct from
2031
+ # `tenant` for the ownership-binding tools (save_artifact): a report
2032
+ # belongs to the AGENT's tenant — the deployment's tenant in
2033
+ # single-tenant, never the chat that happened to run it.
2034
+ command_tenant: command_tenant(task),
1953
2035
  store_id: profile.store_id,
1954
- # current delegation depth (0 for a top-level turn). Carried in
1955
- # the child command's payload by run_subagent; read here so the child's OWN
1956
- # spawn_subagent tool sees depth+1 and the runtime cap holds down the chain.
2036
+ # the current task id — the save_artifact binding (which run produced
2037
+ # this report), same turn-origin discipline as the rest of ctx.*.
2038
+ task_id: task.id,
2039
+ # current delegation depth (0 for a top-level turn). Set by run_subagent
2040
+ # for children.
1957
2041
  delegation_depth: delegation_depth(task)
1958
2042
  }
1959
2043
  end
@@ -2391,6 +2475,18 @@ module Insika
2391
2475
  # has to travel out of band. Same terminal hook, next door to the delegation
2392
2476
  # one, for the same reason: it fires for a fresh turn and a recovered one.
2393
2477
  finalize_channel_delivery(task, content, state, timing)
2478
+
2479
+ # extracts durable concepts from this turn, off the critical
2480
+ # path — the user already has the answer above. Same terminal hook,
2481
+ # next door to the other two, for the same reason: it fires for a fresh
2482
+ # turn and a recovered one.
2483
+ finalize_knowledge_extraction(task, profile, new_messages)
2484
+
2485
+ # in-session compaction (RFC-0044): when the uncompacted
2486
+ # transcript crossed the threshold this turn, summarize the old prefix
2487
+ # with the cheap model and move the boundary — off the critical path,
2488
+ # same terminal hook, same best-effort discipline as the other three.
2489
+ finalize_compaction(task, profile)
2394
2490
  end
2395
2491
 
2396
2492
  # Records the answer in the outbox and dispatches it. The discriminator is the
@@ -2433,6 +2529,148 @@ module Insika
2433
2529
  nil
2434
2530
  end
2435
2531
 
2532
+ # Turns whose combined transcript slice is this trivially short skip
2533
+ # extraction entirely ("ok thanks" exchanges) — no new config surface,
2534
+ # just avoids a wasted utility-model call.
2535
+ KNOWLEDGE_MIN_CHARS = 200
2536
+
2537
+ # No-op without a knowledge store, without the profile's opt-in
2538
+ # (`knowledge.extract`), without a usable model, or for a trivially short
2539
+ # turn. Otherwise dispatches the extraction off the critical path — the
2540
+ # SAME `dispatch_chain` shape: inline when non-supervised (tests, CLI,
2541
+ # boot sweep), a child of the turn supervisor when serving (survives the
2542
+ # request's own disconnect). Best-effort: any failure is swallowed here,
2543
+ # never re-fails an already-committed turn.
2544
+ def finalize_knowledge_extraction(task, profile, new_messages)
2545
+ return unless @knowledge_store
2546
+
2547
+ config = Coercion.deep_stringify(profile.knowledge)
2548
+ return unless config && Coercion.truthy?(config["extract"])
2549
+ return if knowledge_transcript(new_messages).length < KNOWLEDGE_MIN_CHARS
2550
+
2551
+ extractor = Knowledge::ExtractorFactory.build(config, utility_model: utility_model)
2552
+ return unless extractor
2553
+
2554
+ # Same resolved model as the extractor — a deployment names one
2555
+ # knowledge model, not two. nil consolidator (no model resolvable) is
2556
+ # still meaningful: write_concept's conservative default.
2557
+ consolidator = Knowledge::ConsolidatorFactory.build(config, utility_model: utility_model)
2558
+
2559
+ run = lambda { run_knowledge_extraction(task, profile, config, new_messages, extractor, consolidator) }
2560
+ return run.call unless @supervised
2561
+
2562
+ turn_parent.async do |t|
2563
+ t.annotate("knowledge:#{task.id}")
2564
+ run.call
2565
+ end
2566
+ end
2567
+
2568
+ def run_knowledge_extraction(task, profile, config, new_messages, extractor, consolidator)
2569
+ prompt = knowledge_prompt(config, new_messages)
2570
+ result = extractor.extract(prompt: prompt)
2571
+ result[:concepts].each do |concept|
2572
+ outcome = Knowledge.write_concept(
2573
+ store: @knowledge_store, agent_id: profile.id, concept: concept, session_id: task.session_id,
2574
+ tenant: task_tenant(task), consolidator: consolidator
2575
+ )
2576
+ emit_knowledge_event(outcome, profile, task)
2577
+ end
2578
+ rescue StandardError
2579
+ nil # best-effort: extraction never re-fails an already-committed turn.
2580
+ end
2581
+
2582
+ # :new/:related taught the agent something; :contradicting needs a
2583
+ # human; :same is a silent reinforcement (no event — a popular concept
2584
+ # would otherwise spam the stream every time it's confirmed).
2585
+ def emit_knowledge_event(outcome, profile, task)
2586
+ case outcome[:verdict]
2587
+ when :new, :related
2588
+ emit(:knowledge_learned, { name: outcome[:name], type: outcome[:type], agent: profile.id }, task: task)
2589
+ when :contradicting
2590
+ emit(:knowledge_conflict, { name: outcome[:name], agent: profile.id }, task: task)
2591
+ end
2592
+ end
2593
+
2594
+ def knowledge_prompt(config, new_messages)
2595
+ base = Coercion.presence(config["prompt"]) || Knowledge::DEFAULT_PROMPT
2596
+ <<~PROMPT
2597
+ #{base.rstrip}
2598
+
2599
+ ## The conversation
2600
+
2601
+ #{knowledge_transcript(new_messages)}
2602
+ PROMPT
2603
+ end
2604
+
2605
+ # Redacted (RFC's PII rule applies to what reaches the model too, not
2606
+ # just what gets persisted).
2607
+ def knowledge_transcript(new_messages)
2608
+ redacted, = Insika::Safety::Detectors.redact(
2609
+ new_messages.each_with_index.map { |m, i| "[#{i}] #{m['role'] || m[:role]}: #{m['content'] || m[:content]}" }
2610
+ .join("\n")
2611
+ )
2612
+ redacted
2613
+ end
2614
+
2615
+ def utility_model
2616
+ return nil unless @settings_store
2617
+
2618
+ @settings_store.get["utility_model"]
2619
+ end
2620
+
2621
+ # in-session compaction (RFC-0044). Platform-gated
2622
+ # (Settings compaction.enabled — parity when off), planned over the
2623
+ # POST-append transcript (persist_turn already wrote this turn's messages),
2624
+ # dispatched off the critical path — the SAME shape as
2625
+ # finalize_knowledge_extraction: inline when non-supervised, a child of the
2626
+ # turn supervisor when serving. Best-effort: any failure leaves the session
2627
+ # record untouched and the next turn re-plans.
2628
+ def finalize_compaction(task, profile)
2629
+ return unless task.session_id && @settings_store
2630
+
2631
+ config = Coercion.deep_stringify(@settings_store.get["compaction"])
2632
+ return unless config && Coercion.truthy?(config["enabled"])
2633
+
2634
+ session = @session_store.find(task.session_id)
2635
+ return unless session
2636
+
2637
+ state = session.respond_to?(:compaction) ? session.compaction : nil
2638
+ plan = Insika::Compaction.plan(messages: session.messages, state: state, config: config)
2639
+ return unless plan
2640
+
2641
+ # compaction.model -> platform utility_model -> inert (never a guess);
2642
+ # `insika doctor` warns on enabled-with-no-model. @llm rides along so a
2643
+ # deployment (or spec) that injects its LLM seam covers this call too.
2644
+ summarizer = Compaction::SummarizerFactory.build(config, utility_model: utility_model, llm: @llm)
2645
+ return unless summarizer
2646
+
2647
+ run = lambda { run_compaction(task, profile, session, state, plan, config, summarizer) }
2648
+ return run.call unless @supervised
2649
+
2650
+ turn_parent.async do |t|
2651
+ t.annotate("compaction:#{task.id}")
2652
+ run.call
2653
+ end
2654
+ end
2655
+
2656
+ def run_compaction(task, profile, session, state, plan, config, summarizer)
2657
+ prompt = Insika::Compaction.prompt(messages: session.messages, plan: plan,
2658
+ previous: state && state["summary"],
2659
+ base: config["prompt"])
2660
+ result = summarizer.summarize(prompt: prompt)
2661
+ updated = @session_store.set_compaction(task.session_id, summary: result[:summary],
2662
+ upto: plan.upto, model: summarizer.model)
2663
+ # counts and ids only — the summary text never enters the stream. Feeds
2664
+ # the insika.context.compacted counter (Telemetry::Recorder).
2665
+ emit(:context_compacted,
2666
+ { task_id: task.id, agent: profile.id, from: plan.from, upto: plan.upto,
2667
+ messages: plan.count, runs: updated.compaction && updated.compaction["runs"],
2668
+ model: summarizer.model, cost: result[:cost] },
2669
+ task: task)
2670
+ rescue StandardError
2671
+ nil # best-effort: compaction never re-fails an already-committed turn.
2672
+ end
2673
+
2436
2674
  # ONE supervisor fiber for the whole chain. Sequential deliver
2437
2675
  # calls, so balloon N+1 cannot overtake balloon N on the wire. Still off the
2438
2676
  # session's FIFO — the customer's next message does not wait on this turn's
@@ -2546,11 +2784,13 @@ module Insika
2546
2784
 
2547
2785
  # Stage 6 (factory): the ONLY point that touches the gem. lazy require,
2548
2786
  # confined — not covered by unit (factory line). It also loads the system
2549
- # builtins (load_skill/tool_search/remember) that the ChatBuilder assembles at
2550
- # stage 5 — lazy, so the core installs without ruby_llm.
2787
+ # builtins (load_skill/load_knowledge/tool_search/remember) that the
2788
+ # ChatBuilder assembles at stage 5 — lazy, so the core installs without
2789
+ # ruby_llm.
2551
2790
  def create_chat(profile, state)
2552
2791
  require "ruby_llm"
2553
2792
  require_relative "tools/load_skill"
2793
+ require_relative "tools/load_knowledge"
2554
2794
  require_relative "tools/tool_search"
2555
2795
  require_relative "tools/remember"
2556
2796
  require_relative "tools/subagent"
@@ -70,7 +70,7 @@ module Insika
70
70
  def quiet?(time)
71
71
  return false unless @quiet_hours
72
72
 
73
- local = in_zone(@quiet_hours.timezone, time) { |t| t.getlocal }
73
+ local = Insika::Timezone.in_zone(@quiet_hours.timezone, time) { |t| t.getlocal }
74
74
  minutes = local.hour * 60 + local.min
75
75
  start_min = minutes_of(@quiet_hours.start)
76
76
  end_min = minutes_of(@quiet_hours.end)
@@ -136,7 +136,7 @@ module Insika
136
136
  # a bogus IANA zone is a malformed policy — refused HERE, where the
137
137
  # doctor can name it. An unknown ENV["TZ"] silently behaves as UTC, so
138
138
  # existence is checked against the OS tz database, not by asking Time.
139
- unless zone_known?(timezone.to_s)
139
+ unless Insika::Timezone.known?(timezone.to_s)
140
140
  raise Insika::ValidationError,
141
141
  "followup.policy.quiet_hours.timezone is not a valid IANA timezone: #{timezone.inspect}"
142
142
  end
@@ -190,29 +190,6 @@ module Insika
190
190
  text.to_s.unicode_normalize(:nfd).gsub(/\p{Mn}/, "").downcase
191
191
  end
192
192
 
193
- # Yields `time` interpreted in the given IANA zone (via a save/restore of
194
- # ENV["TZ"] — the stdlib-only route to the OS tz database; see #quiet?).
195
- def in_zone(zone, time)
196
- previous = ENV["TZ"]
197
- ENV["TZ"] = zone
198
- yield time
199
- ensure
200
- ENV["TZ"] = previous
201
- end
202
-
203
- # The candidate tz-data roots (TZDIR first — Ruby's own lookup env). The
204
- # zone name maps to a FILE under the root ("America/Sao_Paulo" ->
205
- # "America/Sao_Paulo").
206
- TZ_ROOTS = ([ENV["TZDIR"]] +
207
- %w[/usr/share/zoneinfo /usr/share/lib/zoneinfo /etc/zoneinfo])
208
- .compact.freeze
209
-
210
- def zone_known?(zone)
211
- return true if zone == "UTC" || zone == "Etc/UTC"
212
-
213
- TZ_ROOTS.any? { |root| File.directory?(root) && File.exist?(File.join(root, zone)) }
214
- end
215
-
216
193
  def minutes_of(hhmm)
217
194
  h, m = hhmm.split(":").map(&:to_i)
218
195
  h * 60 + m
@@ -63,6 +63,11 @@ module Insika
63
63
  # -> [Evals::Golden] one agent's cases.
64
64
  def for_agent(agent_id) = all.select { |g| g.agent == agent_id.to_s }
65
65
 
66
+ # -> [Evals::Golden] one tenant's cases (C3.1's isolation query —
67
+ # `run_persona_eval` never lists or runs a case outside the calling agent's
68
+ # own tenant, "platform" being the single-tenant default).
69
+ def for_tenant(tenant) = all.select { |g| g.tenant == tenant.to_s }
70
+
66
71
  # -> [String] ids whose stored mapping no longer validates (an edit that broke the
67
72
  # shape). The Studio shows these; they never silently vanish from a run.
68
73
  def invalid
@@ -130,13 +135,23 @@ module Insika
130
135
  # in the store as it is on disk — but it is NEVER dropped when present: a case
131
136
  # that silently lost its requirements would come back as a failure on every
132
137
  # deployment that lacks the tool, which is the exact lie `requires` exists to end.
138
+ # Same for `persona`: a simulated case that lost its persona would
139
+ # silently stop being simulated and replay as an empty script.
133
140
  def case_hash(golden)
134
- h = { "id" => golden.id, "agent" => golden.agent, "turns" => golden.turns }
141
+ h = if golden.simulated?
142
+ { "id" => golden.id, "agent" => golden.agent, "persona" => golden.persona.to_h }
143
+ else
144
+ { "id" => golden.id, "agent" => golden.agent, "turns" => golden.turns }
145
+ end
135
146
  h["requires"] = golden.requires unless golden.requires.empty?
136
147
  # Same rule for `reference`: omitted when absent, never dropped
137
148
  # when present. A case that lost its reference in a round-trip would stop being
138
149
  # compared against the incumbent and the report would look identical.
139
150
  h["reference"] = golden.reference unless golden.reference.empty?
151
+ # `tenant` follows the same omit-when-default rule: "platform" is the
152
+ # loader's own default, so leaving it out reproduces the same case; an
153
+ # explicit tenant is a case's isolation boundary and must never round-trip away.
154
+ h["tenant"] = golden.tenant unless golden.tenant == "platform"
140
155
  h.merge("expect" => golden.expect)
141
156
  end
142
157
 
@@ -7,7 +7,7 @@ module Insika
7
7
  # The pack's DATA (D7): a `sku` regex for the SKU shape. The engine only
8
8
  # APPLIES it — it never guesses what a SKU looks like. No NLP, no LLM, no gem.
9
9
  #
10
- # REVIEW-DECISION (rfc-0029 v2): grounding is **SKU-only**. The techspec's
10
+ # REVIEW-DECISION: grounding is **SKU-only**. The techspec's
11
11
  # name half (name_keys indexing ledger lines to flag names) cannot flag
12
12
  # anything: a name found in the text that also came from the ledger is grounded
13
13
  # by construction, and detecting a name that matches NO ledger line needs a