insika 0.3.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (190) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +180 -0
  3. data/README.md +45 -10
  4. data/bin/insika +684 -0
  5. data/bin/insika-router +87 -0
  6. data/docs/AGENTS.md +94 -403
  7. data/docs/API.md +5 -5
  8. data/docs/ARCHITECTURE.md +3 -2
  9. data/docs/ARTIFACTS.md +95 -0
  10. data/docs/BENCHMARK.md +2 -2
  11. data/docs/CHANNELS.md +14 -14
  12. data/docs/CONTEXT.md +9 -7
  13. data/docs/DEMO.md +80 -0
  14. data/docs/DEPLOY.md +71 -3
  15. data/docs/EMBEDDING.md +1 -1
  16. data/docs/EVALS.md +128 -3
  17. data/docs/FACTS.md +3 -3
  18. data/docs/HARVEST.md +5 -6
  19. data/docs/KNOWLEDGE.md +290 -0
  20. data/docs/LOADTEST.md +2 -2
  21. data/docs/MEDIA.md +128 -0
  22. data/docs/OBSERVABILITY.md +15 -10
  23. data/docs/OUTCOMES.md +137 -0
  24. data/docs/PLUGINS.md +51 -6
  25. data/docs/POLICY.md +216 -0
  26. data/docs/REFINEMENT.md +14 -9
  27. data/docs/RELEASING.md +4 -4
  28. data/docs/ROUTER.md +213 -0
  29. data/docs/RUNNING-LOCAL.md +3 -3
  30. data/docs/SCHEDULING.md +121 -0
  31. data/docs/SECURITY.md +22 -6
  32. data/docs/SKILLS.md +11 -2
  33. data/docs/SOAK.md +2 -2
  34. data/docs/TEMPLATES.md +134 -0
  35. data/docs/TOOLS.md +152 -27
  36. data/docs/WHY.md +1 -1
  37. data/docs/WORKFLOWS.md +2 -2
  38. data/docs/_includes/head_custom.html +5 -0
  39. data/docs/_includes/title.html +13 -0
  40. data/docs/_sass/color_schemes/insika.scss +32 -0
  41. data/docs/_sass/custom/custom.scss +199 -0
  42. data/docs/_sass/custom/setup.scss +26 -0
  43. data/docs/assets/img/favicon.svg +7 -0
  44. data/docs/assets/img/insika-mark.svg +7 -0
  45. data/docs/core-concepts.md +21 -0
  46. data/docs/domain.md +4 -4
  47. data/docs/improve.md +20 -0
  48. data/docs/index.md +8 -5
  49. data/docs/integrate.md +20 -0
  50. data/docs/operate.md +13 -6
  51. data/docs/prompts/ADD-TOOL.md +118 -0
  52. data/docs/prompts/DIAGNOSE-TURN.md +65 -0
  53. data/docs/prompts/GO-LIVE.md +138 -0
  54. data/docs/prompts/RUN-EXAMPLES.md +70 -0
  55. data/docs/reference.md +19 -0
  56. data/docs/ship.md +10 -2
  57. data/docs/start-here.md +18 -0
  58. data/lib/insika/agent_profile.rb +73 -16
  59. data/lib/insika/artifact_signing.rb +82 -0
  60. data/lib/insika/artifact_store.rb +160 -0
  61. data/lib/insika/channel_delivery.rb +1 -1
  62. data/lib/insika/chat_builder.rb +22 -2
  63. data/lib/insika/commands/agent_payload.rb +2 -2
  64. data/lib/insika/commands/backfill_knowledge.rb +145 -0
  65. data/lib/insika/commands/delete_artifact.rb +35 -0
  66. data/lib/insika/commands/delete_concept.rb +34 -0
  67. data/lib/insika/commands/delete_mcp.rb +6 -2
  68. data/lib/insika/commands/delete_tenant_data.rb +15 -3
  69. data/lib/insika/commands/gate_refinement.rb +1 -1
  70. data/lib/insika/commands/refresh_mcp_tools.rb +47 -0
  71. data/lib/insika/commands/restore_concept.rb +34 -0
  72. data/lib/insika/commands/seed_demo_data.rb +31 -0
  73. data/lib/insika/commands/upsert_mcp.rb +6 -3
  74. data/lib/insika/commands/write_concept.rb +57 -0
  75. data/lib/insika/context/priority.rb +2 -0
  76. data/lib/insika/context/providers/knowledge.rb +108 -0
  77. data/lib/insika/context/providers/prompt.rb +30 -24
  78. data/lib/insika/cron.rb +189 -0
  79. data/lib/insika/demo/agent_attrs.rb +43 -0
  80. data/lib/insika/demo/golden_cases.rb +81 -0
  81. data/lib/insika/demo/seeder.rb +336 -0
  82. data/lib/insika/doctor.rb +176 -8
  83. data/lib/insika/dsl/definition.rb +3 -2
  84. data/lib/insika/dsl/runtime.rb +60 -79
  85. data/lib/insika/dsl/server_boot.rb +23 -1
  86. data/lib/insika/dsl/system.rb +10 -2
  87. data/lib/insika/dsl.rb +103 -2
  88. data/lib/insika/env_schema.rb +16 -1
  89. data/lib/insika/evals/golden.rb +41 -4
  90. data/lib/insika/evals/judge.rb +47 -2
  91. data/lib/insika/evals/pairwise.rb +11 -0
  92. data/lib/insika/evals/persona.rb +98 -0
  93. data/lib/insika/evals/runner.rb +9 -0
  94. data/lib/insika/evals/simulator.rb +225 -0
  95. data/lib/insika/evals/transport.rb +83 -1
  96. data/lib/insika/event_stream.rb +10 -0
  97. data/lib/insika/executor.rb +231 -55
  98. data/lib/insika/followup_policy.rb +2 -25
  99. data/lib/insika/golden_store.rb +16 -1
  100. data/lib/insika/grounding/matcher.rb +1 -1
  101. data/lib/insika/knowledge.rb +680 -0
  102. data/lib/insika/knowledge_store.rb +140 -0
  103. data/lib/insika/mcp_client.rb +94 -0
  104. data/lib/insika/mcp_json.rb +74 -0
  105. data/lib/insika/mcp_live_tool.rb +43 -0
  106. data/lib/insika/mcp_store.rb +98 -26
  107. data/lib/insika/mcp_tool_ingestor.rb +30 -8
  108. data/lib/insika/mcp_tool_registry.rb +100 -0
  109. data/lib/insika/media.rb +115 -31
  110. data/lib/insika/message_origin.rb +1 -1
  111. data/lib/insika/middleware.rb +9 -0
  112. data/lib/insika/onboarding.rb +17 -1
  113. data/lib/insika/outcome_store.rb +1 -1
  114. data/lib/insika/overlay_tool_registry.rb +37 -17
  115. data/lib/insika/packaging.rb +2 -2
  116. data/lib/insika/profile_source.rb +8 -1
  117. data/lib/insika/prompt_catalog.rb +10 -0
  118. data/lib/insika/retention.rb +36 -1
  119. data/lib/insika/router/app.rb +157 -0
  120. data/lib/insika/router/backend_pool.rb +98 -0
  121. data/lib/insika/router/hash_ring.rb +55 -0
  122. data/lib/insika/router/proxy_body.rb +34 -0
  123. data/lib/insika/router/session_key.rb +54 -0
  124. data/lib/insika/router.rb +18 -0
  125. data/lib/insika/schedule.rb +177 -0
  126. data/lib/insika/schedule_engine.rb +314 -0
  127. data/lib/insika/schedule_store.rb +208 -0
  128. data/lib/insika/server/app.rb +105 -15
  129. data/lib/insika/server/rack_app.rb +5 -1
  130. data/lib/insika/server/responses.rb +1 -1
  131. data/lib/insika/skill_catalog.rb +12 -0
  132. data/lib/insika/steer_injector.rb +21 -10
  133. data/lib/insika/studio/app.rb +567 -45
  134. data/lib/insika/studio/assets/dist/application.css +1 -1
  135. data/lib/insika/studio/assets/dist/application.js +21 -21
  136. data/lib/insika/studio/forms.rb +46 -5
  137. data/lib/insika/studio/nav_icons.rb +14 -1
  138. data/lib/insika/studio/views/_agent_tab_cache.erb +25 -0
  139. data/lib/insika/studio/views/_agent_tab_config.erb +514 -0
  140. data/lib/insika/studio/views/_agent_tab_history.erb +24 -0
  141. data/lib/insika/studio/views/_agent_tab_loops.erb +54 -0
  142. data/lib/insika/studio/views/_agent_tab_memory.erb +51 -0
  143. data/lib/insika/studio/views/_agent_tab_outcomes.erb +31 -0
  144. data/lib/insika/studio/views/_agent_tab_prompts.erb +108 -0
  145. data/lib/insika/studio/views/_agent_tab_skills.erb +38 -0
  146. data/lib/insika/studio/views/_agents_master.erb +44 -0
  147. data/lib/insika/studio/views/_message.erb +49 -32
  148. data/lib/insika/studio/views/agent_detail.erb +61 -820
  149. data/lib/insika/studio/views/agents.erb +70 -57
  150. data/lib/insika/studio/views/artifact.erb +23 -0
  151. data/lib/insika/studio/views/artifacts.erb +59 -0
  152. data/lib/insika/studio/views/evals.erb +2 -2
  153. data/lib/insika/studio/views/facts.erb +1 -1
  154. data/lib/insika/studio/views/funnel.erb +1 -1
  155. data/lib/insika/studio/views/home.erb +106 -67
  156. data/lib/insika/studio/views/knowledge.erb +123 -0
  157. data/lib/insika/studio/views/layout.erb +14 -11
  158. data/lib/insika/studio/views/mcp.erb +174 -80
  159. data/lib/insika/studio/views/session.erb +231 -177
  160. data/lib/insika/studio/views/settings.erb +39 -1
  161. data/lib/insika/studio/views/skills.erb +1 -1
  162. data/lib/insika/studio/views/tools.erb +24 -9
  163. data/lib/insika/templates/browser-agent/README.md +36 -0
  164. data/lib/insika/templates/browser-agent/agent.rb +49 -0
  165. data/lib/insika/templates/daily-digest/README.md +38 -0
  166. data/lib/insika/templates/daily-digest/agent.rb +77 -0
  167. data/lib/insika/templates/repo-explorer/README.md +36 -0
  168. data/lib/insika/templates/repo-explorer/agent.rb +45 -0
  169. data/lib/insika/templates/research-analyst/README.md +26 -0
  170. data/lib/insika/templates/research-analyst/agent.rb +58 -0
  171. data/lib/insika/templates/review-panel/README.md +20 -0
  172. data/lib/insika/templates/review-panel/agent.rb +50 -0
  173. data/lib/insika/templates/travel-planner/README.md +35 -0
  174. data/lib/insika/templates/travel-planner/agent.rb +87 -0
  175. data/lib/insika/templates.rb +112 -0
  176. data/lib/insika/tick.rb +24 -12
  177. data/lib/insika/timezone.rb +45 -0
  178. data/lib/insika/tools/generate_image.rb +52 -7
  179. data/lib/insika/tools/load_knowledge.rb +74 -0
  180. data/lib/insika/tools/run_persona_eval.rb +328 -0
  181. data/lib/insika/tools/save_artifact.rb +95 -0
  182. data/lib/insika/turn_output.rb +1 -1
  183. data/lib/insika/turn_state.rb +15 -4
  184. data/lib/insika/version.rb +1 -1
  185. data/lib/insika/wiring/graph.rb +184 -12
  186. data/lib/insika/wiring/graph_chat.rb +102 -0
  187. data/lib/insika.rb +57 -0
  188. metadata +105 -5
  189. data/docs/build.md +0 -14
  190. data/docs/understand.md +0 -10
@@ -22,7 +22,8 @@ module Insika
22
22
  delegation_store: nil, channel_delivery: nil, llm: nil,
23
23
  context_trace_store: nil, reliability: nil, media: nil, media_output: nil,
24
24
  grounding_enforcer: nil, cache_series_store: nil,
25
- contact_store: nil, followup_store: nil, model_visible_trace_store: nil)
25
+ contact_store: nil, followup_store: nil, model_visible_trace_store: nil,
26
+ knowledge_store: nil)
26
27
  @context_builder = context_builder
27
28
  @policy_engine = policy_engine
28
29
  @middleware = middleware
@@ -98,6 +99,10 @@ module Insika
98
99
  # the platform layer of the queue policy (nil = per-agent and
99
100
  # defaults only, which is `followup` with no window — today's behavior).
100
101
  @settings_store = settings_store
102
+ # LEARNED concepts, extracted from a turn's transcript after it
103
+ # completes. nil = the loop is off (parity — every write path below is
104
+ # skipped, zero allocations). Gated per-agent by `profile.knowledge`.
105
+ @knowledge_store = knowledge_store
101
106
  # RubyLLM glue (stages 5-7): chat assembly delegated to ChatBuilder. Its
102
107
  # optional deps tool_catalog (Tool Search) and memory_store (cross-session
103
108
  # memory) matter only to it — nil = parity (deferred
@@ -123,7 +128,11 @@ module Insika
123
128
  # tools gated by a parsed policy AND both stores present. nil = never
124
129
  # wired (parity).
125
130
  contact_store: contact_store,
126
- followup_store: followup_store
131
+ followup_store: followup_store,
132
+ # the builder wires the load_knowledge system tool gated by
133
+ # @knowledge_store + profile.knowledge["retrieve"]. nil = never
134
+ # wired (parity — the write path above stays unaffected either way).
135
+ knowledge_store: knowledge_store
127
136
  )
128
137
  # Stage-3-tail tool assembly (capability resolution, instantiation,
129
138
  # injection, dedup join, ToolEnvelope wrap) — extracted collaborator.
@@ -175,6 +184,20 @@ module Insika
175
184
  # scans, nothing mines).
176
185
  attr_accessor :harvest_engine
177
186
 
187
+ # Forces the long-lived turn supervisor (and its tick/alert/distill/harvest
188
+ # children — see #turn_parent) to start NOW instead of lazily on the first
189
+ # served turn. Without this, a deployment whose only agents are scheduled
190
+ # (no live chat) never fires the tick until some unrelated turn happens to
191
+ # land first — observed live as 16+ minutes of silence after a clean boot.
192
+ # Call once, right after `supervised = true`, from the composition root
193
+ # (Server::Boot / config.ru / DSL::ServerBoot). A no-op outside a live
194
+ # reactor (nothing to bind the supervisor to yet) — the lazy path in
195
+ # #turn_parent still covers that case, e.g. specs that never enter Async.
196
+ def start_supervisor!
197
+ turn_parent if @supervised && Async::Task.current?
198
+ nil
199
+ end
200
+
178
201
  # closes the TURN intake for shutdown. Armed by Insika::Shutdown
179
202
  # when the process is asked to stop: from here on a new top-level turn is left
180
203
  # `:queued` (durable — the next boot's recovery replays it) instead of
@@ -1262,6 +1285,13 @@ module Insika
1262
1285
  response = @hooks.around(:agent, state) do |s|
1263
1286
  result = @reliability ? run_reliable_ask(task, s, filter, timing)
1264
1287
  : run_single_ask(task, s, filter, timing)
1288
+ # A steered message that no tool batch absorbed (a text-only turn: the
1289
+ # injector only drains at a tool boundary) is still in the mailbox when
1290
+ # RubyLLM's loop returns. ONE extra round, in the SAME turn, answers the
1291
+ # burst coherently -- without it the release would spawn a follow-up
1292
+ # whose answer has no delivery channel under request/response.
1293
+ # Read AFTER it: the extra round replaces both the response and `asked`.
1294
+ absorb_steer_overflow(task, s, timing, result)
1265
1295
  output = result[:output]
1266
1296
  asked = result[:asked]
1267
1297
  result[:response]
@@ -1369,10 +1399,11 @@ module Insika
1369
1399
  #
1370
1400
  # Content parts on the command -> a turn: audio parts are transcribed (the
1371
1401
  # text enters the message marked `source: :voice` — the consumer's signal
1372
- # the person SPOKE), image parts become the ask's attachments (the model
1373
- # sees them; the provider bills them — usage flows) and the first URL is
1374
- # deposited as `ctx.image_url` for data/HTTP tools. The engine transports
1375
- # media, never meaning: no speech/vision logic beyond the call itself.
1402
+ # the person SPOKE), image and document parts become the ask's attachments
1403
+ # (the model sees them; the provider bills them — usage flows) and the
1404
+ # first URL of each kind is deposited as `ctx.image_url` / `ctx.document_url`
1405
+ # for data/HTTP tools. The engine transports media, never meaning: no
1406
+ # speech/vision logic beyond the call itself.
1376
1407
  def run_media_stage(task, state)
1377
1408
  # a consumer that pre-transcribed voice text labels it `source: voice`;
1378
1409
  # the marker rides the turn even when there are no audio PARTS left.
@@ -1383,20 +1414,33 @@ module Insika
1383
1414
 
1384
1415
  voice = Insika::Media.audio_parts(parts)
1385
1416
  if voice.any?
1386
- text = voice.map { |p| media_transcribe(p.url) }.reject(&:empty?).join(" ")
1417
+ text = voice.map { |p| media_transcribe(p.url, state) }.reject(&:empty?).join(" ")
1387
1418
  state.message = [state.message.to_s, text].reject(&:empty?).join("\n")
1388
1419
  state.message_source = :voice
1389
1420
  end
1390
1421
 
1391
1422
  images = Insika::Media.image_parts(parts)
1392
1423
  if images.any?
1393
- state.media_attachments = images.map { |p| media_attachment(p.url) }
1424
+ state.image_attachments = images.map { |p| media_attachment(p.url) }
1425
+ state.media_attachments = state.image_attachments
1394
1426
  # First image URL for data tools (`{{ctx.image_url}}`) — photo analysis
1395
1427
  # outside the prompt. The model still sees the attachment; the tool
1396
1428
  # gets the original URL (its own egress applies when it fetches).
1397
1429
  state.turn_context = (state.turn_context || {}).merge(image_url: images.first.url)
1398
1430
  end
1399
1431
 
1432
+ # Documents (a prescription, a recipe, an invoice) ride the SAME
1433
+ # attachments array as images — RubyLLM's `ask(with:)` takes both, and
1434
+ # the attachment content-sniffs PDF magic bytes when the URL has no
1435
+ # extension — capped separately (MAX_DOCUMENT_BYTES) since a document
1436
+ # is not a photo.
1437
+ documents = Insika::Media.document_parts(parts)
1438
+ if documents.any?
1439
+ doc_attachments = documents.map { |p| media_attachment(p.url, max_bytes: Insika::Media::MAX_DOCUMENT_BYTES) }
1440
+ state.media_attachments = Array(state.media_attachments) + doc_attachments
1441
+ state.turn_context = (state.turn_context || {}).merge(document_url: documents.first.url)
1442
+ end
1443
+
1400
1444
  # A media-only turn (a voice note with no caption) is legitimate — the
1401
1445
  # surfaces admit it — but it must leave this stage with something to ask
1402
1446
  # about. Empty text AND no attachment means the parts carried nothing the
@@ -1410,41 +1454,30 @@ module Insika
1410
1454
  # The STT seam: the injected transcriber (specs), else the default
1411
1455
  # (fetch + RubyLLM::Transcription — lazy require). A failed transcription
1412
1456
  # fails the turn loudly (MediaError -> :media): a voice message that was
1413
- # not heard must not become a hallucinated one.
1414
- def media_transcribe(url)
1415
- transcriber = @media || (@default_transcriber ||= Insika::Media.default_transcriber(
1457
+ # not heard must not become a hallucinated one. The default is rebuilt
1458
+ # PER CALL (never memoized) because its vocabulary `prompt:` is resolved
1459
+ # from THIS turn's profile — a memoized seam would freeze the first
1460
+ # agent's prompt (or its absence) for every agent sharing the executor.
1461
+ def media_transcribe(url, state)
1462
+ transcriber = @media || Insika::Media.default_transcriber(
1416
1463
  stt_model: Insika::EnvSchema.read("INSIKA_STT_MODEL"),
1417
- stt_language: Insika::EnvSchema.read("INSIKA_STT_LANGUAGE")
1418
- ))
1464
+ stt_language: Insika::EnvSchema.read("INSIKA_STT_LANGUAGE"),
1465
+ stt_prompt: resolved_stt_prompt(state.profile)
1466
+ )
1419
1467
  transcriber.call(url)
1420
1468
  end
1421
1469
 
1422
- # An image part -> the ask's attachment. RubyLLM required lazily (load-guard).
1423
- #
1424
- # The bytes come through OUR fetch (`Media.fetch_binary`), which is
1425
- # egress-guarded — the URL is CONSUMER input, so a private/metadata target
1426
- # fails the turn loudly at :media — and SIZE-CAPPED. Handing the raw URL to
1427
- # `RubyLLM::Attachment` instead left the fetch to the gem, whose
1428
- # `fetch_content` reads the whole response with no ceiling: a hostile URL
1429
- # answering an endless body grows this process until it dies. An io-like
1430
- # source (StringIO) is the branch of Attachment that takes bytes we already
1431
- # hold; the provider then gets base64 rather than the URL, which every
1432
- # vision provider accepts.
1433
- def media_attachment(url)
1434
- require "ruby_llm"
1435
- require "stringio"
1436
-
1437
- bytes = Insika::Media.fetch_binary(url, max_bytes: Insika::Media::MAX_IMAGE_BYTES)
1438
- RubyLLM::Attachment.new(StringIO.new(bytes), filename: media_filename(url))
1470
+ # Resolution order: the agent profile's own vocabulary hint
1471
+ # (it knows its catalog) beats the deployment-wide default, which beats
1472
+ # nothing (no prompt: kwarg at all).
1473
+ def resolved_stt_prompt(profile)
1474
+ Insika::Coercion.presence(profile&.stt_prompt) || Insika::EnvSchema.read("INSIKA_STT_PROMPT")
1439
1475
  end
1440
1476
 
1441
- # The URL's basename, for the attachment's mime sniff (".png" -> image/png;
1442
- # a URL with no filename falls back to the content sniff RubyLLM does).
1443
- def media_filename(url)
1444
- name = File.basename(URI.parse(url).path.to_s)
1445
- name.empty? ? nil : name
1446
- rescue URI::InvalidURIError
1447
- nil
1477
+ # An image/document part -> the ask's attachment (Insika::Media.url_attachment
1478
+ # — egress-guarded, size-capped fetch; the caller picks the ceiling).
1479
+ def media_attachment(url, max_bytes: Insika::Media::MAX_IMAGE_BYTES)
1480
+ Insika::Media.url_attachment(url, max_bytes: max_bytes)
1448
1481
  end
1449
1482
 
1450
1483
  # --- WS4 intent routing --------------------------------------------
@@ -1634,7 +1667,39 @@ module Insika
1634
1667
  def wire_chat_output(task, state, output)
1635
1668
  chat = state.chat
1636
1669
  chat.after_message { |message| output.message_ended(message) } if chat.respond_to?(:after_message)
1637
- install_steer_injector(task, state)
1670
+ state.steer_injector = install_steer_injector(task, state)
1671
+ end
1672
+
1673
+ # The extra round for a steered message that never met a tool boundary (a
1674
+ # text-only turn closes no batch, so `SteerInjector` never got its cue). The
1675
+ # burst is appended to the history and the model answers ALL of it in one go,
1676
+ # inside the SAME turn — without this the release would spawn a follow-up whose
1677
+ # answer has no delivery channel under request/response.
1678
+ #
1679
+ # Skipped when the turn HALTED: there is no next model step by construction.
1680
+ # `release_steered` (execute's ensure) stays the fallback there, as it is for a
1681
+ # failed or cancelled turn, and for a transport that cannot `complete`.
1682
+ #
1683
+ # At most one extra round: if the model ignores the burst, the turn ends anyway.
1684
+ def absorb_steer_overflow(task, state, timing, result)
1685
+ injector = state.steer_injector
1686
+ return unless injector && state.chat.respond_to?(:complete)
1687
+ return unless state.actor&.user_messages_posted&.positive?
1688
+ return if halted?(result[:response])
1689
+ return if injector.absorb_pending!.zero?
1690
+
1691
+ # TWO provider round trips now. Bank the first one's tokens BEFORE the response
1692
+ # is replaced — stage 6's merge only sees whatever `result[:response]` ends up
1693
+ # being, so without this the extra round silently erases the first round's cost
1694
+ # from the terminal usage and from the EdgeLimiter's budget.
1695
+ state.usage = merge_usage(with_model_source(usage_of(result[:response]), state.model_selection),
1696
+ state.usage)
1697
+ # COMPLETE, not `ask(nil)`: the messages are already in the history, and an
1698
+ # ask with no text appends an EMPTY user message after them — which is both a
1699
+ # duplicate turn-opener and something providers refuse outright.
1700
+ extra = state.chat.complete(&turn_chunk_handler(task, state, result[:output], timing))
1701
+ result[:response] = extra
1702
+ result[:asked] = extra
1638
1703
  end
1639
1704
 
1640
1705
  # The ask itself, chunk-by-chunk (WS3 attempts and the plain path share it).
@@ -1644,9 +1709,27 @@ module Insika
1644
1709
  # :ttft on EVERY content chunk (3 chunks = 3 insika.ttft frames); the spec
1645
1710
  # passed because FakeChat emits a single chunk.
1646
1711
  def ask_on(task, state, chat, output, timing)
1712
+ each_chunk = turn_chunk_handler(task, state, output, timing)
1713
+ # WS9: image parts ride the ask as attachments (only then — a chat whose
1714
+ # ask has no `with:` keeps working, and the plain path is byte-identical).
1715
+ # An image with no caption asks with NIL, not "": an empty text part is a
1716
+ # thing some providers refuse, and nil is how RubyLLM says "attachments
1717
+ # only".
1718
+ if state.media_attachments
1719
+ text = state.message.to_s.empty? ? nil : state.message
1720
+ chat.ask(text, with: state.media_attachments, &each_chunk)
1721
+ else
1722
+ chat.ask(state.message, &each_chunk)
1723
+ end
1724
+ end
1725
+
1726
+ # The per-round chunk sink. Built FRESH per round: `ttft_sent` is that round's
1727
+ # own bookkeeping, and the steer overflow round is a second round on the same
1728
+ # chat.
1729
+ def turn_chunk_handler(task, state, output, timing)
1647
1730
  public_thinking = state.profile.stream_public?(:thinking)
1648
1731
  ttft_sent = false
1649
- each_chunk = lambda do |chunk|
1732
+ lambda do |chunk|
1650
1733
  emit_thinking(chunk, task, public: public_thinking)
1651
1734
  next unless chunk.content
1652
1735
 
@@ -1657,17 +1740,6 @@ module Insika
1657
1740
  end
1658
1741
  output.push(chunk.content)
1659
1742
  end
1660
- # WS9: image parts ride the ask as attachments (only then — a chat whose
1661
- # ask has no `with:` keeps working, and the plain path is byte-identical).
1662
- # An image with no caption asks with NIL, not "": an empty text part is a
1663
- # thing some providers refuse, and nil is how RubyLLM says "attachments
1664
- # only".
1665
- if state.media_attachments
1666
- text = state.message.to_s.empty? ? nil : state.message
1667
- chat.ask(text, with: state.media_attachments, &each_chunk)
1668
- else
1669
- chat.ask(state.message, &each_chunk)
1670
- end
1671
1743
  end
1672
1744
 
1673
1745
  # The provider's TTFB as a live event (data: ttft_ms) — only under
@@ -1950,10 +2022,17 @@ module Insika
1950
2022
  # the memory scope carries a customer — the backend identifies the store,
1951
2023
  # not the shopper (WS8 keeps the two scopes separate).
1952
2024
  tenant: command_tenant(task) || task.session_id,
2025
+ # the DECLARED tenant alone (nil in single-tenant). Distinct from
2026
+ # `tenant` for the ownership-binding tools (save_artifact): a report
2027
+ # belongs to the AGENT's tenant — the deployment's tenant in
2028
+ # single-tenant, never the chat that happened to run it.
2029
+ command_tenant: command_tenant(task),
1953
2030
  store_id: profile.store_id,
1954
- # current delegation depth (0 for a top-level turn). Carried in
1955
- # the child command's payload by run_subagent; read here so the child's OWN
1956
- # spawn_subagent tool sees depth+1 and the runtime cap holds down the chain.
2031
+ # the current task id — the save_artifact binding (which run produced
2032
+ # this report), same turn-origin discipline as the rest of ctx.*.
2033
+ task_id: task.id,
2034
+ # current delegation depth (0 for a top-level turn). Set by run_subagent
2035
+ # for children.
1957
2036
  delegation_depth: delegation_depth(task)
1958
2037
  }
1959
2038
  end
@@ -2391,6 +2470,12 @@ module Insika
2391
2470
  # has to travel out of band. Same terminal hook, next door to the delegation
2392
2471
  # one, for the same reason: it fires for a fresh turn and a recovered one.
2393
2472
  finalize_channel_delivery(task, content, state, timing)
2473
+
2474
+ # extracts durable concepts from this turn, off the critical
2475
+ # path — the user already has the answer above. Same terminal hook,
2476
+ # next door to the other two, for the same reason: it fires for a fresh
2477
+ # turn and a recovered one.
2478
+ finalize_knowledge_extraction(task, profile, new_messages)
2394
2479
  end
2395
2480
 
2396
2481
  # Records the answer in the outbox and dispatches it. The discriminator is the
@@ -2433,6 +2518,95 @@ module Insika
2433
2518
  nil
2434
2519
  end
2435
2520
 
2521
+ # Turns whose combined transcript slice is this trivially short skip
2522
+ # extraction entirely ("ok thanks" exchanges) — no new config surface,
2523
+ # just avoids a wasted utility-model call.
2524
+ KNOWLEDGE_MIN_CHARS = 200
2525
+
2526
+ # No-op without a knowledge store, without the profile's opt-in
2527
+ # (`knowledge.extract`), without a usable model, or for a trivially short
2528
+ # turn. Otherwise dispatches the extraction off the critical path — the
2529
+ # SAME `dispatch_chain` shape: inline when non-supervised (tests, CLI,
2530
+ # boot sweep), a child of the turn supervisor when serving (survives the
2531
+ # request's own disconnect). Best-effort: any failure is swallowed here,
2532
+ # never re-fails an already-committed turn.
2533
+ def finalize_knowledge_extraction(task, profile, new_messages)
2534
+ return unless @knowledge_store
2535
+
2536
+ config = Coercion.deep_stringify(profile.knowledge)
2537
+ return unless config && Coercion.truthy?(config["extract"])
2538
+ return if knowledge_transcript(new_messages).length < KNOWLEDGE_MIN_CHARS
2539
+
2540
+ extractor = Knowledge::ExtractorFactory.build(config, utility_model: utility_model)
2541
+ return unless extractor
2542
+
2543
+ # Same resolved model as the extractor — a deployment names one
2544
+ # knowledge model, not two. nil consolidator (no model resolvable) is
2545
+ # still meaningful: write_concept's conservative default.
2546
+ consolidator = Knowledge::ConsolidatorFactory.build(config, utility_model: utility_model)
2547
+
2548
+ run = lambda { run_knowledge_extraction(task, profile, config, new_messages, extractor, consolidator) }
2549
+ return run.call unless @supervised
2550
+
2551
+ turn_parent.async do |t|
2552
+ t.annotate("knowledge:#{task.id}")
2553
+ run.call
2554
+ end
2555
+ end
2556
+
2557
+ def run_knowledge_extraction(task, profile, config, new_messages, extractor, consolidator)
2558
+ prompt = knowledge_prompt(config, new_messages)
2559
+ result = extractor.extract(prompt: prompt)
2560
+ result[:concepts].each do |concept|
2561
+ outcome = Knowledge.write_concept(
2562
+ store: @knowledge_store, agent_id: profile.id, concept: concept, session_id: task.session_id,
2563
+ tenant: task_tenant(task), consolidator: consolidator
2564
+ )
2565
+ emit_knowledge_event(outcome, profile, task)
2566
+ end
2567
+ rescue StandardError
2568
+ nil # best-effort: extraction never re-fails an already-committed turn.
2569
+ end
2570
+
2571
+ # :new/:related taught the agent something; :contradicting needs a
2572
+ # human; :same is a silent reinforcement (no event — a popular concept
2573
+ # would otherwise spam the stream every time it's confirmed).
2574
+ def emit_knowledge_event(outcome, profile, task)
2575
+ case outcome[:verdict]
2576
+ when :new, :related
2577
+ emit(:knowledge_learned, { name: outcome[:name], type: outcome[:type], agent: profile.id }, task: task)
2578
+ when :contradicting
2579
+ emit(:knowledge_conflict, { name: outcome[:name], agent: profile.id }, task: task)
2580
+ end
2581
+ end
2582
+
2583
+ def knowledge_prompt(config, new_messages)
2584
+ base = Coercion.presence(config["prompt"]) || Knowledge::DEFAULT_PROMPT
2585
+ <<~PROMPT
2586
+ #{base.rstrip}
2587
+
2588
+ ## The conversation
2589
+
2590
+ #{knowledge_transcript(new_messages)}
2591
+ PROMPT
2592
+ end
2593
+
2594
+ # Redacted (RFC's PII rule applies to what reaches the model too, not
2595
+ # just what gets persisted).
2596
+ def knowledge_transcript(new_messages)
2597
+ redacted, = Insika::Safety::Detectors.redact(
2598
+ new_messages.each_with_index.map { |m, i| "[#{i}] #{m['role'] || m[:role]}: #{m['content'] || m[:content]}" }
2599
+ .join("\n")
2600
+ )
2601
+ redacted
2602
+ end
2603
+
2604
+ def utility_model
2605
+ return nil unless @settings_store
2606
+
2607
+ @settings_store.get["utility_model"]
2608
+ end
2609
+
2436
2610
  # ONE supervisor fiber for the whole chain. Sequential deliver
2437
2611
  # calls, so balloon N+1 cannot overtake balloon N on the wire. Still off the
2438
2612
  # session's FIFO — the customer's next message does not wait on this turn's
@@ -2546,11 +2720,13 @@ module Insika
2546
2720
 
2547
2721
  # Stage 6 (factory): the ONLY point that touches the gem. lazy require,
2548
2722
  # confined — not covered by unit (factory line). It also loads the system
2549
- # builtins (load_skill/tool_search/remember) that the ChatBuilder assembles at
2550
- # stage 5 — lazy, so the core installs without ruby_llm.
2723
+ # builtins (load_skill/load_knowledge/tool_search/remember) that the
2724
+ # ChatBuilder assembles at stage 5 — lazy, so the core installs without
2725
+ # ruby_llm.
2551
2726
  def create_chat(profile, state)
2552
2727
  require "ruby_llm"
2553
2728
  require_relative "tools/load_skill"
2729
+ require_relative "tools/load_knowledge"
2554
2730
  require_relative "tools/tool_search"
2555
2731
  require_relative "tools/remember"
2556
2732
  require_relative "tools/subagent"
@@ -70,7 +70,7 @@ module Insika
70
70
  def quiet?(time)
71
71
  return false unless @quiet_hours
72
72
 
73
- local = in_zone(@quiet_hours.timezone, time) { |t| t.getlocal }
73
+ local = Insika::Timezone.in_zone(@quiet_hours.timezone, time) { |t| t.getlocal }
74
74
  minutes = local.hour * 60 + local.min
75
75
  start_min = minutes_of(@quiet_hours.start)
76
76
  end_min = minutes_of(@quiet_hours.end)
@@ -136,7 +136,7 @@ module Insika
136
136
  # a bogus IANA zone is a malformed policy — refused HERE, where the
137
137
  # doctor can name it. An unknown ENV["TZ"] silently behaves as UTC, so
138
138
  # existence is checked against the OS tz database, not by asking Time.
139
- unless zone_known?(timezone.to_s)
139
+ unless Insika::Timezone.known?(timezone.to_s)
140
140
  raise Insika::ValidationError,
141
141
  "followup.policy.quiet_hours.timezone is not a valid IANA timezone: #{timezone.inspect}"
142
142
  end
@@ -190,29 +190,6 @@ module Insika
190
190
  text.to_s.unicode_normalize(:nfd).gsub(/\p{Mn}/, "").downcase
191
191
  end
192
192
 
193
- # Yields `time` interpreted in the given IANA zone (via a save/restore of
194
- # ENV["TZ"] — the stdlib-only route to the OS tz database; see #quiet?).
195
- def in_zone(zone, time)
196
- previous = ENV["TZ"]
197
- ENV["TZ"] = zone
198
- yield time
199
- ensure
200
- ENV["TZ"] = previous
201
- end
202
-
203
- # The candidate tz-data roots (TZDIR first — Ruby's own lookup env). The
204
- # zone name maps to a FILE under the root ("America/Sao_Paulo" ->
205
- # "America/Sao_Paulo").
206
- TZ_ROOTS = ([ENV["TZDIR"]] +
207
- %w[/usr/share/zoneinfo /usr/share/lib/zoneinfo /etc/zoneinfo])
208
- .compact.freeze
209
-
210
- def zone_known?(zone)
211
- return true if zone == "UTC" || zone == "Etc/UTC"
212
-
213
- TZ_ROOTS.any? { |root| File.directory?(root) && File.exist?(File.join(root, zone)) }
214
- end
215
-
216
193
  def minutes_of(hhmm)
217
194
  h, m = hhmm.split(":").map(&:to_i)
218
195
  h * 60 + m
@@ -63,6 +63,11 @@ module Insika
63
63
  # -> [Evals::Golden] one agent's cases.
64
64
  def for_agent(agent_id) = all.select { |g| g.agent == agent_id.to_s }
65
65
 
66
+ # -> [Evals::Golden] one tenant's cases (C3.1's isolation query —
67
+ # `run_persona_eval` never lists or runs a case outside the calling agent's
68
+ # own tenant, "platform" being the single-tenant default).
69
+ def for_tenant(tenant) = all.select { |g| g.tenant == tenant.to_s }
70
+
66
71
  # -> [String] ids whose stored mapping no longer validates (an edit that broke the
67
72
  # shape). The Studio shows these; they never silently vanish from a run.
68
73
  def invalid
@@ -130,13 +135,23 @@ module Insika
130
135
  # in the store as it is on disk — but it is NEVER dropped when present: a case
131
136
  # that silently lost its requirements would come back as a failure on every
132
137
  # deployment that lacks the tool, which is the exact lie `requires` exists to end.
138
+ # Same for `persona`: a simulated case that lost its persona would
139
+ # silently stop being simulated and replay as an empty script.
133
140
  def case_hash(golden)
134
- h = { "id" => golden.id, "agent" => golden.agent, "turns" => golden.turns }
141
+ h = if golden.simulated?
142
+ { "id" => golden.id, "agent" => golden.agent, "persona" => golden.persona.to_h }
143
+ else
144
+ { "id" => golden.id, "agent" => golden.agent, "turns" => golden.turns }
145
+ end
135
146
  h["requires"] = golden.requires unless golden.requires.empty?
136
147
  # Same rule for `reference`: omitted when absent, never dropped
137
148
  # when present. A case that lost its reference in a round-trip would stop being
138
149
  # compared against the incumbent and the report would look identical.
139
150
  h["reference"] = golden.reference unless golden.reference.empty?
151
+ # `tenant` follows the same omit-when-default rule: "platform" is the
152
+ # loader's own default, so leaving it out reproduces the same case; an
153
+ # explicit tenant is a case's isolation boundary and must never round-trip away.
154
+ h["tenant"] = golden.tenant unless golden.tenant == "platform"
140
155
  h.merge("expect" => golden.expect)
141
156
  end
142
157
 
@@ -7,7 +7,7 @@ module Insika
7
7
  # The pack's DATA (D7): a `sku` regex for the SKU shape. The engine only
8
8
  # APPLIES it — it never guesses what a SKU looks like. No NLP, no LLM, no gem.
9
9
  #
10
- # REVIEW-DECISION (rfc-0029 v2): grounding is **SKU-only**. The techspec's
10
+ # REVIEW-DECISION: grounding is **SKU-only**. The techspec's
11
11
  # name half (name_keys indexing ledger lines to flag names) cannot flag
12
12
  # anything: a name found in the text that also came from the ledger is grounded
13
13
  # by construction, and detecting a name that matches NO ledger line needs a