insika 0.8.0 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (276) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +312 -2
  3. data/README.md +10 -6
  4. data/bin/insika +4 -1
  5. data/bin/insika-router +0 -1
  6. data/docs/AGENTS.md +142 -18
  7. data/docs/API.md +73 -0
  8. data/docs/ARCHITECTURE.md +56 -50
  9. data/docs/ARTIFACTS.md +33 -3
  10. data/docs/BENCHMARK.md +50 -7
  11. data/docs/CHANNELS.md +19 -2
  12. data/docs/CONTEXT.md +96 -27
  13. data/docs/CROSS-HARNESS-BENCH.md +157 -0
  14. data/docs/DEPLOY.md +70 -13
  15. data/docs/EMBEDDING.md +10 -0
  16. data/docs/EVALS.md +129 -10
  17. data/docs/FACTS.md +12 -6
  18. data/docs/HARVEST.md +8 -3
  19. data/docs/KNOWLEDGE.md +14 -0
  20. data/docs/MEDIA.md +6 -2
  21. data/docs/OBSERVABILITY.md +164 -28
  22. data/docs/PERFORMANCE.md +110 -0
  23. data/docs/PLUGINS.md +4 -4
  24. data/docs/POLICY.md +26 -14
  25. data/docs/RELEASING.md +9 -0
  26. data/docs/ROUTER.md +7 -0
  27. data/docs/RUBYLLM_2_MIGRATION.md +307 -0
  28. data/docs/SECURITY.md +34 -1
  29. data/docs/SHARED-CONVERSATIONS.md +40 -0
  30. data/docs/SKILLS.md +5 -3
  31. data/docs/STUDIO.md +122 -0
  32. data/docs/TOOLS.md +238 -42
  33. data/docs/operate.md +3 -0
  34. data/docs/prompts/ADD-TOOL.md +12 -2
  35. data/docs/prompts/DIAGNOSE-TURN.md +3 -0
  36. data/docs/prompts/GO-LIVE.md +3 -1
  37. data/lib/insika/agent_defaults.rb +140 -0
  38. data/lib/insika/agent_file_store.rb +12 -0
  39. data/lib/insika/agent_profile.rb +99 -17
  40. data/lib/insika/artifact_signing.rb +11 -0
  41. data/lib/insika/channels/web/widget.js +33 -0
  42. data/lib/insika/channels/web.rb +5 -2
  43. data/lib/insika/chat_builder.rb +124 -28
  44. data/lib/insika/checkpoint.rb +6 -1
  45. data/lib/insika/checkpoint_store.rb +59 -7
  46. data/lib/insika/commands/agent_payload.rb +3 -3
  47. data/lib/insika/commands/delete_kit.rb +26 -0
  48. data/lib/insika/commands/gate_harvest.rb +3 -0
  49. data/lib/insika/commands/review_guardrail_flag.rb +38 -0
  50. data/lib/insika/commands/run_distillation.rb +5 -8
  51. data/lib/insika/commands/run_harvest.rb +17 -1
  52. data/lib/insika/commands/seed_session.rb +128 -0
  53. data/lib/insika/commands/send_message.rb +9 -3
  54. data/lib/insika/commands/update_settings.rb +23 -1
  55. data/lib/insika/commands/write_kit.rb +27 -0
  56. data/lib/insika/compaction.rb +33 -7
  57. data/lib/insika/config_store.rb +87 -7
  58. data/lib/insika/context/builder.rb +29 -9
  59. data/lib/insika/context/priority.rb +5 -0
  60. data/lib/insika/context/provider.rb +7 -2
  61. data/lib/insika/context/providers/briefing.rb +11 -8
  62. data/lib/insika/context/providers/fence_notice.rb +27 -0
  63. data/lib/insika/context/providers/knowledge.rb +32 -6
  64. data/lib/insika/context/providers/memory.rb +61 -8
  65. data/lib/insika/context/providers/pending_confirmation.rb +52 -0
  66. data/lib/insika/context/providers/prompt.rb +17 -4
  67. data/lib/insika/context/providers/session.rb +46 -6
  68. data/lib/insika/context/reranker.rb +75 -0
  69. data/lib/insika/distill.rb +22 -13
  70. data/lib/insika/doctor.rb +142 -4
  71. data/lib/insika/dsl/runtime.rb +37 -4
  72. data/lib/insika/dsl/server_boot.rb +4 -12
  73. data/lib/insika/dsl.rb +19 -5
  74. data/lib/insika/edge_limiter.rb +27 -16
  75. data/lib/insika/egress_guard.rb +11 -3
  76. data/lib/insika/env_schema.rb +33 -73
  77. data/lib/insika/errors.rb +1 -0
  78. data/lib/insika/evals/assertions.rb +259 -18
  79. data/lib/insika/evals/golden.rb +230 -4
  80. data/lib/insika/evals/judge.rb +19 -10
  81. data/lib/insika/evals/moderator_compare.rb +47 -0
  82. data/lib/insika/evals/report.rb +61 -9
  83. data/lib/insika/evals/runner.rb +64 -4
  84. data/lib/insika/evals/simulator.rb +21 -3
  85. data/lib/insika/evals/transport.rb +317 -15
  86. data/lib/insika/event_stream.rb +63 -7
  87. data/lib/insika/evidence.rb +105 -17
  88. data/lib/insika/executor.rb +698 -164
  89. data/lib/insika/fence.rb +96 -0
  90. data/lib/insika/golden_store.rb +3 -0
  91. data/lib/insika/harvest.rb +26 -14
  92. data/lib/insika/harvest_engine.rb +1 -0
  93. data/lib/insika/harvest_store.rb +12 -0
  94. data/lib/insika/http_client.rb +87 -4
  95. data/lib/insika/kits.rb +54 -0
  96. data/lib/insika/knowledge.rb +34 -20
  97. data/lib/insika/llm_configurator.rb +46 -4
  98. data/lib/insika/llm_provider_store.rb +1 -3
  99. data/lib/insika/llm_trace_store.rb +54 -0
  100. data/lib/insika/mcp_client.rb +27 -63
  101. data/lib/insika/mcp_live_tool.rb +55 -9
  102. data/lib/insika/mcp_store.rb +37 -3
  103. data/lib/insika/mcp_tool_registry.rb +76 -26
  104. data/lib/insika/media.rb +38 -25
  105. data/lib/insika/memory_store.rb +16 -3
  106. data/lib/insika/memory_trim.rb +60 -0
  107. data/lib/insika/middleware.rb +12 -3
  108. data/lib/insika/model_metrics_store.rb +285 -0
  109. data/lib/insika/model_registry.rb +110 -0
  110. data/lib/insika/model_resolver.rb +1 -1
  111. data/lib/insika/model_selection.rb +14 -21
  112. data/lib/insika/model_visible.rb +17 -6
  113. data/lib/insika/onboarding.rb +1 -0
  114. data/lib/insika/outcome_store.rb +1 -1
  115. data/lib/insika/overlay_tool_registry.rb +30 -2
  116. data/lib/insika/pack_importer.rb +54 -9
  117. data/lib/insika/packaging.rb +14 -5
  118. data/lib/insika/pending_action_store.rb +45 -18
  119. data/lib/insika/plugin/loader.rb +4 -2
  120. data/lib/insika/prefix_fingerprint.rb +32 -27
  121. data/lib/insika/profile_source.rb +5 -0
  122. data/lib/insika/provider_error_classifier.rb +18 -3
  123. data/lib/insika/queue_policy.rb +7 -2
  124. data/lib/insika/refinement/proposer.rb +42 -17
  125. data/lib/insika/reliability.rb +12 -3
  126. data/lib/insika/router/app.rb +14 -5
  127. data/lib/insika/router/session_key.rb +35 -5
  128. data/lib/insika/safety/config.rb +20 -3
  129. data/lib/insika/safety/corpus.rb +10 -3
  130. data/lib/insika/safety/factory.rb +39 -1
  131. data/lib/insika/safety/input_guardrail.rb +21 -1
  132. data/lib/insika/safety/judge_moderator.rb +70 -0
  133. data/lib/insika/safety/output_validator.rb +29 -5
  134. data/lib/insika/safety/prompt_echo.rb +89 -0
  135. data/lib/insika/schema_guard.rb +53 -76
  136. data/lib/insika/server/app.rb +58 -7
  137. data/lib/insika/server/rack_app.rb +2 -0
  138. data/lib/insika/server/responses.rb +50 -5
  139. data/lib/insika/session_actor.rb +58 -28
  140. data/lib/insika/session_store.rb +163 -11
  141. data/lib/insika/settings_store.rb +57 -2
  142. data/lib/insika/shared_conversations.rb +305 -0
  143. data/lib/insika/skill_catalog.rb +28 -2
  144. data/lib/insika/skill_store.rb +6 -0
  145. data/lib/insika/soak/runner.rb +1 -1
  146. data/lib/insika/spoken_transcript.rb +31 -0
  147. data/lib/insika/steer_injector.rb +7 -48
  148. data/lib/insika/store.rb +29 -1
  149. data/lib/insika/stores/json_value.rb +37 -0
  150. data/lib/insika/stores/memory.rb +22 -35
  151. data/lib/insika/stores/postgres.rb +295 -0
  152. data/lib/insika/stores/sqlite.rb +257 -43
  153. data/lib/insika/stores/turn_counter.rb +25 -0
  154. data/lib/insika/studio/app.rb +1459 -163
  155. data/lib/insika/studio/assets/dist/application.css +1 -1
  156. data/lib/insika/studio/assets/dist/application.js +25 -25
  157. data/lib/insika/studio/assets/dist/fonts/ibm-plex-mono-latin-400-normal.woff2 +0 -0
  158. data/lib/insika/studio/assets/dist/fonts/ibm-plex-mono-latin-500-normal.woff2 +0 -0
  159. data/lib/insika/studio/assets/dist/fonts/ibm-plex-mono-latin-600-normal.woff2 +0 -0
  160. data/lib/insika/studio/assets/dist/fonts/ibm-plex-sans-latin-wght-normal.woff2 +0 -0
  161. data/lib/insika/studio/forms.rb +189 -12
  162. data/lib/insika/studio/nav_icons.rb +3 -0
  163. data/lib/insika/studio/views/_agent_config_retrieval.erb +73 -0
  164. data/lib/insika/studio/views/_agent_picker.erb +15 -0
  165. data/lib/insika/studio/views/_agent_tab_config.erb +83 -243
  166. data/lib/insika/studio/views/_agent_tab_history.erb +56 -1
  167. data/lib/insika/studio/views/_agent_tab_performance.erb +34 -0
  168. data/lib/insika/studio/views/_agent_tab_prompts.erb +7 -9
  169. data/lib/insika/studio/views/_agent_tab_skills.erb +16 -7
  170. data/lib/insika/studio/views/_agent_tab_tools.erb +90 -0
  171. data/lib/insika/studio/views/_agents_master.erb +8 -12
  172. data/lib/insika/studio/views/_cfg_alerts.erb +4 -0
  173. data/lib/insika/studio/views/_cfg_budget.erb +20 -0
  174. data/lib/insika/studio/views/_cfg_distill.erb +26 -0
  175. data/lib/insika/studio/views/_cfg_followup.erb +30 -0
  176. data/lib/insika/studio/views/_cfg_grounding.erb +21 -0
  177. data/lib/insika/studio/views/_cfg_guardrails.erb +47 -0
  178. data/lib/insika/studio/views/_cfg_harvest.erb +35 -0
  179. data/lib/insika/studio/views/_cfg_reliability.erb +32 -0
  180. data/lib/insika/studio/views/_chats_master.erb +66 -0
  181. data/lib/insika/studio/views/_customers_master.erb +37 -0
  182. data/lib/insika/studio/views/_empty.erb +15 -0
  183. data/lib/insika/studio/views/_facts_master.erb +42 -0
  184. data/lib/insika/studio/views/_harvest_candidate_detail.erb +14 -0
  185. data/lib/insika/studio/views/_identity.erb +16 -0
  186. data/lib/insika/studio/views/_kpi.erb +17 -0
  187. data/lib/insika/studio/views/_message.erb +40 -2
  188. data/lib/insika/studio/views/_model_chart.erb +82 -0
  189. data/lib/insika/studio/views/_refinement_master.erb +38 -0
  190. data/lib/insika/studio/views/{tool_edit.erb → _tool_edit.erb} +38 -11
  191. data/lib/insika/studio/views/_tool_users.erb +17 -0
  192. data/lib/insika/studio/views/_why.erb +53 -0
  193. data/lib/insika/studio/views/agent_detail.erb +17 -10
  194. data/lib/insika/studio/views/agents.erb +28 -16
  195. data/lib/insika/studio/views/approvals.erb +16 -11
  196. data/lib/insika/studio/views/artifact.erb +20 -7
  197. data/lib/insika/studio/views/artifacts.erb +19 -14
  198. data/lib/insika/studio/views/chats.erb +22 -29
  199. data/lib/insika/studio/views/customer.erb +88 -43
  200. data/lib/insika/studio/views/customers.erb +16 -26
  201. data/lib/insika/studio/views/evals.erb +49 -31
  202. data/lib/insika/studio/views/fact_detail.erb +114 -0
  203. data/lib/insika/studio/views/facts.erb +15 -116
  204. data/lib/insika/studio/views/followups.erb +63 -52
  205. data/lib/insika/studio/views/funnel.erb +104 -43
  206. data/lib/insika/studio/views/harvest.erb +85 -59
  207. data/lib/insika/studio/views/home.erb +138 -23
  208. data/lib/insika/studio/views/inbox.erb +36 -0
  209. data/lib/insika/studio/views/kits.erb +176 -0
  210. data/lib/insika/studio/views/knowledge.erb +92 -80
  211. data/lib/insika/studio/views/layout.erb +42 -11
  212. data/lib/insika/studio/views/live.erb +45 -0
  213. data/lib/insika/studio/views/mcp.erb +43 -13
  214. data/lib/insika/studio/views/models.erb +187 -0
  215. data/lib/insika/studio/views/parity.erb +38 -32
  216. data/lib/insika/studio/views/playground.erb +13 -9
  217. data/lib/insika/studio/views/refinement.erb +15 -224
  218. data/lib/insika/studio/views/refinement_detail.erb +243 -0
  219. data/lib/insika/studio/views/session.erb +153 -118
  220. data/lib/insika/studio/views/settings.erb +86 -22
  221. data/lib/insika/studio/views/skills.erb +53 -19
  222. data/lib/insika/studio/views/system_files.erb +7 -7
  223. data/lib/insika/studio/views/task.erb +45 -15
  224. data/lib/insika/studio/views/tasks.erb +35 -18
  225. data/lib/insika/studio/views/tools.erb +62 -73
  226. data/lib/insika/task_actor.rb +4 -1
  227. data/lib/insika/task_store.rb +67 -1
  228. data/lib/insika/telemetry/pricing.rb +16 -14
  229. data/lib/insika/telemetry/recorder.rb +115 -9
  230. data/lib/insika/telemetry/ruby_llm_instrumenter.rb +110 -0
  231. data/lib/insika/telemetry.rb +12 -2
  232. data/lib/insika/templates/daily-digest/README.md +13 -0
  233. data/lib/insika/templates/daily-digest/agent.rb +22 -11
  234. data/lib/insika/testing/store_contract.rb +65 -0
  235. data/lib/insika/text_tool_call.rb +66 -0
  236. data/lib/insika/tick.rb +7 -1
  237. data/lib/insika/token_store.rb +5 -0
  238. data/lib/insika/tool_assembly.rb +21 -13
  239. data/lib/insika/tool_batch.rb +1 -1
  240. data/lib/insika/tool_catalog.rb +12 -2
  241. data/lib/insika/tool_definition.rb +82 -13
  242. data/lib/insika/tool_envelope.rb +266 -29
  243. data/lib/insika/tool_manifest.rb +1 -0
  244. data/lib/insika/tool_store.rb +12 -4
  245. data/lib/insika/tool_trace_store.rb +15 -8
  246. data/lib/insika/tool_usage_report.rb +14 -4
  247. data/lib/insika/tools/a2a_remote.rb +1 -1
  248. data/lib/insika/tools/agent_enum.rb +1 -1
  249. data/lib/insika/tools/confirm_pending.rb +112 -0
  250. data/lib/insika/tools/data_defined_tool.rb +24 -15
  251. data/lib/insika/tools/generate_image.rb +2 -2
  252. data/lib/insika/tools/load_knowledge.rb +7 -3
  253. data/lib/insika/tools/load_skill.rb +3 -3
  254. data/lib/insika/tools/present.rb +115 -0
  255. data/lib/insika/tools/remember.rb +7 -2
  256. data/lib/insika/tools/run_persona_eval.rb +9 -6
  257. data/lib/insika/tools/save_artifact.rb +16 -6
  258. data/lib/insika/tools/schedule_followup.rb +3 -3
  259. data/lib/insika/tools/stuck_signal.rb +3 -3
  260. data/lib/insika/tools/subagent.rb +5 -5
  261. data/lib/insika/tools/subagents.rb +3 -3
  262. data/lib/insika/tools/tool_search.rb +11 -7
  263. data/lib/insika/tools/tts.rb +2 -2
  264. data/lib/insika/tools/update_briefing.rb +4 -4
  265. data/lib/insika/turn_output.rb +2 -2
  266. data/lib/insika/turn_state.rb +31 -1
  267. data/lib/insika/turn_timing.rb +51 -14
  268. data/lib/insika/version.rb +1 -1
  269. data/lib/insika/wiring/graph.rb +88 -16
  270. data/lib/insika/wiring/graph_chat.rb +4 -0
  271. data/lib/insika/workflow.rb +25 -64
  272. data/lib/insika.rb +23 -3
  273. metadata +74 -13
  274. data/lib/insika/commands/import_mcp_tools.rb +0 -48
  275. data/lib/insika/mcp_http_client.rb +0 -67
  276. data/lib/insika/mcp_tool_ingestor.rb +0 -165
data/lib/insika/dsl.rb CHANGED
@@ -122,13 +122,13 @@ module Insika
122
122
  # `agent { }` block that needs it instead, where `Builder#mcp` auto-adds
123
123
  # "mcp:<name>" to THAT agent's `tools_allow_groups` (below).
124
124
  def mcp(name, transport: nil, command: nil, args: nil, url: nil,
125
- headers: nil, env: nil, description: nil, enabled: true)
125
+ headers: nil, env: nil, description: nil, enabled: true, tools: nil)
126
126
  n = name.to_s
127
127
  raise ArgumentError, "duplicate mcp instance in system: #{n}" if @mcp_instances.any? { |m| m[:name] == n }
128
128
 
129
129
  @mcp_instances << { name: n, transport: transport&.to_s, command: command, args: args,
130
130
  url: url, headers: headers, env: env, description: description,
131
- enabled: enabled }
131
+ enabled: enabled, tools: tools }
132
132
  n
133
133
  end
134
134
  end
@@ -251,20 +251,22 @@ module Insika
251
251
  # gives its own tool name; `deny_tools` has no group-string equivalent
252
252
  # yet, so a whole MCP group cannot be denied by name today.
253
253
  def mcp(name, transport: nil, command: nil, args: nil, url: nil,
254
- headers: nil, env: nil, description: nil, enabled: true)
254
+ headers: nil, env: nil, description: nil, enabled: true, tools: nil)
255
255
  n = name.to_s
256
256
  raise ArgumentError, "duplicate mcp instance in agent: #{n}" if @mcp_instances.any? { |m| m[:name] == n }
257
257
 
258
258
  @mcp_instances << { name: n, transport: transport&.to_s, command: command, args: args,
259
259
  url: url, headers: headers, env: env, description: description,
260
- enabled: enabled }
260
+ enabled: enabled, tools: tools }
261
261
  group = "mcp:#{n}"
262
262
  (@config[:tools_allow_groups] ||= []) << group unless Array(@config[:tools_allow_groups]).include?(group)
263
263
  n
264
264
  end
265
265
 
266
266
  # --- knobs -----------------------------------------------------------
267
+ def shared_conversations(on = true) = @config[:shared_conversations] = on
267
268
  def memory(on = true) = @config[:memory] = on
269
+ def memory_retrieval(hash) = (@config[:memory_retrieval] ||= {}).merge!(hash.transform_keys(&:to_s))
268
270
 
269
271
  # Spend caps per calendar window (WS2): daily/monthly token budgets for
270
272
  # this agent, per (tenant, agent) when multi-tenant. HARD is the default:
@@ -403,7 +405,7 @@ module Insika
403
405
  # the transcriber should expect on THIS agent's voice notes — passed
404
406
  # straight through to the Whisper-family provider's `prompt:`. Falls
405
407
  # back to INSIKA_STT_PROMPT (deployment default) when unset.
406
- # stt_prompt "Ocean Drop, tênis, boné trucker, chinelo"
408
+ # stt_prompt "Acme Surf, tênis, boné trucker, chinelo"
407
409
  def stt_prompt(text) = @config[:stt_prompt] = text.to_s
408
410
 
409
411
  # The engine's "Tool discipline" block in the system prompt (retry a
@@ -417,6 +419,18 @@ module Insika
417
419
  # SEES: repeated identical tool results collapse to a back-reference.
418
420
  def tool_output_compression(on = true) = @config[:tool_output_compression] = on
419
421
 
422
+ # Third-party text is sanitized before the model reads it (tool results,
423
+ # memory, knowledge) and a one-sentence notice names those blocks as data.
424
+ # Opt-in this release — the goldens were baselined on unfenced bytes:
425
+ # fencing true
426
+ def fencing(on = true) = @config[:fencing] = on
427
+
428
+ # Tools the CUSTOMER confirms. The call is held, the reply asks, and the
429
+ # next message decides through confirm_pending / cancel_pending. Unlike an
430
+ # operator approval the turn ENDS instead of suspending.
431
+ # customer_confirm "create_order"
432
+ def customer_confirm(*names) = @config[:customer_confirm] = names.flatten.map(&:to_s)
433
+
420
434
  # Content-safety guardrails — opt-in and configurable per agent.
421
435
  # Pure config-over-code: the hash is stored on the profile and consumed by
422
436
  # Safety::Config.from_profile. Merges, so repeated calls accumulate.
@@ -57,7 +57,7 @@ module Insika
57
57
  @event_stream = event_stream
58
58
  end
59
59
 
60
- def call(state, &nxt)
60
+ def call(state, prepare: nil, &nxt)
61
61
  edge = platform_edge
62
62
  limits = state.profile.limits || {}
63
63
  # A resume (crash/pause recovery) re-enters the pipeline for a turn that was
@@ -100,9 +100,12 @@ module Insika
100
100
  # turn rides the same rule: the follow-up policy is the
101
101
  # volume control, not the budget wall.
102
102
  budget_on = budget_configured?(state)
103
- budget_enforce(state) unless resumed || scheduled
103
+ budget_notes = []
104
+ budget_enforce(state, notes: budget_notes) unless resumed || scheduled
104
105
 
105
106
  result = begin
107
+ prepare&.call(state)
108
+ budget_notes.each { |note| inject_budget_note(state, note) }
106
109
  nxt.call(state)
107
110
  ensure
108
111
  # A turn that FAILED after burning tokens still SPENT them: record the
@@ -154,14 +157,22 @@ module Insika
154
157
  # blind. Same billed-spend rule as `Evals::Runner#billed_tokens`.
155
158
  # nil usage (workflow turn / provider without counts) records nothing.
156
159
  def record_usage(state, window)
157
- usage = state.usage || {}
158
- tokens = usage[:total_tokens].to_i + usage[:cached_tokens].to_i +
159
- usage[:cache_creation_tokens].to_i
160
+ tokens = billed_tokens(state)
160
161
  return if tokens.zero?
161
162
 
162
163
  @ledger.add(TOKENS_KIND, state.profile.id.to_s, window: window, by: tokens)
163
164
  end
164
165
 
166
+ # Native rerank counts remain separate from chat usage and retain unknown
167
+ # fields. Only reported token buckets contribute to spend enforcement.
168
+ def billed_tokens(state)
169
+ usage = state.usage || {}
170
+ usage[:total_tokens].to_i + usage[:cached_tokens].to_i + usage[:cache_creation_tokens].to_i +
171
+ Array(state.rerank_usage).sum do |entry|
172
+ %w[input_tokens output_tokens cache_read_tokens cache_write_tokens].sum { |key| entry[key].to_i }
173
+ end
174
+ end
175
+
165
176
  # Graceful halt: safe reply + audit metadata, short-circuit (no nxt).
166
177
  # `detail` carries only ids/counters — never message content.
167
178
  def block(state, edge, category:, detail:)
@@ -187,7 +198,7 @@ module Insika
187
198
 
188
199
  # -> truthy (the budget hash) when budget checks ran. Raises BudgetExceeded
189
200
  # on a HARD cap breach.
190
- def budget_enforce(state, now: Time.now)
201
+ def budget_enforce(state, notes:, now: Time.now)
191
202
  budget = state.profile.respond_to?(:budget) ? state.profile.budget : nil
192
203
  return nil if budget.nil? || @budget_ledger.nil?
193
204
 
@@ -202,9 +213,9 @@ module Insika
202
213
  retry_after: @budget_ledger.reset_in(w[:window], now: now)
203
214
  )
204
215
  end
205
- warn_budget(state, tenant, agent, w, spent, now, level: "cap")
216
+ warn_budget(state, tenant, agent, w, spent, now, level: "cap", notes: notes)
206
217
  elsif spent >= w[:alert_at]
207
- warn_budget(state, tenant, agent, w, spent, now, level: "alert_at")
218
+ warn_budget(state, tenant, agent, w, spent, now, level: "alert_at", notes: notes)
208
219
  end
209
220
  end
210
221
  budget
@@ -240,10 +251,9 @@ module Insika
240
251
  # (window) cell: the `alert_at` crossing and the real soft-cap crossing are
241
252
  # separate markers, so the cap event is never swallowed by the 80% one that
242
253
  # fired earlier (WS2).
243
- def warn_budget(state, tenant, agent, w, spent, now, level:)
244
- inject_budget_note(state,
245
- "[budget: agent '#{agent}' is at #{spent}/#{w[:cap]} tokens this " \
246
- "#{w[:window]} window — keep this turn cheap]")
254
+ def warn_budget(state, tenant, agent, w, spent, now, level:, notes:)
255
+ notes << "[budget: agent '#{agent}' is at #{spent}/#{w[:cap]} tokens this " \
256
+ "#{w[:window]} window — keep this turn cheap]"
247
257
  return if @budget_ledger.mark_alert(tenant: tenant, agent: agent, window: w[:window],
248
258
  level: level, now: now)
249
259
 
@@ -263,12 +273,15 @@ module Insika
263
273
 
264
274
  # Appends the note to the assembled system prompt: the real Data package is
265
275
  # immutable (with), the specs' minimal Struct is mutable — both duck-typed.
276
+ # The note is per-turn data, so it also lands in the volatile layer — below
277
+ # the cache breakpoint, never inside the cached identity prefix.
266
278
  def inject_budget_note(state, note)
267
279
  ctx = state.context
268
280
  return if ctx.nil?
269
281
 
270
282
  if ctx.respond_to?(:with)
271
- state.context = ctx.with(system: "#{ctx.system}\n\n#{note}")
283
+ volatile = [ctx.system_volatile, note].reject { |s| s.to_s.empty? }.join("\n\n")
284
+ state.context = ctx.with(system: "#{ctx.system}\n\n#{note}", system_volatile: volatile)
272
285
  elsif ctx.respond_to?(:system=)
273
286
  ctx.system = "#{ctx.system}\n\n#{note}"
274
287
  end
@@ -277,9 +290,7 @@ module Insika
277
290
  # The turn's REAL billed spend (input + output + cached + cache_creation —
278
291
  # the A4 rule) on the calendar windows.
279
292
  def record_budget_usage(state, now: Time.now)
280
- usage = state.usage || {}
281
- tokens = usage[:total_tokens].to_i + usage[:cached_tokens].to_i +
282
- usage[:cache_creation_tokens].to_i
293
+ tokens = billed_tokens(state)
283
294
  return if tokens.zero?
284
295
 
285
296
  @budget_ledger.add(tenant: budget_tenant(state), agent: state.profile.id.to_s,
@@ -31,7 +31,11 @@ module Insika
31
31
  module_function
32
32
 
33
33
  # -> nil (allowed) | String (block reason).
34
- def violation(url, allow_http: false, host_allowlist: nil, allow_private: false)
34
+ #
35
+ # `pin_private_only:` applies the allowlist to private destinations only, for
36
+ # a URL the operator fixed and the model cannot shape (an MCP server): a
37
+ # public host passes, a private one still needs allow_private AND the list.
38
+ def violation(url, allow_http: false, host_allowlist: nil, allow_private: false, pin_private_only: false)
35
39
  uri = begin
36
40
  URI.parse(url.to_s)
37
41
  rescue URI::InvalidURIError
@@ -43,13 +47,17 @@ module Insika
43
47
 
44
48
  host = uri.host
45
49
  return "missing host" if host.nil? || host.empty?
46
- return "host not in allowlist" if host_allowlist && !host_allowlist.include?(host)
50
+ listed = host_allowlist.nil? || host_allowlist.include?(host)
51
+ return "host not in allowlist" unless listed || pin_private_only
47
52
 
48
53
  addrs = resolve(host)
49
54
  return "host did not resolve" if addrs.empty?
55
+ return nil unless addrs.any? { |ip| blocked?(ip) }
56
+
50
57
  # allow_private skips the private-network block (trusted internal API,
51
58
  # Without it, a private/loopback/metadata target is always blocked.
52
- return "private-network destination blocked" if !allow_private && addrs.any? { |ip| blocked?(ip) }
59
+ return "private-network destination blocked" unless allow_private
60
+ return "host not in allowlist" unless listed
53
61
 
54
62
  nil
55
63
  end
@@ -9,7 +9,7 @@ module Insika
9
9
  # · `validate(env)` — returns structured Findings: a value that fails its type
10
10
  # (INSIKA_PORT=abc), an UNKNOWN key inside a namespace the engine owns
11
11
  # (INSIKA_EGRES_ALLOW_HTTP — a typo the runtime would otherwise ignore in
12
- # silence), and a DEPRECATED legacy key still set under the old HARNESS_ prefix.
12
+ # silence), and a REMOVED legacy key still set under the old HARNESS_ prefix.
13
13
  # Unknown-key detection is scoped to the OWNED prefixes only, so the platform's
14
14
  # own vars (Railway's RAILWAY_*, PORT, PATH, the litestream sidecar's
15
15
  # LITESTREAM_*, the deployment's DEEPSEEK_*/CONSUMER_*) are never flagged.
@@ -19,12 +19,10 @@ module Insika
19
19
  # RAISES ConfigError only when strictness is on (INSIKA_CONFIG_STRICT truthy,
20
20
  # or `strict: true`).
21
21
  #
22
- # RENAME (env pass 2): the engine's owned prefix is now INSIKA_. The old HARNESS_
23
- # names still work — `reconcile_legacy!` backfills INSIKA_* from any HARNESS_* alias
24
- # at boot (new name wins), and `read` gives the same dual-read to the self-contained
25
- # readers (Telemetry, TurnTiming, SubagentGraph) that receive an env hash directly.
26
- # A legacy name in use surfaces as a `:deprecated` warning (never fatal) so operators
27
- # get a clear "rename to INSIKA_*" without a broken boot.
22
+ # RENAME: the engine's owned prefix is INSIKA_. The old HARNESS_ names were removed
23
+ # in 1.0 and are no longer read. HARNESS_ stays an OWNED prefix only so a forgotten
24
+ # legacy name is reported (`:removed`, an error) instead of ignored in silence — a
25
+ # stale HARNESS_DB would otherwise boot the engine on the ephemeral memory backend.
28
26
  #
29
27
  # Insika::Doctor reuses `validate` for its `env:*` checks; the boot roots call
30
28
  # `enforce!`. A deployment layers its own keys on via `extra:` (see
@@ -66,13 +64,13 @@ module Insika
66
64
  def to_h = { "key" => key, "kind" => kind.to_s, "severity" => severity.to_s, "message" => message }
67
65
  end
68
66
 
69
- # The engine's owned prefix and its deprecated predecessor.
67
+ # The engine's owned prefix and its removed predecessor.
70
68
  PREFIX = "INSIKA_"
71
69
  LEGACY_PREFIX = "HARNESS_"
72
70
 
73
71
  # Prefixes the engine fully OWNS: an unknown key under one of these is a typo, not
74
- # a foreign var. INSIKA_ (current) and HARNESS_ (legacy, still honored during the
75
- # deprecation window). Deliberately NOT OPENCLAW_ (the OpenClaw gateway product
72
+ # a foreign var. INSIKA_ (current) and HARNESS_ (removed in 1.0, reported so it is
73
+ # not silently ignored). Deliberately NOT OPENCLAW_ (the OpenClaw gateway product
76
74
  # sets its own OPENCLAW_HOME/_STATE_DIR/… on the same host; the engine reads none
77
75
  # of them), nor LITESTREAM_ (the sidecar owns it), nor OTEL_ (the OpenTelemetry
78
76
  # SDK owns its env).
@@ -95,19 +93,26 @@ module Insika
95
93
  # The engine's own keys. Deployment/app keys (DEEPSEEK_*, CONSUMER_*, …) are NOT
96
94
  # here — a root passes them as `extra:`.
97
95
  DEFAULT = [
96
+ spec(name: "INSIKA_DATABASE_URL", type: :string, secret: true, description: "Postgres URL for the store (postgres://...). Set, it replaces INSIKA_DB: N workers write without SQLite's single file lock. Needs the pg gem."),
97
+ spec(name: "INSIKA_DATABASE_POOL", type: :integer, description: "Postgres connections per process (default 10)."),
98
98
  spec(name: "INSIKA_DB", type: :path, description: "SQLite path; durable config+state. Unset -> ephemeral memory."),
99
99
  spec(name: "INSIKA_BIND", description: "Bind address for the transport server."),
100
100
  spec(name: "INSIKA_PORT", type: :integer, description: "Port for the transport server."),
101
+ spec(name: "INSIKA_CONVERSATIONS_URL", type: :url, description: "Required central history URL for shared_conversations profiles."),
102
+ spec(name: "INSIKA_CONVERSATIONS_TOKEN", secret: true, description: "Scoped central history bearer token."),
101
103
  spec(name: "INSIKA_PUBLIC_URL", type: :url, description: "Public base URL (A2A agent card, links)."),
102
104
  spec(name: "INSIKA_ENV", description: "Environment name shown in the Studio (falls back to RACK_ENV)."),
103
105
  spec(name: "INSIKA_A2A_AGENT", description: "Agent id to expose over inbound A2A (opt-in)."),
104
106
  spec(name: "INSIKA_A2A_REMOTES", type: :csv, description: "Comma-separated remote A2A endpoints."),
105
107
  spec(name: "INSIKA_EGRESS_ALLOW_HTTP", type: :boolean, description: "Allow plain http egress from data-tools, channel callbacks and media fetches (default: https only)."),
106
108
  spec(name: "INSIKA_EGRESS_ALLOW_PRIVATE", type: :boolean, description: "Allow egress to private/loopback ranges (SSRF guard off)."),
107
- spec(name: "INSIKA_EGRESS_HOSTS", type: :csv, description: "Comma-separated host allowlist for data-tool egress (media fetches are NOT pinned by it)."),
109
+ spec(name: "INSIKA_EGRESS_HOSTS", type: :csv, description: "Comma-separated host allowlist for data-tool egress (media fetches are NOT pinned by it; MCP servers only when they resolve to a private address)."),
108
110
  spec(name: "INSIKA_OTEL", type: :boolean, description: "Turn on OpenTelemetry export (opt-in)."),
111
+ spec(name: "INSIKA_MODEL_REFRESH_HOURS", type: :integer, description: "Hours between refreshes of RubyLLM's model catalog (ids, prices), kept in the store and shared by every worker. Default 24; 0 disables (the gem's bundled catalog stays)."),
109
112
  spec(name: "INSIKA_MODEL_PRICING", description: "JSON rates table (USD per million tokens) for the estimated-cost attribute; unset -> no cost reported."),
110
- spec(name: "INSIKA_TURN_TIMING", type: :boolean, description: "Emit per-turn TTFB breakdown in responses (opt-in)."),
113
+ spec(name: "INSIKA_SLOW_STORE_MS", type: :integer, description: "Log every SQLite store call that takes at least this many ms, with its caller (off when unset)."),
114
+ spec(name: "INSIKA_MODEL_VISIBLE_TRACES", type: :boolean, description: "Record the full model request per turn (model_visible_traces). Default on; 0 stops the write (heavy on SQLite)."),
115
+ spec(name: "INSIKA_TURN_TIMING", type: :boolean, description: "Expose the per-turn TTFB breakdown in responses (opt-in; Studio always records it)."),
111
116
  spec(name: "INSIKA_SUBAGENT_DEPTH_CAP", type: :integer, description: "Max delegation depth in the subagent graph (default 5)."),
112
117
  spec(name: "INSIKA_SUBAGENT_FANOUT_CAP", type: :integer, description: "Max parallel children in spawn_subagents (default 8)."),
113
118
  spec(name: "INSIKA_ARTIFACT_SIGNING_KEY", secret: true, description: "HMAC key for signed artifact links (INSIKA_ARTIFACT_SIGNING_TTL). Unset -> no signed artifact surface."),
@@ -139,6 +144,8 @@ module Insika
139
144
  spec(name: "INSIKA_ROUTER_BACKEND_PORT", type: :integer, description: "the engine port on every DNS-resolved backend pod (required with INSIKA_ROUTER_BACKENDS_DNS)."),
140
145
  spec(name: "INSIKA_ROUTER_DNS_INTERVAL", type: :integer, description: "seconds between `insika-router` DNS re-resolves (default 15)."),
141
146
  spec(name: "INSIKA_ROUTER_BODY_MAX_BYTES", type: :integer, description: "size cap `insika-router` will parse looking for a session key before falling back to round-robin (default 262144; never bounds what is forwarded)."),
147
+ spec(name: "INSIKA_WORKERS", type: :integer, description: "Engine processes behind the session-sticky `insika-router` in one container (deploy/workers.sh). Unset or 1 = one process, no router."),
148
+ spec(name: "INSIKA_WORKER_BASE_PORT", type: :integer, description: "First local port for the INSIKA_WORKERS processes (default 9300)."),
142
149
  spec(name: "INSIKA_ROUTER_BACKEND_TIMEOUT", type: :integer, description: "`insika-router`'s connect/read timeout to a backend, in seconds (default 10)."),
143
150
  spec(name: "INSIKA_ROUTER_HOST", description: "bind address for `insika-router` itself (default 0.0.0.0)."),
144
151
  spec(name: "INSIKA_ROUTER_PORT", type: :integer, description: "listen port for `insika-router` itself (default 9090)."),
@@ -150,54 +157,12 @@ module Insika
150
157
  spec(name: "OTEL_SERVICE_NAME", description: "Service name for OTEL spans (default: insika).")
151
158
  ].freeze
152
159
 
153
- # -- dual-read (rename compat) -------------------------------------
154
-
155
- # HARNESS_X -> INSIKA_X for the deprecated alias of a canonical key; nil if `name`
156
- # is not an INSIKA_ key.
157
- def legacy_alias(name)
158
- s = name.to_s
159
- s.start_with?(PREFIX) ? LEGACY_PREFIX + s[PREFIX.length..] : nil
160
- end
161
-
162
- # The current (canonical) name for any owned key: HARNESS_X -> INSIKA_X; an INSIKA_
163
- # key or any non-owned key is returned unchanged.
164
- def canonical(name)
165
- s = name.to_s
166
- s.start_with?(LEGACY_PREFIX) ? PREFIX + s[LEGACY_PREFIX.length..] : s
167
- end
168
-
169
160
  def legacy?(name) = name.to_s.start_with?(LEGACY_PREFIX)
170
161
  def owned?(name) = OWNED_PREFIXES.any? { |p| name.to_s.start_with?(p) }
171
162
 
172
- # Reads a canonical INSIKA_* key, falling back to the deprecated HARNESS_* alias
173
- # (new name wins). For the self-contained readers that get an env hash directly and
174
- # so never see `reconcile_legacy!`'s process-wide backfill. -> value | nil.
175
- def read(canonical_name, env = ENV)
176
- value = env[canonical_name]
177
- return value if present?(value)
178
-
179
- (legacy = legacy_alias(canonical_name)) ? env[legacy] : value
180
- end
181
-
182
- # BOOT backfill: copies every still-set HARNESS_* var to its INSIKA_* name (new name
183
- # wins if both are set) so the process ENV speaks the new names before any read.
184
- # Generic (not schema-bound) so plugin/deployment HARNESS_* keys migrate too. WARNS
185
- # once with the migrated list. -> [migrated legacy names]. Idempotent.
186
- def reconcile_legacy!(env = ENV, warn: method(:default_warn))
187
- migrated = env.keys.map(&:to_s).select { |k| k.start_with?(LEGACY_PREFIX) }.sort.each_with_object([]) do |legacy, acc|
188
- canonical = PREFIX + legacy[LEGACY_PREFIX.length..]
189
- next if present?(env[canonical]) # the new name already wins
190
- next if env[legacy].nil?
191
-
192
- env[canonical] = env[legacy]
193
- acc << legacy
194
- end
195
- unless migrated.empty?
196
- warn.call("deprecation: #{migrated.join(', ')} — HARNESS_* env vars are renamed to INSIKA_*; " \
197
- "honored via the new names this run, please update your environment (legacy names removed in a future release)")
198
- end
199
- migrated
200
- end
163
+ # Reads an INSIKA_* key from `env`. The single reader for the self-contained
164
+ # readers (Telemetry, TurnTiming, SubagentGraph) that get an env hash directly.
165
+ def read(name, env = ENV) = env[name]
201
166
 
202
167
  # -- validation ----------------------------------------------------
203
168
 
@@ -208,33 +173,28 @@ module Insika
208
173
  findings = []
209
174
 
210
175
  # VALUE + required checks over the KNOWN specs (owned or not — DEEPSEEK_API_KEY
211
- # is not prefixed). Dual-read: the canonical name wins, the legacy alias is honored.
176
+ # is not prefixed).
212
177
  specs.each_value do |s|
213
- legacy = legacy_alias(s.name)
214
- set_name = if env.key?(s.name) then s.name
215
- elsif legacy && env.key?(legacy) then legacy
216
- end
217
-
218
- if set_name.nil?
178
+ unless env.key?(s.name)
219
179
  findings << Finding.new(key: s.name, kind: :missing_required, severity: :error,
220
180
  message: "#{s.name} is required (unset)") if s.required?
221
181
  next
222
182
  end
223
183
 
224
- msg = s.error_for(env[set_name])
225
- findings << Finding.new(key: set_name, kind: :invalid, severity: :error, message: "#{set_name} #{msg}") if msg
184
+ msg = s.error_for(env[s.name])
185
+ findings << Finding.new(key: s.name, kind: :invalid, severity: :error, message: "#{s.name} #{msg}") if msg
226
186
  end
227
187
 
228
- # OWNED-PREFIX scan: a legacy alias of a known key is DEPRECATED (warn); any other
229
- # owned key with no matching spec is UNKNOWN (a typo the runtime would ignore).
188
+ # OWNED-PREFIX scan: a HARNESS_ key is REMOVED (no longer read); any other owned
189
+ # key with no matching spec is UNKNOWN (a typo the runtime would ignore).
230
190
  env.each_key do |raw|
231
191
  key = raw.to_s
232
192
  next unless owned?(key)
233
193
 
234
- if legacy?(key) && specs.key?(canonical(key))
235
- findings << Finding.new(key: key, kind: :deprecated, severity: :warn,
236
- message: "#{key} is deprecated — rename to #{canonical(key)}")
237
- elsif !specs.key?(canonical(key)) && !specs.key?(key)
194
+ if legacy?(key)
195
+ findings << Finding.new(key: key, kind: :removed, severity: :error,
196
+ message: "#{key} was removed in 1.0 and is ignored — rename to #{PREFIX}#{key.delete_prefix(LEGACY_PREFIX)}")
197
+ elsif !specs.key?(key)
238
198
  findings << Finding.new(key: key, kind: :unknown, severity: :error,
239
199
  message: "#{key} is not a known config key (typo? unknown key in an owned namespace)")
240
200
  end
@@ -248,7 +208,7 @@ module Insika
248
208
 
249
209
  # BOOT GATE. Validates the environment, WARNS every finding via `warn` (a callable
250
210
  # taking a String), and RAISES ConfigError only when strict. `strict` defaults to
251
- # the INSIKA_CONFIG_STRICT flag (HARNESS_CONFIG_STRICT still honored). -> [Finding]
211
+ # the INSIKA_CONFIG_STRICT flag. -> [Finding]
252
212
  # (also on the happy path). The root keeps booting on warnings (last-known-good).
253
213
  def enforce!(env = ENV, extra: [], strict: nil, warn: method(:default_warn))
254
214
  strict = truthy?(read("INSIKA_CONFIG_STRICT", env)) if strict.nil?
data/lib/insika/errors.rb CHANGED
@@ -8,6 +8,7 @@ module Insika
8
8
 
9
9
  class ValidationError < Error; end # Malformed Command -> HTTP 422, no Task created
10
10
  class NotFoundError < Error; end # nonexistent session/task/agent -> HTTP 404
11
+ class ConflictError < Error; end # the write contradicts state that already exists -> HTTP 409
11
12
 
12
13
  # Policy Engine denied -> :policy_denied event, task :failed
13
14
  class PolicyDenied < Error