insika 0.9.0 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (257) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +237 -2
  3. data/README.md +5 -3
  4. data/bin/insika +3 -0
  5. data/bin/insika-router +0 -1
  6. data/docs/AGENTS.md +90 -7
  7. data/docs/ARCHITECTURE.md +12 -7
  8. data/docs/ARTIFACTS.md +33 -3
  9. data/docs/BENCHMARK.md +50 -7
  10. data/docs/CONTEXT.md +63 -0
  11. data/docs/CROSS-HARNESS-BENCH.md +157 -0
  12. data/docs/DEPLOY.md +57 -11
  13. data/docs/EMBEDDING.md +10 -0
  14. data/docs/EVALS.md +33 -4
  15. data/docs/FACTS.md +8 -6
  16. data/docs/HARVEST.md +8 -3
  17. data/docs/KNOWLEDGE.md +7 -0
  18. data/docs/MEDIA.md +6 -2
  19. data/docs/OBSERVABILITY.md +143 -22
  20. data/docs/PERFORMANCE.md +110 -0
  21. data/docs/PLUGINS.md +4 -4
  22. data/docs/POLICY.md +22 -13
  23. data/docs/RELEASING.md +5 -0
  24. data/docs/ROUTER.md +7 -0
  25. data/docs/RUBYLLM_2_MIGRATION.md +307 -0
  26. data/docs/SECURITY.md +7 -0
  27. data/docs/SHARED-CONVERSATIONS.md +40 -0
  28. data/docs/SKILLS.md +5 -3
  29. data/docs/STUDIO.md +122 -0
  30. data/docs/TOOLS.md +111 -9
  31. data/docs/operate.md +3 -0
  32. data/lib/insika/agent_defaults.rb +140 -0
  33. data/lib/insika/agent_file_store.rb +12 -0
  34. data/lib/insika/agent_profile.rb +78 -8
  35. data/lib/insika/artifact_signing.rb +11 -0
  36. data/lib/insika/chat_builder.rb +79 -25
  37. data/lib/insika/checkpoint.rb +6 -1
  38. data/lib/insika/checkpoint_store.rb +59 -7
  39. data/lib/insika/commands/agent_payload.rb +3 -3
  40. data/lib/insika/commands/delete_kit.rb +26 -0
  41. data/lib/insika/commands/gate_harvest.rb +3 -0
  42. data/lib/insika/commands/review_guardrail_flag.rb +38 -0
  43. data/lib/insika/commands/run_harvest.rb +17 -1
  44. data/lib/insika/commands/seed_session.rb +11 -1
  45. data/lib/insika/commands/send_message.rb +9 -3
  46. data/lib/insika/commands/update_settings.rb +23 -1
  47. data/lib/insika/commands/write_kit.rb +27 -0
  48. data/lib/insika/compaction.rb +33 -7
  49. data/lib/insika/config_store.rb +87 -7
  50. data/lib/insika/context/priority.rb +3 -0
  51. data/lib/insika/context/provider.rb +2 -2
  52. data/lib/insika/context/providers/knowledge.rb +25 -2
  53. data/lib/insika/context/providers/memory.rb +53 -4
  54. data/lib/insika/context/providers/pending_confirmation.rb +52 -0
  55. data/lib/insika/context/providers/prompt.rb +17 -4
  56. data/lib/insika/context/providers/session.rb +39 -3
  57. data/lib/insika/context/reranker.rb +75 -0
  58. data/lib/insika/distill.rb +22 -13
  59. data/lib/insika/doctor.rb +34 -4
  60. data/lib/insika/dsl/runtime.rb +36 -4
  61. data/lib/insika/dsl/server_boot.rb +4 -12
  62. data/lib/insika/dsl.rb +13 -5
  63. data/lib/insika/edge_limiter.rb +23 -15
  64. data/lib/insika/egress_guard.rb +11 -3
  65. data/lib/insika/env_schema.rb +33 -73
  66. data/lib/insika/evals/assertions.rb +171 -16
  67. data/lib/insika/evals/golden.rb +141 -4
  68. data/lib/insika/evals/judge.rb +19 -10
  69. data/lib/insika/evals/moderator_compare.rb +47 -0
  70. data/lib/insika/evals/report.rb +61 -9
  71. data/lib/insika/evals/runner.rb +44 -4
  72. data/lib/insika/evals/simulator.rb +10 -1
  73. data/lib/insika/evals/transport.rb +200 -0
  74. data/lib/insika/event_stream.rb +63 -7
  75. data/lib/insika/evidence.rb +29 -8
  76. data/lib/insika/executor.rb +668 -141
  77. data/lib/insika/harvest.rb +26 -14
  78. data/lib/insika/harvest_engine.rb +1 -0
  79. data/lib/insika/harvest_store.rb +12 -0
  80. data/lib/insika/http_client.rb +87 -4
  81. data/lib/insika/kits.rb +54 -0
  82. data/lib/insika/knowledge.rb +34 -20
  83. data/lib/insika/llm_configurator.rb +46 -4
  84. data/lib/insika/llm_provider_store.rb +1 -3
  85. data/lib/insika/llm_trace_store.rb +54 -0
  86. data/lib/insika/mcp_client.rb +27 -63
  87. data/lib/insika/mcp_live_tool.rb +55 -9
  88. data/lib/insika/mcp_store.rb +32 -1
  89. data/lib/insika/mcp_tool_registry.rb +69 -26
  90. data/lib/insika/media.rb +38 -25
  91. data/lib/insika/memory_store.rb +4 -3
  92. data/lib/insika/memory_trim.rb +60 -0
  93. data/lib/insika/middleware.rb +12 -3
  94. data/lib/insika/model_metrics_store.rb +285 -0
  95. data/lib/insika/model_registry.rb +110 -0
  96. data/lib/insika/model_resolver.rb +1 -1
  97. data/lib/insika/model_selection.rb +14 -21
  98. data/lib/insika/model_visible.rb +17 -6
  99. data/lib/insika/onboarding.rb +1 -0
  100. data/lib/insika/outcome_store.rb +1 -1
  101. data/lib/insika/overlay_tool_registry.rb +25 -2
  102. data/lib/insika/pack_importer.rb +54 -9
  103. data/lib/insika/packaging.rb +14 -5
  104. data/lib/insika/pending_action_store.rb +45 -18
  105. data/lib/insika/plugin/loader.rb +4 -2
  106. data/lib/insika/profile_source.rb +4 -0
  107. data/lib/insika/provider_error_classifier.rb +18 -3
  108. data/lib/insika/queue_policy.rb +7 -2
  109. data/lib/insika/refinement/proposer.rb +42 -17
  110. data/lib/insika/reliability.rb +12 -3
  111. data/lib/insika/router/app.rb +14 -5
  112. data/lib/insika/router/session_key.rb +35 -5
  113. data/lib/insika/safety/config.rb +20 -3
  114. data/lib/insika/safety/corpus.rb +10 -3
  115. data/lib/insika/safety/factory.rb +39 -1
  116. data/lib/insika/safety/input_guardrail.rb +21 -1
  117. data/lib/insika/safety/judge_moderator.rb +70 -0
  118. data/lib/insika/safety/output_validator.rb +29 -5
  119. data/lib/insika/safety/prompt_echo.rb +89 -0
  120. data/lib/insika/schema_guard.rb +53 -76
  121. data/lib/insika/server/app.rb +17 -8
  122. data/lib/insika/server/responses.rb +19 -1
  123. data/lib/insika/session_actor.rb +58 -28
  124. data/lib/insika/session_store.rb +159 -10
  125. data/lib/insika/settings_store.rb +47 -1
  126. data/lib/insika/shared_conversations.rb +305 -0
  127. data/lib/insika/skill_catalog.rb +28 -2
  128. data/lib/insika/skill_store.rb +6 -0
  129. data/lib/insika/soak/runner.rb +1 -1
  130. data/lib/insika/steer_injector.rb +7 -48
  131. data/lib/insika/store.rb +29 -1
  132. data/lib/insika/stores/json_value.rb +37 -0
  133. data/lib/insika/stores/memory.rb +22 -35
  134. data/lib/insika/stores/postgres.rb +295 -0
  135. data/lib/insika/stores/sqlite.rb +257 -43
  136. data/lib/insika/stores/turn_counter.rb +25 -0
  137. data/lib/insika/studio/app.rb +1453 -161
  138. data/lib/insika/studio/assets/dist/application.css +1 -1
  139. data/lib/insika/studio/assets/dist/application.js +25 -25
  140. data/lib/insika/studio/assets/dist/fonts/ibm-plex-mono-latin-400-normal.woff2 +0 -0
  141. data/lib/insika/studio/assets/dist/fonts/ibm-plex-mono-latin-500-normal.woff2 +0 -0
  142. data/lib/insika/studio/assets/dist/fonts/ibm-plex-mono-latin-600-normal.woff2 +0 -0
  143. data/lib/insika/studio/assets/dist/fonts/ibm-plex-sans-latin-wght-normal.woff2 +0 -0
  144. data/lib/insika/studio/forms.rb +171 -9
  145. data/lib/insika/studio/nav_icons.rb +3 -0
  146. data/lib/insika/studio/views/_agent_config_retrieval.erb +73 -0
  147. data/lib/insika/studio/views/_agent_picker.erb +15 -0
  148. data/lib/insika/studio/views/_agent_tab_config.erb +78 -242
  149. data/lib/insika/studio/views/_agent_tab_history.erb +56 -1
  150. data/lib/insika/studio/views/_agent_tab_performance.erb +34 -0
  151. data/lib/insika/studio/views/_agent_tab_prompts.erb +7 -9
  152. data/lib/insika/studio/views/_agent_tab_skills.erb +16 -7
  153. data/lib/insika/studio/views/_agent_tab_tools.erb +90 -0
  154. data/lib/insika/studio/views/_agents_master.erb +8 -12
  155. data/lib/insika/studio/views/_cfg_alerts.erb +4 -0
  156. data/lib/insika/studio/views/_cfg_budget.erb +20 -0
  157. data/lib/insika/studio/views/_cfg_distill.erb +26 -0
  158. data/lib/insika/studio/views/_cfg_followup.erb +30 -0
  159. data/lib/insika/studio/views/_cfg_grounding.erb +21 -0
  160. data/lib/insika/studio/views/_cfg_guardrails.erb +47 -0
  161. data/lib/insika/studio/views/_cfg_harvest.erb +35 -0
  162. data/lib/insika/studio/views/_cfg_reliability.erb +32 -0
  163. data/lib/insika/studio/views/_chats_master.erb +66 -0
  164. data/lib/insika/studio/views/_customers_master.erb +37 -0
  165. data/lib/insika/studio/views/_empty.erb +15 -0
  166. data/lib/insika/studio/views/_facts_master.erb +42 -0
  167. data/lib/insika/studio/views/_harvest_candidate_detail.erb +14 -0
  168. data/lib/insika/studio/views/_identity.erb +16 -0
  169. data/lib/insika/studio/views/_kpi.erb +17 -0
  170. data/lib/insika/studio/views/_message.erb +40 -2
  171. data/lib/insika/studio/views/_model_chart.erb +82 -0
  172. data/lib/insika/studio/views/_refinement_master.erb +38 -0
  173. data/lib/insika/studio/views/{tool_edit.erb → _tool_edit.erb} +32 -9
  174. data/lib/insika/studio/views/_tool_users.erb +17 -0
  175. data/lib/insika/studio/views/_why.erb +53 -0
  176. data/lib/insika/studio/views/agent_detail.erb +17 -10
  177. data/lib/insika/studio/views/agents.erb +28 -16
  178. data/lib/insika/studio/views/approvals.erb +16 -11
  179. data/lib/insika/studio/views/artifact.erb +20 -7
  180. data/lib/insika/studio/views/artifacts.erb +19 -14
  181. data/lib/insika/studio/views/chats.erb +22 -29
  182. data/lib/insika/studio/views/customer.erb +88 -43
  183. data/lib/insika/studio/views/customers.erb +16 -26
  184. data/lib/insika/studio/views/evals.erb +49 -31
  185. data/lib/insika/studio/views/fact_detail.erb +114 -0
  186. data/lib/insika/studio/views/facts.erb +15 -116
  187. data/lib/insika/studio/views/followups.erb +63 -52
  188. data/lib/insika/studio/views/funnel.erb +104 -43
  189. data/lib/insika/studio/views/harvest.erb +85 -59
  190. data/lib/insika/studio/views/home.erb +138 -23
  191. data/lib/insika/studio/views/inbox.erb +36 -0
  192. data/lib/insika/studio/views/kits.erb +176 -0
  193. data/lib/insika/studio/views/knowledge.erb +92 -80
  194. data/lib/insika/studio/views/layout.erb +42 -11
  195. data/lib/insika/studio/views/live.erb +45 -0
  196. data/lib/insika/studio/views/mcp.erb +43 -13
  197. data/lib/insika/studio/views/models.erb +187 -0
  198. data/lib/insika/studio/views/parity.erb +38 -32
  199. data/lib/insika/studio/views/playground.erb +13 -9
  200. data/lib/insika/studio/views/refinement.erb +15 -224
  201. data/lib/insika/studio/views/refinement_detail.erb +243 -0
  202. data/lib/insika/studio/views/session.erb +152 -117
  203. data/lib/insika/studio/views/settings.erb +86 -22
  204. data/lib/insika/studio/views/skills.erb +53 -19
  205. data/lib/insika/studio/views/system_files.erb +7 -7
  206. data/lib/insika/studio/views/task.erb +45 -15
  207. data/lib/insika/studio/views/tasks.erb +35 -18
  208. data/lib/insika/studio/views/tools.erb +62 -73
  209. data/lib/insika/task_actor.rb +4 -1
  210. data/lib/insika/task_store.rb +67 -1
  211. data/lib/insika/telemetry/pricing.rb +16 -14
  212. data/lib/insika/telemetry/recorder.rb +103 -9
  213. data/lib/insika/telemetry/ruby_llm_instrumenter.rb +110 -0
  214. data/lib/insika/telemetry.rb +12 -2
  215. data/lib/insika/templates/daily-digest/README.md +13 -0
  216. data/lib/insika/templates/daily-digest/agent.rb +22 -11
  217. data/lib/insika/testing/store_contract.rb +65 -0
  218. data/lib/insika/text_tool_call.rb +66 -0
  219. data/lib/insika/tick.rb +7 -1
  220. data/lib/insika/token_store.rb +5 -0
  221. data/lib/insika/tool_batch.rb +1 -1
  222. data/lib/insika/tool_catalog.rb +12 -2
  223. data/lib/insika/tool_definition.rb +10 -4
  224. data/lib/insika/tool_envelope.rb +165 -28
  225. data/lib/insika/tool_manifest.rb +1 -0
  226. data/lib/insika/tool_store.rb +3 -0
  227. data/lib/insika/tool_trace_store.rb +15 -8
  228. data/lib/insika/tool_usage_report.rb +2 -2
  229. data/lib/insika/tools/a2a_remote.rb +1 -1
  230. data/lib/insika/tools/agent_enum.rb +1 -1
  231. data/lib/insika/tools/confirm_pending.rb +112 -0
  232. data/lib/insika/tools/data_defined_tool.rb +23 -15
  233. data/lib/insika/tools/generate_image.rb +2 -2
  234. data/lib/insika/tools/load_knowledge.rb +7 -3
  235. data/lib/insika/tools/load_skill.rb +3 -3
  236. data/lib/insika/tools/present.rb +1 -8
  237. data/lib/insika/tools/remember.rb +7 -2
  238. data/lib/insika/tools/run_persona_eval.rb +3 -5
  239. data/lib/insika/tools/save_artifact.rb +16 -6
  240. data/lib/insika/tools/schedule_followup.rb +3 -3
  241. data/lib/insika/tools/stuck_signal.rb +3 -3
  242. data/lib/insika/tools/subagent.rb +5 -5
  243. data/lib/insika/tools/subagents.rb +3 -3
  244. data/lib/insika/tools/tool_search.rb +7 -5
  245. data/lib/insika/tools/tts.rb +2 -2
  246. data/lib/insika/tools/update_briefing.rb +4 -4
  247. data/lib/insika/turn_output.rb +2 -2
  248. data/lib/insika/turn_state.rb +18 -0
  249. data/lib/insika/turn_timing.rb +51 -14
  250. data/lib/insika/version.rb +1 -1
  251. data/lib/insika/wiring/graph.rb +81 -16
  252. data/lib/insika/workflow.rb +25 -64
  253. data/lib/insika.rb +19 -3
  254. metadata +69 -13
  255. data/lib/insika/commands/import_mcp_tools.rb +0 -48
  256. data/lib/insika/mcp_http_client.rb +0 -67
  257. data/lib/insika/mcp_tool_ingestor.rb +0 -165
@@ -24,14 +24,37 @@ module Insika
24
24
 
25
25
  # Backend by config: INSIKA_DB set → durable SQLite (survives restart, the
26
26
  # prerequisite for Recovery); missing → ephemeral Memory (dev/demo). The same
27
- # rule lived verbatim in both roots. Dual-read honors the legacy HARNESS_DB alias.
27
+ # rule lived verbatim in both roots.
28
+ # LITESTREAM_REPLICA_URL is the same switch deploy/entrypoint.sh uses: set, and
29
+ # Litestream owns the checkpoints.
30
+ # Every backend counts its calls into the running turn (TurnTiming).
28
31
  def backend_from_env(env = ENV)
32
+ Insika::Stores::TurnCounter.attach(raw_backend_from_env(env))
33
+ end
34
+
35
+ def raw_backend_from_env(env = ENV)
36
+ url = Insika::EnvSchema.read("INSIKA_DATABASE_URL", env)
37
+ if url && !url.empty?
38
+ pool = Insika::EnvSchema.read("INSIKA_DATABASE_POOL", env)
39
+ return Insika::Stores::Postgres.new(url: url, pool: pool && !pool.empty? ? Integer(pool) : 10)
40
+ end
41
+
29
42
  db = Insika::EnvSchema.read("INSIKA_DB", env)
30
- db && !db.empty? ? Insika::Stores::SQLite.new(path: db) : Insika::Stores::Memory.new
43
+ return Insika::Stores::Memory.new unless db && !db.empty?
44
+
45
+ slow = Insika::EnvSchema.read("INSIKA_SLOW_STORE_MS", env)
46
+ Insika::Stores::SQLite.new(path: db, autocheckpoint: env["LITESTREAM_REPLICA_URL"].to_s.empty?,
47
+ slow_ms: slow && Float(slow))
48
+ end
49
+
50
+ # On unless INSIKA_MODEL_VISIBLE_TRACES is set to a false value.
51
+ def model_visible_traces?(env = ENV)
52
+ value = Insika::EnvSchema.read("INSIKA_MODEL_VISIBLE_TRACES", env)
53
+ value.nil? || Insika::EnvSchema.truthy?(value)
31
54
  end
32
55
 
33
56
  # an integer tick knob from the env, nil when unset (the caller's
34
- # default wins). Dual-read honors the deprecated HARNESS_* alias.
57
+ # default wins).
35
58
  def tick_env(name, env = ENV)
36
59
  value = Insika::EnvSchema.read(name, env)
37
60
  value&.to_i
@@ -43,6 +66,8 @@ module Insika
43
66
  def spine(backend:, extra_policy_builtins: {})
44
67
  session_store = Insika::SessionStore.new(store: backend)
45
68
  task_store = Insika::TaskStore.new(store: backend)
69
+ llm_trace_store = Insika::LLMTraceStore.new(store: backend)
70
+ model_metrics_store = Insika::ModelMetricsStore.new(store: backend)
46
71
  checkpoint_store = Insika::CheckpointStore.new(store: backend)
47
72
  pending_action_store = Insika::PendingActionStore.new(store: backend)
48
73
  delegation_store = Insika::DelegationStore.new(store: backend)
@@ -122,7 +147,7 @@ module Insika
122
147
 
123
148
  Spine.new(
124
149
  backend: backend, event_stream: Insika::EventStream.new,
125
- session_store: session_store, task_store: task_store,
150
+ session_store: session_store, task_store: task_store, llm_trace_store: llm_trace_store, model_metrics_store: model_metrics_store,
126
151
  checkpoint_store: checkpoint_store, pending_action_store: pending_action_store,
127
152
  delegation_store: delegation_store,
128
153
  memory_store: memory_store, memory_audit_store: memory_audit_store,
@@ -168,6 +193,12 @@ module Insika
168
193
  # expose it. The gem's require lives IN the factory block (loaded on
169
194
  # the 1st instance, turn time -> wiring-load stays gem-free).
170
195
  register_artifact_tool(spine)
196
+ # Turns read profiles with the platform agent defaults applied; the
197
+ # authoring commands the roots register keep the agent's own record.
198
+ if (settings_store = executor_extra[:settings_store])
199
+ profiles = Insika::AgentDefaults::ProfileSource.new(Insika::ProfileSource.coerce(profiles),
200
+ settings_store: settings_store)
201
+ end
171
202
  spine.hooks.register(:task, after: guardrails.output_validator)
172
203
  middleware = Insika::MiddlewareStack.new([edge_limiter, guardrails.input_guardrail].compact)
173
204
 
@@ -192,11 +223,16 @@ module Insika
192
223
  # the profile's `reliability` data gates it, so the bare wiring is unchanged.
193
224
  reliability = Insika::Reliability.new(circuit_store: spine.circuit_state,
194
225
  event_stream: spine.event_stream)
226
+ if (url = Insika::Coercion.presence(Insika::EnvSchema.read("INSIKA_CONVERSATIONS_URL")))
227
+ executor_extra = {shared_conversations: Insika::SharedConversations.new(
228
+ url: url, token: Insika::EnvSchema.read("INSIKA_CONVERSATIONS_TOKEN"), store: spine.backend)}.merge(executor_extra)
229
+ end
195
230
  executor = Insika::Executor.new(
196
231
  context_builder: context_builder, policy_engine: policy_engine,
197
232
  middleware: middleware, hooks: spine.hooks,
198
233
  tool_registry: tool_registry, skill_catalog: skill_catalog, profiles: profiles,
199
234
  session_store: spine.session_store, task_store: spine.task_store,
235
+ llm_trace_store: spine.llm_trace_store, model_metrics_store: spine.model_metrics_store,
200
236
  checkpoint_store: spine.checkpoint_store, event_stream: spine.event_stream,
201
237
  workflow_registry: spine.workflow_registry, pending_action_store: spine.pending_action_store,
202
238
  capability_registry: spine.capability_registry, tool_catalog: tool_catalog,
@@ -213,11 +249,14 @@ module Insika
213
249
  # the model-visible trace rides executor_extra like the
214
250
  # context trace — a module graph that omits it still builds (nil =
215
251
  # parity). The base graph wires it over the same backend the context
216
- # trace uses.
217
- model_visible_trace_store: Insika::ModelVisibleTraceStore.new(store: spine.backend),
252
+ # trace uses. INSIKA_MODEL_VISIBLE_TRACES=0 stops the WRITE only (nothing in
253
+ # the product reads it back; it was ~80% of the bytes a turn wrote): the
254
+ # purge paths below keep their own store, so old records still go.
255
+ model_visible_trace_store: model_visible_traces? ? Insika::ModelVisibleTraceStore.new(store: spine.backend) : nil,
218
256
  # the per-turn extraction hook (nil = the loop is off,
219
257
  # parity). Gated per-agent by `profile.knowledge`.
220
258
  knowledge_store: spine.knowledge_store,
259
+ # llm stays graph-scoped; Executor binds instrumentation on operation copies.
221
260
  **executor_extra
222
261
  )
223
262
 
@@ -228,6 +267,10 @@ module Insika
228
267
  profiles: profiles, proposal_store: spine.proposal_store,
229
268
  session_store: spine.session_store, memory_store: spine.memory_store,
230
269
  settings_store: executor_extra[:settings_store],
270
+ distiller_factory: ->(config) {
271
+ Insika::Distill::DistillerFactory.build(config,
272
+ utility_model: executor_extra[:settings_store]&.get&.[]("utility_model"), llm: executor_extra[:llm])
273
+ },
231
274
  event_stream: spine.event_stream
232
275
  )
233
276
 
@@ -297,6 +340,11 @@ module Insika
297
340
  profiles: profiles, executor: executor, budget_ledger: spine.budget_ledger,
298
341
  event_stream: spine.event_stream
299
342
  )
343
+ # RubyLLM's model catalog (prices included) in the store, refreshed
344
+ # daily by one worker and reloaded by the rest.
345
+ executor.tick.model_registry = Insika::ModelRegistry.new(
346
+ store: spine.backend, interval: (tick_env("INSIKA_MODEL_REFRESH_HOURS") || 24) * 3600
347
+ )
300
348
  # the distillation engine — the tick-duty that finds idle
301
349
  # customer sessions and distills them on its own worker fiber (a
302
350
  # supervisor child, started in serving mode next to the tick).
@@ -323,7 +371,11 @@ module Insika
323
371
  skill_store: skill_catalog.store, # the harvest's dedup reads the authored skills
324
372
  tool_trace_store: executor_extra[:tool_trace_store],
325
373
  settings_store: executor_extra[:settings_store],
326
- negative_list: nil, miner_factory: nil,
374
+ negative_list: nil,
375
+ miner_factory: ->(config) {
376
+ Insika::Harvest::MinerFactory.build(config,
377
+ utility_model: executor_extra[:settings_store]&.get&.[]("utility_model"), llm: executor_extra[:llm])
378
+ },
327
379
  event_stream: spine.event_stream
328
380
  )
329
381
  executor.harvest_engine = Insika::HarvestEngine.new(
@@ -343,6 +395,7 @@ module Insika
343
395
  Graph::Result.new(
344
396
  backend: spine.backend, event_stream: spine.event_stream,
345
397
  session_store: spine.session_store, task_store: spine.task_store,
398
+ llm_trace_store: spine.llm_trace_store, model_metrics_store: spine.model_metrics_store,
346
399
  checkpoint_store: spine.checkpoint_store, pending_action_store: spine.pending_action_store,
347
400
  delegation_store: spine.delegation_store,
348
401
  memory_store: spine.memory_store, memory_audit_store: spine.memory_audit_store,
@@ -367,7 +420,8 @@ module Insika
367
420
  tool_catalog: tool_catalog, skill_catalog: skill_catalog, prompt_catalog: prompt_catalog,
368
421
  hooks: spine.hooks, guardrails: guardrails, middleware: middleware,
369
422
  context_providers: context_providers, context_builder: context_builder,
370
- policy_engine: policy_engine, profiles: profiles, executor: executor, bus: bus
423
+ # the agent's own records: the Studio and the roots author through them
424
+ policy_engine: policy_engine, profiles: Insika::AgentDefaults.own(profiles), executor: executor, bus: bus
371
425
  )
372
426
  end
373
427
 
@@ -536,6 +590,11 @@ module Insika
536
590
  bus.register(:delete_artifact,
537
591
  Insika::Commands::DeleteArtifact.new(artifact_store: spine.artifact_store,
538
592
  event_stream: spine.event_stream))
593
+ # a person's verdict on a guardrail flag (false positive / resolved)
594
+ # — the Session viewer's button and the Review inbox.
595
+ bus.register(:review_guardrail_flag,
596
+ Insika::Commands::ReviewGuardrailFlag.new(session_store: spine.session_store,
597
+ event_stream: spine.event_stream))
539
598
  bus.register(:revoke_contact,
540
599
  Insika::Commands::RevokeContact.new(contact_store: spine.contact_store,
541
600
  followup_store: spine.followup_store,
@@ -561,7 +620,11 @@ module Insika
561
620
  skill_store: skill_catalog.store,
562
621
  tool_trace_store: executor_extra[:tool_trace_store],
563
622
  settings_store: executor_extra[:settings_store],
564
- negative_list: nil, miner_factory: nil,
623
+ negative_list: nil,
624
+ miner_factory: ->(config) {
625
+ Insika::Harvest::MinerFactory.build(config,
626
+ utility_model: executor_extra[:settings_store]&.get&.[]("utility_model"), llm: executor_extra[:llm])
627
+ },
565
628
  event_stream: spine.event_stream
566
629
  ))
567
630
  bus.register(:gate_harvest,
@@ -575,7 +638,7 @@ module Insika
575
638
  harvest_store: spine.harvest_store,
576
639
  skill_store: skill_catalog.store,
577
640
  skill_catalog: skill_catalog,
578
- profile_source: profiles, criterion: nil, conversion_gate: nil,
641
+ profile_source: Insika::AgentDefaults.own(profiles), criterion: nil, conversion_gate: nil,
579
642
  event_stream: spine.event_stream
580
643
  ))
581
644
  bus.register(:rollback_harvest,
@@ -583,7 +646,7 @@ module Insika
583
646
  harvest_store: spine.harvest_store,
584
647
  skill_store: skill_catalog.store,
585
648
  skill_catalog: skill_catalog,
586
- profile_source: profiles, event_stream: spine.event_stream
649
+ profile_source: Insika::AgentDefaults.own(profiles), event_stream: spine.event_stream
587
650
  ))
588
651
  bus.register(:reject_harvest,
589
652
  Insika::Commands::RejectHarvest.new(
@@ -601,7 +664,7 @@ module Insika
601
664
  # dispatches this bus command; nothing else writes the demo agent.
602
665
  config_store = Insika::ConfigStore.new(store: spine.backend)
603
666
  demo_seeder = Insika::Demo::Seeder.new(
604
- profiles: profiles, store: spine.backend, session_store: spine.session_store,
667
+ profiles: Insika::AgentDefaults.own(profiles), store: spine.backend, session_store: spine.session_store,
605
668
  task_store: spine.task_store, outcome_store: spine.outcome_store,
606
669
  funnel_store: spine.funnel_store, followup_store: spine.followup_store,
607
670
  refinement_store: spine.refinement_store, pending_action_store: spine.pending_action_store,
@@ -618,7 +681,7 @@ module Insika
618
681
  # Infra spine (phase 1 output). Value object — the roots read these to promote
619
682
  # them to their historic public constants (SESSION_STORE, REGISTRY, ...).
620
683
  Spine = Struct.new(
621
- :backend, :event_stream, :session_store, :task_store, :checkpoint_store,
684
+ :backend, :event_stream, :session_store, :task_store, :llm_trace_store, :model_metrics_store, :checkpoint_store,
622
685
  :pending_action_store, :delegation_store, :memory_store, :memory_audit_store,
623
686
  :refinement_store,
624
687
  :harvest_store,
@@ -636,7 +699,7 @@ module Insika
636
699
  # deployment's overlay, or the same code registry at the base).
637
700
  Result = Struct.new(
638
701
  :backend, :event_stream,
639
- :session_store, :task_store, :checkpoint_store, :pending_action_store, :delegation_store,
702
+ :session_store, :task_store, :llm_trace_store, :model_metrics_store, :checkpoint_store, :pending_action_store, :delegation_store,
640
703
  :memory_store, :memory_audit_store, :refinement_store, :outbox_store, :shadow_pair_store,
641
704
  :inbound_log, :token_store,
642
705
  :budget_ledger, :circuit_state, :outcome_store, :funnel_store,
@@ -653,7 +716,7 @@ module Insika
653
716
  :profiles, :executor, :bus,
654
717
  keyword_init: true
655
718
  ) do
656
- def durable? = backend.is_a?(Insika::Stores::SQLite)
719
+ def durable? = Insika::Stores.durable?(backend)
657
720
  end
658
721
 
659
722
  # The save_artifact tool, registered OPTIONAL on the shared code registry
@@ -685,7 +748,9 @@ module Insika
685
748
  require "ruby_llm"
686
749
  require_relative "../tools/run_persona_eval"
687
750
  Insika::Tools::RunPersonaEval.new(
688
- golden_store: golden_store, profiles: graph.profiles, tool_registry: graph.tool_registry,
751
+ golden_store: golden_store, tool_registry: graph.tool_registry,
752
+ # the target as its turns see it: kit tools count in the derived swap list
753
+ profiles: Insika::AgentDefaults.for_turns(graph.profiles, settings_store),
689
754
  runtime: GraphChat.new(graph: graph), graph: graph, settings_store: settings_store,
690
755
  budget_ledger: graph.budget_ledger, event_stream: graph.event_stream, llm: llm
691
756
  )
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "json_schemer"
4
+
3
5
  module Insika
4
6
  # Workflow surface (COMPETITIVE-ANALYSIS). A workflow is a Ruby
5
7
  # callable orchestrating RubyLLM Agents/Workflows FROM WITHIN (RubyLLM First);
@@ -13,9 +15,7 @@ module Insika
13
15
  # A schema is any dry-schema-compatible validator: `#call(value)` returning a
14
16
  # result that responds to `#success?` and `#errors` (dry-schema's own contract).
15
17
  # As a batteries-included default, a plain JSON Schema Hash (the interlingua the
16
- # ToolDefinition already speaks) is accepted and validated by the zero-dependency
17
- # Schema below — "config over convention": bring dry-schema for richer contracts,
18
- # or hand a JSON Schema for the common case.
18
+ # ToolDefinition already speaks) is accepted and validated by JSONSchemer.
19
19
  module Workflow
20
20
  # Bundles a registered workflow's callable factory with its metadata (schemas +
21
21
  # description). Built lazily by WorkflowRegistry#definition — it does NOT resolve
@@ -74,13 +74,8 @@ module Insika
74
74
  end
75
75
  end
76
76
 
77
- # Zero-dependency instance validator for a plain JSON Schema Hash, speaking the
78
- # same safe subset (object/array/string/number/integer/boolean + enum) the
79
- # ToolDefinition validates its parameters against. It implements the SAME
80
- # `#call(value) -> result{#success?, #errors}` contract as dry-schema, so the
81
- # Definition treats both uniformly. Not a full JSON Schema engine (no
82
- # composition/$ref — the subset the insika supports everywhere); permissive on
83
- # unknown keys (JSON Schema's additionalProperties default).
77
+ # JSON Schema validation with the same result contract as dry-schema.
78
+ # References may resolve within the supplied document, never over the network.
84
79
  class Schema
85
80
  # dry-schema-compatible result. `errors` is { "field.path" => ["message", …] }.
86
81
  Result = Data.define(:errors) do
@@ -104,66 +99,33 @@ module Insika
104
99
  end
105
100
 
106
101
  @json_schema = Insika::Coercion.deep_stringify(json_schema)
102
+ @validator = JSONSchemer.schema(@json_schema, ref_resolver: ->(uri) {
103
+ raise Insika::ValidationError, "external schema reference is not allowed: #{uri}"
104
+ })
107
105
  end
108
106
 
109
107
  def call(value)
110
- Result.new(errors: collect(@json_schema, value, "").freeze)
111
- end
112
-
113
- private
114
-
115
- # -> { path => [messages] }. Empty = valid.
116
- def collect(schema, value, path)
117
- type = schema["type"].to_s
118
- return { key(path) => ["must be #{type}, got #{ruby_type(value)}"] } unless type.empty? || matches?(type, value)
119
-
120
108
  errors = {}
121
- case type
122
- when "object" then check_object(schema, value, path, errors)
123
- when "array" then check_array(schema, value, path, errors)
124
- end
125
- check_enum(schema, value, path, errors)
126
- errors
127
- end
128
-
129
- def check_object(schema, value, path, errors)
130
- props = schema["properties"] || {}
131
- Array(schema["required"]).each do |name|
132
- errors[key(join(path, name))] = ["is required"] unless value.key?(name.to_s) || value.key?(name.to_sym)
133
- end
134
- props.each do |name, subschema|
135
- next unless value.key?(name.to_s) || value.key?(name.to_sym)
136
-
137
- sub = value[name.to_s] || value[name.to_sym]
138
- errors.merge!(collect(subschema || {}, sub, join(path, name)))
109
+ @validator.validate(Insika::Coercion.deep_stringify(value)).each do |error|
110
+ path = error["data_pointer"].split("/").drop(1).map { |part| part.gsub("~1", "/").gsub("~0", "~") }.join(".")
111
+ if error["type"] == "required"
112
+ error.fetch("details").fetch("missing_keys").each do |name|
113
+ (errors[join(path, name)] ||= []) << "is required"
114
+ end
115
+ else
116
+ message = case error["type"]
117
+ when "object", "array", "string", "integer", "number", "boolean", "null"
118
+ "must be #{error['type']}, got #{ruby_type(error['data'])}"
119
+ when "enum" then "must be one of #{error['schema']['enum'].inspect}"
120
+ else error["error"]
121
+ end
122
+ (errors[path.empty? ? "(root)" : path] ||= []) << message
123
+ end
139
124
  end
125
+ Result.new(errors: errors.freeze)
140
126
  end
141
127
 
142
- def check_array(schema, value, path, errors)
143
- items = schema["items"]
144
- return if items.nil?
145
-
146
- value.each_with_index { |el, i| errors.merge!(collect(items, el, "#{join(path, i)}")) }
147
- end
148
-
149
- def check_enum(schema, value, path, errors)
150
- return unless schema.key?("enum")
151
-
152
- allowed = Array(schema["enum"])
153
- errors[key(path)] = ["must be one of #{allowed.inspect}"] unless allowed.include?(value)
154
- end
155
-
156
- def matches?(type, value)
157
- case type
158
- when "object" then value.is_a?(Hash)
159
- when "array" then value.is_a?(Array)
160
- when "string" then value.is_a?(String)
161
- when "integer" then value.is_a?(Integer) && !value.is_a?(TrueClass) && !value.is_a?(FalseClass)
162
- when "number" then value.is_a?(Numeric) && !value.is_a?(TrueClass) && !value.is_a?(FalseClass)
163
- when "boolean" then value == true || value == false
164
- else true # unknown type in the schema -> do not block (permissive)
165
- end
166
- end
128
+ private
167
129
 
168
130
  def ruby_type(value)
169
131
  case value
@@ -179,7 +141,6 @@ module Insika
179
141
  end
180
142
 
181
143
  def join(path, segment) = path.empty? ? segment.to_s : "#{path}.#{segment}"
182
- def key(path) = path.empty? ? "(root)" : path
183
144
  end
184
145
  end
185
146
  end
data/lib/insika.rb CHANGED
@@ -45,6 +45,7 @@ require_relative "insika/cron"
45
45
  require_relative "insika/schedule"
46
46
  require_relative "insika/schedule_store"
47
47
  require_relative "insika/schedule_engine"
48
+ require_relative "insika/model_registry"
48
49
  # the report destination — one record per run, no
49
50
  # versioning; the signing half of the sharing link (HMAC, no secret stored).
50
51
  require_relative "insika/artifact_store"
@@ -72,6 +73,7 @@ require_relative "insika/checkpoint"
72
73
  require_relative "insika/command"
73
74
  require_relative "insika/hooks"
74
75
  require_relative "insika/middleware"
76
+ require_relative "insika/memory_trim"
75
77
  # Production edge: windowed counters + the rate-limit/cost
76
78
  # Middleware. Both inert until configured (nil/0 = off).
77
79
  require_relative "insika/usage_ledger"
@@ -85,9 +87,11 @@ require_relative "insika/tool_output_compressor"
85
87
  # lazily, like the Executor's create_chat.
86
88
  require_relative "insika/safety/corpus"
87
89
  require_relative "insika/safety/detectors"
90
+ require_relative "insika/safety/prompt_echo"
88
91
  require_relative "insika/safety/safe_responses"
89
92
  require_relative "insika/safety/config"
90
93
  require_relative "insika/safety/moderator"
94
+ require_relative "insika/safety/judge_moderator"
91
95
  require_relative "insika/safety/output_filter"
92
96
  require_relative "insika/safety/input_guardrail"
93
97
  require_relative "insika/safety/output_validator"
@@ -104,9 +108,11 @@ require_relative "insika/context/providers/skill"
104
108
  require_relative "insika/context/providers/skill_trigger"
105
109
  require_relative "insika/context/providers/tool_search"
106
110
  require_relative "insika/context/providers/memory"
111
+ require_relative "insika/context/reranker"
107
112
  require_relative "insika/context/providers/knowledge"
108
113
  require_relative "insika/context/providers/session"
109
114
  require_relative "insika/context/providers/briefing"
115
+ require_relative "insika/context/providers/pending_confirmation"
110
116
  # the cache-prefix hash chain + the per-agent cache-hit series.
111
117
  # Both are referenced by the Executor at runtime, so they load before it.
112
118
  require_relative "insika/prefix_fingerprint"
@@ -123,8 +129,11 @@ require_relative "insika/prompt_catalog"
123
129
  require_relative "insika/plugin"
124
130
  require_relative "insika/plugin/loader"
125
131
  require_relative "insika/store"
132
+ require_relative "insika/stores/json_value"
126
133
  require_relative "insika/stores/memory"
134
+ require_relative "insika/stores/turn_counter"
127
135
  require_relative "insika/stores/sqlite"
136
+ require_relative "insika/stores/postgres"
128
137
  require_relative "insika/session_store"
129
138
  require_relative "insika/task_store"
130
139
  require_relative "insika/checkpoint_store"
@@ -147,6 +156,8 @@ require_relative "insika/skill_store"
147
156
  require_relative "insika/secret_masking"
148
157
  require_relative "insika/tool_store"
149
158
  require_relative "insika/tool_trace_store"
159
+ require_relative "insika/llm_trace_store"
160
+ require_relative "insika/model_metrics_store"
150
161
  require_relative "insika/tool_usage_report"
151
162
  require_relative "insika/context_trace_store"
152
163
  # the model-visible payload of one ask + its durable trace —
@@ -182,6 +193,8 @@ require_relative "insika/overlay_tool_registry"
182
193
  require_relative "insika/settings_store"
183
194
  require_relative "insika/llm_provider_store"
184
195
  require_relative "insika/llm_configurator"
196
+ require_relative "insika/agent_defaults"
197
+ require_relative "insika/kits"
185
198
  # LLM-first onboarding surface: serves start.md + models.json + the
186
199
  # public docs. Reads the settings/provider stores above at call-time (order is free).
187
200
  require_relative "insika/onboarding"
@@ -193,6 +206,7 @@ require_relative "insika/onboarding"
193
206
  require_relative "insika/evals/golden"
194
207
  require_relative "insika/evals/assertions"
195
208
  require_relative "insika/evals/judge"
209
+ require_relative "insika/evals/moderator_compare"
196
210
  require_relative "insika/evals/pairwise"
197
211
  require_relative "insika/evals/runner"
198
212
  require_relative "insika/evals/report"
@@ -217,8 +231,6 @@ require_relative "insika/refinement/gate"
217
231
  # candidate format and the gate's Report shape, so it loads after both.
218
232
  require_relative "insika/refinement/panel"
219
233
  require_relative "insika/mcp_store"
220
- require_relative "insika/mcp_http_client"
221
- require_relative "insika/mcp_tool_ingestor"
222
234
  # require "ruby_llm/mcp" itself is LAZY, inside .for — this require_relative only
223
235
  # loads the thin factory (see spec/insika/load_guard_spec.rb).
224
236
  require_relative "insika/mcp_client"
@@ -257,6 +269,8 @@ require_relative "insika/commands/gate_refinement"
257
269
  require_relative "insika/commands/resolve_refinement"
258
270
  require_relative "insika/commands/write_golden"
259
271
  require_relative "insika/commands/update_settings"
272
+ require_relative "insika/commands/write_kit"
273
+ require_relative "insika/commands/delete_kit"
260
274
  require_relative "insika/commands/upsert_llm_provider"
261
275
  require_relative "insika/commands/delete_llm_provider"
262
276
  require_relative "insika/commands/issue_tenant_token"
@@ -278,6 +292,7 @@ require_relative "insika/commands/revoke_contact"
278
292
  # the report destination's Studio mutation — delete one artifact
279
293
  # (a bus command, like every Studio mutation).
280
294
  require_relative "insika/commands/delete_artifact"
295
+ require_relative "insika/commands/review_guardrail_flag"
281
296
  # in-session compaction (RFC-0044) — the summarizer (pure, injected
282
297
  # ask) and the boundary math; the trigger lives in the Executor, the
283
298
  # persistence in SessionStore, the read path in the Session provider.
@@ -323,7 +338,6 @@ require_relative "insika/commands/write_data_tool"
323
338
  require_relative "insika/commands/delete_data_tool"
324
339
  require_relative "insika/commands/restore_data_tool"
325
340
  require_relative "insika/commands/import_tools"
326
- require_relative "insika/commands/import_mcp_tools"
327
341
  require_relative "insika/commands/refresh_mcp_tools"
328
342
  # the demo seed (OSS onboarding): one bundled agent + golden
329
343
  # set + the Seeder that populates every loop's stores at once, and its bus
@@ -344,6 +358,7 @@ require_relative "insika/loop_detector"
344
358
  require_relative "insika/turn_budget"
345
359
  require_relative "insika/balloon_splitter"
346
360
  require_relative "insika/turn_output"
361
+ require_relative "insika/text_tool_call"
347
362
  require_relative "insika/turn_state"
348
363
  require_relative "insika/turn_timing"
349
364
  require_relative "insika/capability/resolved_tool"
@@ -352,6 +367,7 @@ require_relative "insika/spoken_transcript"
352
367
  require_relative "insika/tool_envelope"
353
368
  require_relative "insika/tool_assembly"
354
369
  require_relative "insika/chat_builder"
370
+ require_relative "insika/shared_conversations"
355
371
  require_relative "insika/executor"
356
372
  require_relative "insika/shutdown"
357
373
  require_relative "insika/tick"