insika 0.2.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (300) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +325 -17
  3. data/README.md +52 -11
  4. data/bin/insika +895 -8
  5. data/bin/insika-router +87 -0
  6. data/docs/AGENTS.md +229 -163
  7. data/docs/API.md +56 -0
  8. data/docs/ARCHITECTURE.md +3 -2
  9. data/docs/ARTIFACTS.md +95 -0
  10. data/docs/BENCHMARK.md +2 -2
  11. data/docs/CHANNELS.md +111 -21
  12. data/docs/CONTEXT.md +132 -19
  13. data/docs/DEMO.md +80 -0
  14. data/docs/DEPLOY.md +75 -4
  15. data/docs/EMBEDDING.md +1 -1
  16. data/docs/EVALS.md +146 -2
  17. data/docs/FACTS.md +135 -0
  18. data/docs/HARVEST.md +116 -0
  19. data/docs/KNOWLEDGE.md +290 -0
  20. data/docs/LOADTEST.md +16 -10
  21. data/docs/MEDIA.md +128 -0
  22. data/docs/OBSERVABILITY.md +65 -6
  23. data/docs/OUTCOMES.md +137 -0
  24. data/docs/PLUGINS.md +51 -6
  25. data/docs/POLICY.md +216 -0
  26. data/docs/REFINEMENT.md +17 -12
  27. data/docs/RELEASING.md +27 -0
  28. data/docs/ROUTER.md +213 -0
  29. data/docs/RUNNING-LOCAL.md +6 -6
  30. data/docs/SCHEDULING.md +121 -0
  31. data/docs/SECURITY.md +79 -2
  32. data/docs/SKILLS.md +13 -4
  33. data/docs/SOAK.md +127 -0
  34. data/docs/TEMPLATES.md +134 -0
  35. data/docs/TOOLS.md +220 -27
  36. data/docs/WHY.md +1 -1
  37. data/docs/WORKFLOWS.md +2 -2
  38. data/docs/_includes/head_custom.html +5 -0
  39. data/docs/_includes/title.html +13 -0
  40. data/docs/_sass/color_schemes/insika.scss +32 -0
  41. data/docs/_sass/custom/custom.scss +199 -0
  42. data/docs/_sass/custom/setup.scss +26 -0
  43. data/docs/assets/img/favicon.svg +7 -0
  44. data/docs/assets/img/insika-mark.svg +7 -0
  45. data/docs/core-concepts.md +21 -0
  46. data/docs/domain.md +115 -0
  47. data/docs/improve.md +20 -0
  48. data/docs/index.md +9 -6
  49. data/docs/integrate.md +20 -0
  50. data/docs/operate.md +13 -6
  51. data/docs/prompts/ADD-TOOL.md +118 -0
  52. data/docs/prompts/DIAGNOSE-TURN.md +65 -0
  53. data/docs/prompts/GO-LIVE.md +138 -0
  54. data/docs/prompts/RUN-EXAMPLES.md +70 -0
  55. data/docs/reference.md +19 -0
  56. data/docs/ship.md +10 -2
  57. data/docs/start-here.md +18 -0
  58. data/lib/insika/agent_profile.rb +202 -10
  59. data/lib/insika/artifact_signing.rb +82 -0
  60. data/lib/insika/artifact_store.rb +160 -0
  61. data/lib/insika/balloon_splitter.rb +102 -0
  62. data/lib/insika/budget_ledger.rb +34 -3
  63. data/lib/insika/cache_series_store.rb +49 -0
  64. data/lib/insika/channel_delivery.rb +119 -14
  65. data/lib/insika/channels/relay.rb +77 -3
  66. data/lib/insika/channels/web.rb +2 -2
  67. data/lib/insika/chat_builder.rb +105 -2
  68. data/lib/insika/checkpoint_store.rb +16 -0
  69. data/lib/insika/commands/agent_payload.rb +3 -3
  70. data/lib/insika/commands/backfill_knowledge.rb +145 -0
  71. data/lib/insika/commands/cancel_followup.rb +49 -0
  72. data/lib/insika/commands/delete_artifact.rb +35 -0
  73. data/lib/insika/commands/delete_concept.rb +34 -0
  74. data/lib/insika/commands/delete_mcp.rb +6 -2
  75. data/lib/insika/commands/delete_tenant_data.rb +107 -0
  76. data/lib/insika/commands/export_customer_memory.rb +48 -0
  77. data/lib/insika/commands/forget_customer.rb +117 -0
  78. data/lib/insika/commands/freeze_funnel_baseline.rb +113 -0
  79. data/lib/insika/commands/gate_harvest.rb +138 -0
  80. data/lib/insika/commands/gate_refinement.rb +1 -1
  81. data/lib/insika/commands/judge_shadow_pairs.rb +124 -0
  82. data/lib/insika/commands/memory_forget_fact.rb +20 -4
  83. data/lib/insika/commands/memory_put_fact.rb +23 -4
  84. data/lib/insika/commands/promote_harvest.rb +130 -0
  85. data/lib/insika/commands/record_outcome.rb +46 -0
  86. data/lib/insika/commands/record_shadow_reply.rb +68 -0
  87. data/lib/insika/commands/refresh_mcp_tools.rb +47 -0
  88. data/lib/insika/commands/reject_harvest.rb +38 -0
  89. data/lib/insika/commands/resolve_proposal.rb +108 -0
  90. data/lib/insika/commands/restore_concept.rb +34 -0
  91. data/lib/insika/commands/revoke_contact.rb +49 -0
  92. data/lib/insika/commands/rollback_harvest.rb +86 -0
  93. data/lib/insika/commands/run_distillation.rb +186 -0
  94. data/lib/insika/commands/run_harvest.rb +393 -0
  95. data/lib/insika/commands/seed_demo_data.rb +31 -0
  96. data/lib/insika/commands/send_message.rb +103 -6
  97. data/lib/insika/commands/session_purge.rb +67 -0
  98. data/lib/insika/commands/upsert_mcp.rb +6 -3
  99. data/lib/insika/commands/write_concept.rb +57 -0
  100. data/lib/insika/contact_store.rb +183 -0
  101. data/lib/insika/context/builder.rb +21 -3
  102. data/lib/insika/context/fragment.rb +7 -3
  103. data/lib/insika/context/priority.rb +5 -0
  104. data/lib/insika/context/provider.rb +17 -3
  105. data/lib/insika/context/providers/briefing.rb +96 -0
  106. data/lib/insika/context/providers/knowledge.rb +108 -0
  107. data/lib/insika/context/providers/memory.rb +16 -7
  108. data/lib/insika/context/providers/prompt.rb +57 -23
  109. data/lib/insika/context/providers/skill.rb +2 -0
  110. data/lib/insika/context/providers/tool_search.rb +2 -0
  111. data/lib/insika/context_trace_store.rb +38 -2
  112. data/lib/insika/cron.rb +189 -0
  113. data/lib/insika/demo/agent_attrs.rb +43 -0
  114. data/lib/insika/demo/golden_cases.rb +81 -0
  115. data/lib/insika/demo/seeder.rb +336 -0
  116. data/lib/insika/distill.rb +224 -0
  117. data/lib/insika/distill_engine.rb +169 -0
  118. data/lib/insika/doctor.rb +889 -11
  119. data/lib/insika/dsl/definition.rb +3 -2
  120. data/lib/insika/dsl/runtime.rb +69 -82
  121. data/lib/insika/dsl/server_boot.rb +92 -1
  122. data/lib/insika/dsl/system.rb +10 -2
  123. data/lib/insika/dsl.rb +186 -5
  124. data/lib/insika/edge_limiter.rb +28 -7
  125. data/lib/insika/env_schema.rb +25 -3
  126. data/lib/insika/errors.rb +11 -0
  127. data/lib/insika/evals/assertions.rb +3 -2
  128. data/lib/insika/evals/golden.rb +41 -4
  129. data/lib/insika/evals/judge.rb +47 -2
  130. data/lib/insika/evals/pairwise.rb +29 -6
  131. data/lib/insika/evals/persona.rb +98 -0
  132. data/lib/insika/evals/runner.rb +9 -0
  133. data/lib/insika/evals/simulator.rb +225 -0
  134. data/lib/insika/evals/transport.rb +83 -1
  135. data/lib/insika/event_stream.rb +10 -0
  136. data/lib/insika/evidence.rb +183 -0
  137. data/lib/insika/executor.rb +869 -76
  138. data/lib/insika/followup_engine.rb +207 -0
  139. data/lib/insika/followup_policy.rb +198 -0
  140. data/lib/insika/followup_store.rb +306 -0
  141. data/lib/insika/funnel_declaration.rb +106 -0
  142. data/lib/insika/funnel_fold.rb +179 -0
  143. data/lib/insika/funnel_store.rb +163 -0
  144. data/lib/insika/golden_store.rb +17 -2
  145. data/lib/insika/grounding/matcher.rb +69 -0
  146. data/lib/insika/grounding.rb +44 -0
  147. data/lib/insika/harvest/conversion_gate.rb +159 -0
  148. data/lib/insika/harvest/criterion.rb +98 -0
  149. data/lib/insika/harvest/gate.rb +194 -0
  150. data/lib/insika/harvest/negative_list.rb +199 -0
  151. data/lib/insika/harvest.rb +241 -0
  152. data/lib/insika/harvest_engine.rb +193 -0
  153. data/lib/insika/harvest_store.rb +548 -0
  154. data/lib/insika/knowledge.rb +680 -0
  155. data/lib/insika/knowledge_store.rb +140 -0
  156. data/lib/insika/mcp_client.rb +94 -0
  157. data/lib/insika/mcp_json.rb +74 -0
  158. data/lib/insika/mcp_live_tool.rb +43 -0
  159. data/lib/insika/mcp_store.rb +98 -26
  160. data/lib/insika/mcp_tool_ingestor.rb +30 -8
  161. data/lib/insika/mcp_tool_registry.rb +100 -0
  162. data/lib/insika/media.rb +382 -0
  163. data/lib/insika/memory_audit_store.rb +85 -0
  164. data/lib/insika/memory_store.rb +264 -23
  165. data/lib/insika/message_origin.rb +7 -2
  166. data/lib/insika/middleware.rb +9 -0
  167. data/lib/insika/model_visible.rb +87 -0
  168. data/lib/insika/model_visible_trace_store.rb +66 -0
  169. data/lib/insika/onboarding.rb +22 -1
  170. data/lib/insika/outbox_store.rb +42 -4
  171. data/lib/insika/outcome_store.rb +147 -0
  172. data/lib/insika/overlay_tool_registry.rb +37 -17
  173. data/lib/insika/packaging.rb +163 -0
  174. data/lib/insika/parity/criterion.rb +79 -0
  175. data/lib/insika/parity/verdict.rb +318 -0
  176. data/lib/insika/prefix_fingerprint.rb +58 -0
  177. data/lib/insika/profile_source.rb +29 -1
  178. data/lib/insika/prompt_catalog.rb +10 -0
  179. data/lib/insika/proposal_store.rb +271 -0
  180. data/lib/insika/queue_policy.rb +4 -1
  181. data/lib/insika/refinement/proposer.rb +1 -1
  182. data/lib/insika/reliability.rb +32 -6
  183. data/lib/insika/retention.rb +316 -0
  184. data/lib/insika/router/app.rb +157 -0
  185. data/lib/insika/router/backend_pool.rb +98 -0
  186. data/lib/insika/router/hash_ring.rb +55 -0
  187. data/lib/insika/router/proxy_body.rb +34 -0
  188. data/lib/insika/router/session_key.rb +54 -0
  189. data/lib/insika/router.rb +18 -0
  190. data/lib/insika/routing.rb +101 -0
  191. data/lib/insika/safety/config.rb +44 -4
  192. data/lib/insika/safety/corpus.rb +255 -0
  193. data/lib/insika/safety/detectors.rb +32 -113
  194. data/lib/insika/safety/factory.rb +15 -2
  195. data/lib/insika/safety/grounding_enforcer.rb +59 -0
  196. data/lib/insika/safety/grounding_validator.rb +49 -0
  197. data/lib/insika/safety/input_guardrail.rb +1 -1
  198. data/lib/insika/safety/moderator.rb +1 -1
  199. data/lib/insika/safety/output_filter.rb +9 -5
  200. data/lib/insika/safety/output_validator.rb +11 -5
  201. data/lib/insika/schedule.rb +177 -0
  202. data/lib/insika/schedule_engine.rb +314 -0
  203. data/lib/insika/schedule_store.rb +208 -0
  204. data/lib/insika/schema_guard.rb +35 -0
  205. data/lib/insika/server/app.rb +247 -17
  206. data/lib/insika/server/rack_app.rb +21 -1
  207. data/lib/insika/server/responses.rb +40 -1
  208. data/lib/insika/session_actor.rb +8 -4
  209. data/lib/insika/session_store.rb +65 -2
  210. data/lib/insika/settings_store.rb +10 -0
  211. data/lib/insika/shadow_pair_store.rb +258 -0
  212. data/lib/insika/skill_catalog.rb +16 -0
  213. data/lib/insika/soak/envelope.rb +140 -0
  214. data/lib/insika/soak/report.rb +392 -0
  215. data/lib/insika/soak/runner.rb +554 -0
  216. data/lib/insika/steer_injector.rb +21 -10
  217. data/lib/insika/store.rb +10 -1
  218. data/lib/insika/stores/memory.rb +6 -0
  219. data/lib/insika/stores/sqlite.rb +8 -0
  220. data/lib/insika/studio/app.rb +1437 -54
  221. data/lib/insika/studio/assets/dist/application.css +1 -1
  222. data/lib/insika/studio/assets/dist/application.js +23 -23
  223. data/lib/insika/studio/assets/dist/favicon.svg +6 -0
  224. data/lib/insika/studio/forms.rb +308 -10
  225. data/lib/insika/studio/nav_icons.rb +19 -1
  226. data/lib/insika/studio/views/_agent_tab_cache.erb +25 -0
  227. data/lib/insika/studio/views/_agent_tab_config.erb +514 -0
  228. data/lib/insika/studio/views/_agent_tab_history.erb +24 -0
  229. data/lib/insika/studio/views/_agent_tab_loops.erb +54 -0
  230. data/lib/insika/studio/views/_agent_tab_memory.erb +51 -0
  231. data/lib/insika/studio/views/_agent_tab_outcomes.erb +31 -0
  232. data/lib/insika/studio/views/_agent_tab_prompts.erb +108 -0
  233. data/lib/insika/studio/views/_agent_tab_skills.erb +38 -0
  234. data/lib/insika/studio/views/_agents_master.erb +44 -0
  235. data/lib/insika/studio/views/_message.erb +49 -32
  236. data/lib/insika/studio/views/agent_detail.erb +62 -278
  237. data/lib/insika/studio/views/agents.erb +71 -54
  238. data/lib/insika/studio/views/approvals.erb +4 -1
  239. data/lib/insika/studio/views/artifact.erb +23 -0
  240. data/lib/insika/studio/views/artifacts.erb +59 -0
  241. data/lib/insika/studio/views/chats.erb +4 -1
  242. data/lib/insika/studio/views/customer.erb +94 -0
  243. data/lib/insika/studio/views/customers.erb +32 -0
  244. data/lib/insika/studio/views/evals.erb +6 -3
  245. data/lib/insika/studio/views/facts.erb +133 -0
  246. data/lib/insika/studio/views/followups.erb +125 -0
  247. data/lib/insika/studio/views/funnel.erb +106 -0
  248. data/lib/insika/studio/views/harvest.erb +234 -0
  249. data/lib/insika/studio/views/home.erb +106 -66
  250. data/lib/insika/studio/views/knowledge.erb +123 -0
  251. data/lib/insika/studio/views/layout.erb +15 -11
  252. data/lib/insika/studio/views/mcp.erb +174 -80
  253. data/lib/insika/studio/views/parity.erb +147 -0
  254. data/lib/insika/studio/views/playground.erb +7 -1
  255. data/lib/insika/studio/views/session.erb +233 -124
  256. data/lib/insika/studio/views/settings.erb +41 -1
  257. data/lib/insika/studio/views/skills.erb +1 -2
  258. data/lib/insika/studio/views/system_files.erb +1 -1
  259. data/lib/insika/studio/views/task.erb +13 -0
  260. data/lib/insika/studio/views/tasks.erb +4 -1
  261. data/lib/insika/studio/views/tools.erb +24 -10
  262. data/lib/insika/task_store.rb +21 -1
  263. data/lib/insika/templates/browser-agent/README.md +36 -0
  264. data/lib/insika/templates/browser-agent/agent.rb +49 -0
  265. data/lib/insika/templates/daily-digest/README.md +38 -0
  266. data/lib/insika/templates/daily-digest/agent.rb +77 -0
  267. data/lib/insika/templates/repo-explorer/README.md +36 -0
  268. data/lib/insika/templates/repo-explorer/agent.rb +45 -0
  269. data/lib/insika/templates/research-analyst/README.md +26 -0
  270. data/lib/insika/templates/research-analyst/agent.rb +58 -0
  271. data/lib/insika/templates/review-panel/README.md +20 -0
  272. data/lib/insika/templates/review-panel/agent.rb +50 -0
  273. data/lib/insika/templates/travel-planner/README.md +35 -0
  274. data/lib/insika/templates/travel-planner/agent.rb +87 -0
  275. data/lib/insika/templates.rb +112 -0
  276. data/lib/insika/testing/store_contract.rb +27 -6
  277. data/lib/insika/tick.rb +47 -11
  278. data/lib/insika/timezone.rb +45 -0
  279. data/lib/insika/tool_definition.rb +17 -7
  280. data/lib/insika/tool_envelope.rb +69 -0
  281. data/lib/insika/tool_manifest.rb +5 -1
  282. data/lib/insika/tools/data_defined_tool.rb +10 -0
  283. data/lib/insika/tools/generate_image.rb +89 -0
  284. data/lib/insika/tools/load_knowledge.rb +74 -0
  285. data/lib/insika/tools/run_persona_eval.rb +328 -0
  286. data/lib/insika/tools/save_artifact.rb +95 -0
  287. data/lib/insika/tools/schedule_followup.rb +164 -0
  288. data/lib/insika/tools/tts.rb +47 -0
  289. data/lib/insika/tools/update_briefing.rb +126 -0
  290. data/lib/insika/turn_output.rb +1 -1
  291. data/lib/insika/turn_state.rb +49 -1
  292. data/lib/insika/turn_timing.rb +22 -2
  293. data/lib/insika/version.rb +1 -1
  294. data/lib/insika/vitals.rb +84 -0
  295. data/lib/insika/wiring/graph.rb +472 -13
  296. data/lib/insika/wiring/graph_chat.rb +102 -0
  297. data/lib/insika.rb +149 -3
  298. metadata +181 -5
  299. data/docs/build.md +0 -14
  300. data/docs/understand.md +0 -10
data/bin/insika CHANGED
@@ -4,8 +4,9 @@
4
4
  # `insika` CLI — strict config + doctor --fix. The operator's
5
5
  # DX front door to the strict-config discipline:
6
6
  #
7
- # insika doctor [--fix] [--json] diagnose the deployment config; --fix applies
8
- # the safe autofixes; exits non-zero on any error
7
+ # insika doctor [--fix] [--json] [--domain] diagnose the deployment config; --fix applies
8
+ # the safe autofixes; --domain appends the domain inventory;
9
+ # exits non-zero on any error
9
10
  # insika env [--json] list the known config keys + current (masked) values
10
11
  # insika version print the engine version
11
12
  # insika help this text
@@ -15,6 +16,7 @@
15
16
  # DEEPSEEK, no seeding. So it is safe to run against a live deployment's volume.
16
17
 
17
18
  require "optparse"
19
+ require "fileutils"
18
20
  require_relative "../lib/insika"
19
21
 
20
22
  module Insika
@@ -27,9 +29,19 @@ module Insika
27
29
  when "doctor" then doctor(argv)
28
30
  when "env" then env(argv)
29
31
  when "refine" then refine(argv)
32
+ when "harvest" then harvest(argv)
33
+ when "harvest:negative" then harvest_negative_import(argv)
34
+ when "harvest:criterion" then harvest_criterion_check(argv)
35
+ when "knowledge:backfill" then knowledge_backfill(argv)
36
+ when "knowledge:export" then knowledge_export(argv)
37
+ when "soak" then exit(Insika::Soak::Runner.main(argv, stdout: $stdout, stderr: $stderr))
30
38
  when "evals:import" then evals_import(argv)
31
39
  when "evals:export" then evals_export(argv)
32
40
  when "evals:baseline" then evals_baseline(argv)
41
+ when "evals:simulate" then evals_simulate(argv)
42
+ when "demo:seed" then demo_seed(argv)
43
+ when "mcp" then mcp(argv)
44
+ when "new" then new_from_template(argv)
33
45
  when "version", "--version", "-v" then puts "insika #{Insika::VERSION}"
34
46
  when "help", "--help", "-h" then puts usage
35
47
  else
@@ -45,28 +57,66 @@ module Insika
45
57
  Usage: insika <command> [options]
46
58
 
47
59
  Commands:
48
- doctor [--fix] [--json] diagnose the deployment configuration
60
+ doctor [--fix] [--json] [--domain] diagnose the deployment configuration
49
61
  env [--json] list known config keys + current (masked) values
50
62
  refine --agent ID [...] report what broke in an agent's real traffic
63
+ harvest --agent ID [...] mine one window of finished traffic
64
+ harvest:negative import --agent ID --file F
65
+ seed the profile's negative list from a rules file
66
+ harvest:criterion check --file F
67
+ strict-load the frozen conversion criterion
68
+ knowledge:backfill --agent ID [--since ISO8601] [--json]
69
+ the recovery path for post-turn extraction: replays
70
+ the agent's stored sessions through the same
71
+ extractor a live turn uses
72
+ knowledge:export --agent ID [--out DIR] [--tenant T] [--format md|graphml]
73
+ writes one <name>.md per learned concept under DIR —
74
+ a dump, not a converter (the storage format IS the
75
+ export format), for okf-gem/graphify tooling; --format
76
+ graphml writes one knowledge.graphml graph file instead
77
+ soak --run | --verify FILE | --preflight the 72h soak
51
78
  evals:import [--dir D] load the golden corpus into the store (editable in Studio)
52
79
  evals:export [--dir D] write the stored cases back out as YAML
53
80
  evals:baseline <sub> the accepted state per agent: show | import | export
81
+ evals:simulate --persona F --target <agent|url>
82
+ generate a simulated conversation against an
83
+ agent: a persona model talks to the
84
+ target until max_turns or a stop marker; the
85
+ transcript is scored by the judge panel.
86
+ Requires --staging or --eval-profile (a simulated
87
+ run must not write for real); --eval-profile is
88
+ verified against the target's derived
89
+ side-effect tools (--eval-tools).
90
+ demo:seed [--force] populate the "demo-store" agent with enough data to see
91
+ every Studio loop at once (funnel+baseline, follow-ups,
92
+ refinement runs, approvals, distillation, evals+baseline)
93
+ mcp list the configured MCP instances
94
+ mcp add --name N [...] add/edit an instance (see mcp add --help)
95
+ mcp remove NAME delete an instance
96
+ mcp import FILE.json upsert every entry of a `mcpServers` JSON document
97
+ mcp test NAME connect live, list its tools, print them (or the error)
98
+ mcp refresh NAME same as test, plus caches the result for display
99
+ new --list the template gallery: trail + one-line description
100
+ new <template> [dir] copy a template (default dir: ./<template>) and print the run line
54
101
  version print the engine version
55
102
  help show this help
56
103
 
57
104
  Environment:
58
105
  INSIKA_DB SQLite path (durable). Unset -> ephemeral memory.
59
106
  INSIKA_CONFIG_STRICT when truthy, the server refuses boot on any finding.
107
+ INSIKA_MCP_STDIO set to 1 to allow stdio MCP instances to start
108
+ (arbitrary command execution by config — off by default)
60
109
  TXT
61
110
  end
62
111
 
63
- # insika doctor [--fix] [--json]
112
+ # insika doctor [--fix] [--json] [--domain]
64
113
  def doctor(argv)
65
- opts = { fix: false, json: false }
114
+ opts = { fix: false, json: false, domain: false }
66
115
  OptionParser.new do |o|
67
- o.banner = "Usage: insika doctor [--fix] [--json]"
116
+ o.banner = "Usage: insika doctor [--fix] [--json] [--domain]"
68
117
  o.on("--fix", "apply the safe autofixes, then re-diagnose") { opts[:fix] = true }
69
118
  o.on("--json", "emit the report as JSON") { opts[:json] = true }
119
+ o.on("--domain", "append the domain inventory (declared personas, corpora, funnels, evidence)") { opts[:domain] = true }
70
120
  end.parse!(argv)
71
121
 
72
122
  doc = build_doctor
@@ -74,9 +124,12 @@ module Insika
74
124
 
75
125
  if opts[:json]
76
126
  require "json"
77
- puts JSON.pretty_generate(report.to_h)
127
+ envelope = report.to_h
128
+ envelope["domain"] = doc.domain.to_h if opts[:domain]
129
+ puts JSON.pretty_generate(envelope)
78
130
  else
79
131
  puts report.to_s(color: $stdout.tty?)
132
+ puts doc.domain.to_s if opts[:domain]
80
133
  end
81
134
  exit(report.ok? ? 0 : 1)
82
135
  end
@@ -258,8 +311,552 @@ def baseline_export(path)
258
311
  puts "exported #{merged.size} case(s) from #{store.agents.size} agent baseline(s) to #{path}"
259
312
  end
260
313
 
314
+ # insika evals:simulate --persona F --target <agent|url> [--staging|--eval-profile]
315
+ #
316
+ # Generates a SIMULATED conversation: a persona model (the cheap
317
+ # utility_model) plays a customer with the persona as its whole instruction — and
318
+ # one hard rule: it may ONLY assert what the persona's `knows` contains. The
319
+ # target agent answers over the same Transport the replay uses (HttpTransport
320
+ # against a running deployment, or a thin A2A transport when --target is a URL).
321
+ #
322
+ # SAFETY (fixed in the spec): a simulated conversation must not write for real,
323
+ # because `simulated: true` marks the transcript, it does not disarm the tools.
324
+ # So this CLI REQUIRES one of:
325
+ # --staging the operator declares the target is a staging deployment
326
+ # --eval-profile the target runs an eval profile (side-effect tools swapped
327
+ # for dry-runs) — a VERIFIED declaration, not a trust-me flag:
328
+ # the CLI derives the target's side-effect tools from the
329
+ # deployment's own registry (GET /v1/agents/:id) or the local
330
+ # store (INSIKA_DB), and requires --eval-tools to cover every
331
+ # one of them. An A2A target (no reachable registry) requires
332
+ # the explicit --eval-tools list.
333
+ # and refuses to run without one. Every run is marked `simulated: true`.
334
+ def evals_simulate(argv)
335
+ require "yaml"
336
+ require "json"
337
+ require "fileutils"
338
+ require "time"
339
+ opts = { base_url: ENV["INSIKA_URL"] || "http://localhost:9292",
340
+ token: ENV["OPENCLAW_GATEWAY_TOKEN"] || ENV["ADMIN_TOKEN"] || "local-demo",
341
+ timeout: 120, persona_model: ENV["EVAL_PERSONA_MODEL"],
342
+ judge_model: ENV["EVAL_JUDGE_MODEL"], judge_provider: ENV["EVAL_JUDGE_PROVIDER"] }
343
+ OptionParser.new do |o|
344
+ o.banner = "Usage: insika evals:simulate --persona F --target <agent|url> [options]"
345
+ o.on("--persona FILE", "the persona YAML — a case with a `persona:` key (required)") { |v| opts[:persona] = v }
346
+ o.on("--target T", "agent id (over --base-url) or an A2A http(s) URL (required)") { |v| opts[:target] = v }
347
+ o.on("--base-url URL", "deployment base URL (default #{opts[:base_url]})") { |v| opts[:base_url] = v }
348
+ o.on("--token T", "gateway token (default OPENCLAW_GATEWAY_TOKEN/local-demo)") { |v| opts[:token] = v }
349
+ o.on("--timeout N", Integer, "per-turn read timeout, and the A2A poll budget in seconds (default 120)") { |v| opts[:timeout] = v }
350
+ o.on("--staging", "the target is a staging deployment (side effects acceptable)") { opts[:staging] = true }
351
+ o.on("--eval-profile", "the target runs an eval profile (side-effect tools swapped for fakes)") { opts[:eval_profile] = true }
352
+ o.on("--eval-tools LIST", "comma-separated side-effect tools the eval profile swaps (verified against the derived set; required for an A2A target)") { |v| opts[:eval_tools] = v }
353
+ o.on("--persona-model MODEL", "the model playing the customer (default settings utility_model)") { |v| opts[:persona_model] = v }
354
+ o.on("--judge-model MODEL", "score the transcript's rubric with this model") { |v| opts[:judge_model] = v }
355
+ o.on("--judge-provider PROVIDER", "provider for the judge model") { |v| opts[:judge_provider] = v }
356
+ o.on("--pairwise", "compare the simulated transcript against the case's `reference:`") { opts[:pairwise] = true }
357
+ o.on("--conv ID", "conversation id (default: sim-<case id>-<random>, unique per run)") { |v| opts[:conv] = v }
358
+ o.on("--out FILE", "write the JSON report here") { |v| opts[:out] = v }
359
+ end.parse!(argv)
360
+
361
+ abort "insika evals:simulate: --persona is required" if opts[:persona].to_s.empty?
362
+ abort "insika evals:simulate: --target is required (<agent> or an A2A <url>)" if opts[:target].to_s.empty?
363
+ unless opts[:staging] || opts[:eval_profile]
364
+ abort "insika evals:simulate: a simulated run must not write for real — pass --staging " \
365
+ "(the target is a staging deployment) or --eval-profile (side-effect tools are swapped)"
366
+ end
367
+
368
+ golden = load_sim_persona(opts[:persona])
369
+ # --target names the agent actually simulated. An agent id overrides the
370
+ # persona file's `agent:` (the file is a reusable script); a URL means an A2A
371
+ # endpoint, where the agent id is irrelevant (the transport ignores it).
372
+ a2a_target = opts[:target].to_s.match?(%r{\Ahttps?://})
373
+ target_agent = a2a_target ? golden.agent : opts[:target].to_s
374
+ safety = build_sim_safety(opts, golden, target_agent, a2a_target)
375
+
376
+ settings = eval_settings_from_env
377
+ ask = build_persona_ask(opts, settings)
378
+ judge = build_sim_judge(opts, settings)
379
+
380
+ transport = build_sim_transport(opts)
381
+ simulator = Insika::Evals::Simulator.new(transport: transport, ask: ask, safety: safety)
382
+ # Unique per run: two runs sharing a conv share the deployment SESSION, so the
383
+ # second inherits the first's history — including across the A/B arms of a
384
+ # paired comparison, which it silently invalidates.
385
+ require "securerandom"
386
+ conv = opts[:conv] || "sim-#{golden.id}-#{SecureRandom.hex(4)}"
387
+ run = simulator.run(persona: golden.persona, agent: target_agent, conv: conv)
388
+
389
+ verdict = judge && judge.score_conversation(
390
+ rubric: golden.rubric, transcript: run.transcript, policy: golden.policy,
391
+ min_score: golden.min_score || Insika::Evals::Judge::DEFAULT_MIN_SCORE
392
+ )
393
+
394
+ pairwise = if opts[:pairwise] && golden.reference?
395
+ p = build_sim_pairwise(opts, settings)
396
+ if p
397
+ p.compare_texts(
398
+ ours: Insika::Evals::Pairwise.transcript_text(run.transcript),
399
+ theirs: Insika::Evals::Pairwise.reference_transcript(golden.reference_messages),
400
+ vs: golden.human_assisted? ? "human-assisted" : "agent"
401
+ )
402
+ end
403
+ end
404
+
405
+ at = Time.now.utc.iso8601
406
+ report = { "simulated" => true, "at" => at, "id" => golden.id, "agent" => golden.agent,
407
+ "conv" => conv, "stop" => run.stop.to_s, "turns" => run.turns, "error" => run.error,
408
+ "persona" => golden.persona.to_h, "transcript" => run.transcript }
409
+ if verdict
410
+ report["judge"] = { "score" => verdict.score, "pass" => verdict.pass, "reason" => verdict.reason }
411
+ # A split panel must be visible in the report (the whole point of a panel
412
+ # over a lone judge): one entry per judge model, in settings order.
413
+ report["judge"]["judges"] = verdict.judges if verdict.judges
414
+ end
415
+ report["pairwise"] = { "outcome" => pairwise.outcome, "vs" => pairwise.vs, "reason" => pairwise.reason } if pairwise
416
+ puts simulate_markdown(report, verdict, pairwise)
417
+ if opts[:out]
418
+ FileUtils.mkdir_p(File.dirname(File.expand_path(opts[:out])))
419
+ File.write(opts[:out], JSON.pretty_generate(report))
420
+ puts "report: #{opts[:out]}"
421
+ end
422
+ exit(run.stop == :error || (verdict && !verdict.pass) ? 1 : 0)
423
+ rescue Insika::Evals::PersonaLoader::InvalidPersona, Insika::Evals::GoldenLoader::InvalidGolden,
424
+ Insika::Evals::Simulator::UnsafeTarget => e
425
+ warn "insika evals:simulate: #{e.message}"
426
+ exit 2
427
+ end
428
+
429
+ # --persona: a case YAML carrying a `persona:` key (the alternative
430
+ # shape). A case without turns but with a persona is the only thing this CLI
431
+ # drives — a scripted case has nothing to simulate.
432
+ def load_sim_persona(path)
433
+ raw = YAML.safe_load(File.read(path), permitted_classes: [], aliases: false) || {}
434
+ golden = Insika::Evals::GoldenLoader.build(raw, source: path)
435
+ raise Insika::Evals::GoldenLoader::InvalidGolden, "#{path}: needs a `persona:` (a simulated case)" unless golden.simulated?
436
+
437
+ golden
438
+ rescue Errno::ENOENT => e
439
+ raise Insika::Evals::GoldenLoader::InvalidGolden, e.message
440
+ end
441
+
442
+ # The safety gate for a run. --staging is a straight declaration. --eval-profile
443
+ # is a VERIFIED one: the CLI derives the target's side-effect tools (see
444
+ # derive_side_effect_tools) and refuses unless the operator's --eval-tools names
445
+ # every one of them — an eval profile that leaves a side-effect tool unswapped is
446
+ # a lie, and a bare `--eval-profile` on a write-capable agent is exactly the
447
+ # trust-me flag the docs promise is not what happens.
448
+ def build_sim_safety(opts, golden, target_agent, a2a_target)
449
+ return Insika::Evals::Simulator::Safety.staging if opts[:staging]
450
+
451
+ declared = opts[:eval_tools].to_s.split(",").map(&:strip).reject(&:empty?)
452
+ derived = if a2a_target
453
+ nil # a remote A2A agent has no reachable registry — the list must be declared
454
+ else
455
+ derive_side_effect_tools(opts, target_agent)
456
+ end
457
+
458
+ if derived
459
+ uncovered = derived - declared
460
+ unless uncovered.empty?
461
+ abort "insika evals:simulate: --eval-profile, but the target's derived side-effect " \
462
+ "tool(s) (#{uncovered.join(', ')}) are not declared in --eval-tools — an eval " \
463
+ "profile must swap EVERY side-effect tool (or use --staging)"
464
+ end
465
+ elsif declared.empty?
466
+ abort "insika evals:simulate: cannot derive the target's side-effect tools (no " \
467
+ "/v1/agents/:id answer and no local store) — declare the swap list explicitly " \
468
+ "with --eval-tools, or use --staging"
469
+ end
470
+
471
+ Insika::Evals::Simulator::Safety.new(eval_profile: true, side_effect_tools: derived || [],
472
+ swapped_tools: declared)
473
+ end
474
+
475
+ # The target's side-effect tools — DERIVED, never hand-typed into the safety
476
+ # gate. -> [names] | nil when derivation is impossible. Local store first (the
477
+ # definitive answer when INSIKA_DB points at the deployment); otherwise the
478
+ # deployment's own registry over GET /v1/agents/:id (`side_effect_tools` — the
479
+ # server computes it from the same allowlist − deny resolution the capability
480
+ # view uses).
481
+ def derive_side_effect_tools(opts, agent)
482
+ local = local_side_effect_tools(agent)
483
+ return local unless local.nil?
484
+
485
+ capabilities = Insika::Evals::HttpCapabilities.new(base_url: opts[:base_url], token: opts[:token])
486
+ answer = capabilities.for(agent)
487
+ answer && !answer["side_effect_tools"].nil? ? Array(answer["side_effect_tools"]).map(&:to_s) : nil
488
+ end
489
+
490
+ # Derivation from the local store (INSIKA_DB set): the profile's reachable set
491
+ # intersected with the store-backed overlay registry's side-effect marks. nil
492
+ # when there is no store or the agent is not in it.
493
+ def local_side_effect_tools(agent)
494
+ db = Insika::EnvSchema.read("INSIKA_DB", ENV)
495
+ return nil if db.nil? || db.empty?
496
+
497
+ backend = Insika::Wiring::Graph.backend_from_env
498
+ config_store = Insika::ConfigStore.new(store: backend)
499
+ profile = Insika::StoredProfileSource.new(config_store: config_store).fetch(agent)
500
+ return nil if profile.nil?
501
+
502
+ registry = Insika::OverlayToolRegistry.new(
503
+ base: Insika::ToolRegistry.new, tool_store: Insika::ToolStore.new(config_store: config_store),
504
+ http: Insika::HttpClient.new
505
+ )
506
+ Insika::Evals::EvalProfile.side_effect_tools(profile, registry)
507
+ end
508
+
509
+ # One place for the CLI's provider keys. Every synchronous CLI command that
510
+ # builds its own extractor/miner/judge ask (no full app boot, so LlmConfigurator
511
+ # never ran) calls this first — a judge panel is only as multi-model as the keys
512
+ # this block wires: a settings panel pointing at OpenRouter models silently
513
+ # 401'd here while the engine (LlmConfigurator) knew the key all along.
514
+ def configure_cli_llm!
515
+ require "ruby_llm"
516
+ RubyLLM.configure do |c|
517
+ c.deepseek_api_key = ENV["DEEPSEEK_API_KEY"] if ENV["DEEPSEEK_API_KEY"]
518
+ c.openai_api_key = ENV["OPENAI_API_KEY"] if ENV["OPENAI_API_KEY"]
519
+ c.openai_api_base = ENV["OPENAI_API_BASE"] if ENV["OPENAI_API_BASE"]
520
+ c.openrouter_api_key = ENV["OPENROUTER_API_KEY"] if ENV["OPENROUTER_API_KEY"]
521
+ c.gemini_api_key = ENV["GEMINI_API_KEY"] if ENV["GEMINI_API_KEY"]
522
+ end
523
+ end
524
+
525
+ # The persona MODEL — the cheap side that plays the customer. Defaults to the
526
+ # platform `utility_model` (the same setting the judges and the moderator read);
527
+ # --persona-model overrides. Refuses when neither is set: never guess a model to
528
+ # spend money on (the proposer's rule, applied to the cheap side too).
529
+ def build_persona_ask(opts, settings)
530
+ model = opts[:persona_model] || settings["utility_model"]
531
+ abort "insika evals:simulate: no persona model — set the platform utility_model or pass --persona-model" if model.to_s.empty?
532
+
533
+ configure_cli_llm!
534
+ # ruby_llm_ask runs with assume_model_exists, which requires an explicit
535
+ # provider — a bare model id (e.g. a platform utility_model outside the
536
+ # RubyLLM registry) aborted every simulated run before this fallback.
537
+ provider = ENV["EVAL_PERSONA_PROVIDER"] || settings["default_provider"]
538
+ Insika::Evals::JudgePanel.ruby_llm_ask(model, provider)
539
+ end
540
+
541
+ def build_sim_judge(opts, settings)
542
+ models = if opts[:judge_model]
543
+ [{ "model" => opts[:judge_model], "provider" => opts[:judge_provider] }]
544
+ else
545
+ Array(settings["evals"] && settings["evals"]["judges"])
546
+ end
547
+ return nil if models.reject { |m| (m["model"] || m[:model]).to_s.empty? }.empty?
548
+
549
+ configure_cli_llm!
550
+ Insika::Evals::JudgePanel.build(settings["evals"] || {}, overrides: {
551
+ "judge_model" => opts[:judge_model], "judge_provider" => opts[:judge_provider]
552
+ })&.first
553
+ end
554
+
555
+ # The pairwise judge — the SAME configured panel, asked which conversation served
556
+ # the customer better. Same opt-in as a replay's --pairwise (2 calls per judge).
557
+ def build_sim_pairwise(opts, settings)
558
+ models = if opts[:judge_model]
559
+ [{ "model" => opts[:judge_model], "provider" => opts[:judge_provider] }]
560
+ else
561
+ Array(settings["evals"] && settings["evals"]["judges"])
562
+ end
563
+ return nil if models.reject { |m| (m["model"] || m[:model]).to_s.empty? }.empty?
564
+
565
+ configure_cli_llm!
566
+ Insika::Evals::JudgePanel.pairwise(settings["evals"] || {}, overrides: {
567
+ "judge_model" => opts[:judge_model], "judge_provider" => opts[:judge_provider]
568
+ })&.first
569
+ end
570
+
571
+ # --target <agent|url>: an agent id rides HttpTransport over the deployment's
572
+ # /v1/responses (the same surface the replay uses); a full http(s) URL is an A2A
573
+ # endpoint (an agent that only speaks A2A) driven by the thin A2A transport. The
574
+ # A2A client is built with a SYNCHRONOUS Net::HTTP adapter (the CLI has no
575
+ # reactor) — `post_json(url, body) -> Hash` is the client's whole contract.
576
+ #
577
+ # POLLING: the A2A client's default cadence (0.02s, 30 polls = ~0.6s total) is
578
+ # for in-process loopback. A remote agent takes SECONDS to finish a task, so the
579
+ # CLI polls once per second, bounded by --timeout (the same flag the HTTP
580
+ # transport's read timeout honors).
581
+ def build_sim_transport(opts)
582
+ target = opts[:target]
583
+ if target.to_s.match?(%r{\Ahttps?://})
584
+ require_relative "../lib/insika/server/a2a/client"
585
+ require_relative "../lib/insika/server/a2a/message"
586
+ require "net/http"
587
+ require "uri"
588
+ cadence = 1.0
589
+ client = Insika::Server::A2A::Client.new(
590
+ http: SyncPostJson.new,
591
+ poll_max: [opts[:timeout].to_i, 1].max,
592
+ sleeper: ->(_delay) { sleep(cadence) }
593
+ )
594
+ Insika::Evals::A2ATransport.new(client: client, url: target)
595
+ else
596
+ Insika::Evals::HttpTransport.new(base_url: opts[:base_url], token: opts[:token], timeout: opts[:timeout])
597
+ end
598
+ end
599
+
600
+ # A synchronous A2A JSON-RPC HTTP adapter for the CLI (no async reactor). The A2A
601
+ # client calls `post_json(url, body) -> Hash`; this answers it over Net::HTTP. An
602
+ # unreachable host or a non-JSON body becomes a clean `{ "error" => ... }` envelope —
603
+ # the A2A client turns that into a turn error, never a crash.
604
+ class SyncPostJson
605
+ def post_json(url, body)
606
+ uri = URI.parse(url)
607
+ req = Net::HTTP::Post.new(uri)
608
+ req["Content-Type"] = "application/json"
609
+ req["Accept"] = "application/json"
610
+ req.body = body.is_a?(String) ? body : JSON.generate(body)
611
+ res = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https",
612
+ open_timeout: 10, read_timeout: 120) { |http| http.request(req) }
613
+ JSON.parse(res.body.to_s)
614
+ rescue StandardError => e
615
+ { "error" => { "code" => -32_600, "message" => "A2A call failed: #{e.class}: #{e.message}" } }
616
+ end
617
+ end
618
+
619
+ def eval_settings_from_env
620
+ db = Insika::EnvSchema.read("INSIKA_DB", ENV)
621
+ return {} if db.nil? || db.empty?
622
+
623
+ Insika::SettingsStore.new(config_store: Insika::ConfigStore.new(store: Insika::Stores::SQLite.new(path: db))).get
624
+ rescue StandardError
625
+ {}
626
+ end
627
+
628
+ def simulate_markdown(report, verdict, pairwise = nil)
629
+ lines = ["# Simulated run — #{report['id']} (#{report['agent']})", "",
630
+ "**simulated: true** · stop: #{report['stop']}#{" (turn #{report['turns']})" if report['turns']}" \
631
+ "#{" · error: #{report['error']}" if report['error']}", ""]
632
+ report["transcript"].each do |m|
633
+ role = m[:role].to_s == "user" ? "customer" : "assistant"
634
+ lines << "**#{role}**"
635
+ lines << m[:text].to_s.empty? ? " _#{m[:error] || 'no reply'}_" : " #{m[:text]}"
636
+ lines << " _tools: #{m[:tools].join(', ')}_" if m[:tools] && !m[:tools].empty?
637
+ end
638
+ lines << ""
639
+ if verdict
640
+ lines << "**judge: #{verdict.pass ? '✅' : '❌'} #{verdict.score}** — #{verdict.reason}"
641
+ else
642
+ lines << "_(no judge configured — transcript only)_"
643
+ end
644
+ if pairwise
645
+ mark = { "better" => "🟢", "comparable" => "🟡", "worse" => "🔴" }.fetch(pairwise.outcome, "·")
646
+ lines << "**vs incumbent (#{pairwise.vs}): #{mark} #{pairwise.outcome}** — #{pairwise.reason}"
647
+ end
648
+ "#{lines.join("\n")}\n"
649
+ end
650
+
651
+ # insika mcp list|add|remove|import|test|refresh
652
+ def mcp(argv)
653
+ sub = argv.shift
654
+ case sub
655
+ when "list" then mcp_list
656
+ when "add" then mcp_add(argv)
657
+ when "remove" then mcp_remove(argv)
658
+ when "import" then mcp_import(argv)
659
+ when "test" then mcp_run(argv, verb: "test", persist_label: nil)
660
+ when "refresh" then mcp_run(argv, verb: "refresh", persist_label: "cached")
661
+ else
662
+ warn "insika mcp: unknown subcommand '#{sub}' (list|add|remove|import|test|refresh)"
663
+ exit 2
664
+ end
665
+ rescue Insika::Error => e
666
+ warn "insika mcp: #{e.message}"
667
+ exit 2
668
+ end
669
+
670
+ def mcp_list
671
+ store = mcp_store
672
+ names = store.names
673
+ return puts("no MCP instances configured (insika mcp add / insika mcp import FILE.json)") if names.empty?
674
+
675
+ names.each do |name|
676
+ r = store.get(name)
677
+ status = r["enabled"] ? "enabled" : "disabled"
678
+ target = r["transport"] == "stdio" ? r["command"].to_s : r["url"].to_s
679
+ puts format("%-20s %-8s %-8s %-40s %d tool(s)", name, r["transport"], status, target, Array(r["tools_cache"]).size)
680
+ end
681
+ end
682
+
683
+ def mcp_add(argv)
684
+ opts = { args: [], headers: {}, env: {}, enabled: true }
685
+ OptionParser.new do |o|
686
+ o.banner = "Usage: insika mcp add --name NAME [--transport stdio|http|sse] [options]"
687
+ o.on("--name NAME", "instance name (required)") { |v| opts[:name] = v }
688
+ o.on("--transport T", "stdio|http|sse (default: stdio with --command, else http)") { |v| opts[:transport] = v }
689
+ o.on("--command CMD", "stdio: the executable") { |v| opts[:command] = v }
690
+ o.on("--arg ARG", "stdio: one argv entry (repeatable)") { |v| opts[:args] << v }
691
+ o.on("--url URL", "http/sse: the server URL") { |v| opts[:url] = v }
692
+ o.on("--header 'Name: value'", "http/sse: one header (repeatable)") do |v|
693
+ k, val = v.split(":", 2)
694
+ opts[:headers][k.to_s.strip] = val.to_s.strip
695
+ end
696
+ o.on("--env 'KEY=value'", "stdio: one env var (repeatable)") do |v|
697
+ k, val = v.split("=", 2)
698
+ opts[:env][k.to_s.strip] = val.to_s.strip
699
+ end
700
+ o.on("--description DESC", "free text") { |v| opts[:description] = v }
701
+ o.on("--disabled", "create it disabled") { opts[:enabled] = false }
702
+ end.parse!(argv)
703
+ raise Insika::ValidationError, "--name is required" if opts[:name].nil?
704
+
705
+ opts[:transport] ||= Insika::Coercion.presence(opts[:command]) ? "stdio" : "http"
706
+ record = mcp_store.upsert(opts)
707
+ puts "MCP instance '#{record['name']}' saved (#{record['transport']}, #{record['enabled'] ? 'enabled' : 'disabled'})"
708
+ end
709
+
710
+ def mcp_remove(argv)
711
+ name = argv.shift
712
+ raise Insika::ValidationError, "usage: insika mcp remove NAME" if name.nil?
713
+
714
+ existed = mcp_store.delete(name)
715
+ puts existed ? "MCP instance '#{name}' removed" : "MCP instance '#{name}' did not exist"
716
+ end
717
+
718
+ def mcp_import(argv)
719
+ path = argv.shift
720
+ raise Insika::ValidationError, "usage: insika mcp import FILE.json" if path.nil?
721
+ raise Insika::ValidationError, "no such file: #{path}" unless File.exist?(path)
722
+
723
+ records = Insika::McpJson.import(File.read(path), mcp_store: mcp_store)
724
+ puts "imported #{records.size} MCP instance(s) from #{path}"
725
+ records.each { |r| puts " #{r['name']} (#{r['transport']})" }
726
+ end
727
+
728
+ # Shared by `test`/`refresh` — both connect live and list tools
729
+ # (Insika::McpToolRegistry#refresh always writes tools_cache; `test` differs
730
+ # only in what it prints). Exits 1 (not 2) on a
731
+ # connection/transport failure — the instance is configured correctly, the
732
+ # SERVER is unreachable, which is a finding, not a usage error. Rescues
733
+ # StandardError, not just Insika::Error: the registry's docstring is
734
+ # explicit that a transport failure is the GEM's error class (e.g.
735
+ # ruby_llm-mcp's own AuthenticationRequiredError) and it is this caller's
736
+ # job to turn it into a clean one-liner instead of a raw backtrace.
737
+ def mcp_run(argv, verb:, persist_label:)
738
+ name = argv.shift
739
+ raise Insika::ValidationError, "usage: insika mcp #{verb} NAME" if name.nil?
740
+
741
+ tools = mcp_registry.refresh(name)
742
+ suffix = persist_label ? ", #{persist_label}" : ""
743
+ puts "MCP instance '#{name}': connected, #{tools.size} tool(s)#{suffix}"
744
+ tools.each { |t| puts " #{t['name']} — #{t['description']}" }
745
+ rescue StandardError => e
746
+ warn "MCP instance '#{name}': #{e.message}"
747
+ exit 1
748
+ end
749
+
750
+ # insika new --list | insika new <template> [dir]
751
+ def new_from_template(argv)
752
+ if argv.first == "--list"
753
+ new_list
754
+ else
755
+ new_copy(argv)
756
+ end
757
+ rescue Insika::Error => e
758
+ warn "insika new: #{e.message}"
759
+ exit 2
760
+ end
761
+
762
+ def new_list
763
+ entries = Insika::Templates.all
764
+ return puts("no templates available") if entries.empty?
765
+
766
+ entries.each { |t| puts format("%-18s %-10s %s", t.name, t.trail, t.description) }
767
+ end
768
+
769
+ # Copies the template's directory verbatim (agent.rb + README.md) — no
770
+ # rewriting, the generated script IS the editing surface. Prints the
771
+ # exact run line, including any env the template needs SET (not just
772
+ # available as an override, like MCP_URL's keyless default) — the
773
+ # EgressGuard/stdio-gate failure is silent otherwise, same discipline as
774
+ # the `mcp add`/data-tool egress hints elsewhere in this CLI.
775
+ def new_copy(argv)
776
+ name = argv.shift
777
+ raise Insika::ValidationError, "usage: insika new <template> [dir] (see insika new --list)" if name.nil?
778
+
779
+ entry = Insika::Templates.read(name)
780
+ dest = argv.shift || "./#{name}"
781
+ raise Insika::ValidationError, "already exists: #{dest}" if File.exist?(dest)
782
+
783
+ FileUtils.mkdir_p(dest)
784
+ FileUtils.cp(Insika::Templates.agent_path(name), File.join(dest, "agent.rb"))
785
+ readme = Insika::Templates.readme_path(name)
786
+ FileUtils.cp(readme, File.join(dest, "README.md")) if File.file?(readme)
787
+
788
+ puts "Created #{dest}/ (#{entry.title})"
789
+ puts
790
+ puts "Run it:"
791
+ env_flags = entry.env.map { |e| "#{e}=1" }.join(" ")
792
+ env_flags = "#{env_flags} " unless env_flags.empty?
793
+ puts " #{env_flags}DEEPSEEK_API_KEY=sk-... ruby #{dest}/agent.rb"
794
+ puts "\nRequires: #{entry.requires}" if entry.requires
795
+ end
796
+
797
+ # insika demo:seed [--force]
798
+ #
799
+ # Populates the "demo-store" agent with enough data to see every Studio
800
+ # loop working at once: a funnel with a frozen baseline, follow-ups in
801
+ # all four states, refinement runs across the lifecycle, pending +
802
+ # resolved approvals, distillation proposals/facts, and a golden set
803
+ # with a baseline (Insika::Demo::Seeder — the same object the Studio's
804
+ # "Seed demo data" button dispatches). Safe against a live volume: it
805
+ # only ever touches the demo agent's own records, and is a no-op on a
806
+ # second run unless --force asks for another batch.
807
+ def demo_seed(argv)
808
+ opts = { force: false }
809
+ OptionParser.new do |o|
810
+ o.banner = "Usage: insika demo:seed [--force]"
811
+ o.on("--force", "seed again even if the demo agent already exists") { opts[:force] = true }
812
+ end.parse!(argv)
813
+
814
+ backend = Insika::Wiring::Graph.backend_from_env
815
+ config_store = Insika::ConfigStore.new(store: backend)
816
+ profiles = Insika::StoredProfileSource.new(config_store: config_store)
817
+ seeder = Insika::Demo::Seeder.new(
818
+ profiles: profiles, store: backend,
819
+ session_store: Insika::SessionStore.new(store: backend),
820
+ task_store: Insika::TaskStore.new(store: backend),
821
+ outcome_store: Insika::OutcomeStore.new(store: backend),
822
+ funnel_store: Insika::FunnelStore.new(store: backend),
823
+ followup_store: Insika::FollowupStore.new(store: backend),
824
+ refinement_store: Insika::RefinementStore.new(store: backend),
825
+ pending_action_store: Insika::PendingActionStore.new(store: backend),
826
+ proposal_store: Insika::ProposalStore.new(store: backend),
827
+ memory_store: Insika::MemoryStore.new(store: backend),
828
+ golden_store: Insika::GoldenStore.new(config_store: config_store),
829
+ baseline_store: Insika::BaselineStore.new(config_store: config_store),
830
+ event_stream: Insika::EventStream.new
831
+ )
832
+ result = seeder.seed!(force: opts[:force])
833
+
834
+ unless result[:seeded]
835
+ warn "insika demo:seed: already seeded (#{result[:reason]}) — pass --force to seed again."
836
+ return
837
+ end
838
+
839
+ puts "seeded demo agent '#{result[:agent]}':"
840
+ result[:counts].each { |area, n| puts " #{area}: #{n}" }
841
+ puts "\nOpen /studio/funnel?agent=#{result[:agent]} (also /studio/followups, " \
842
+ "/studio/refinement, /studio/approvals, /studio/harvest, /studio/evals) in " \
843
+ "whichever Studio instance points at this INSIKA_DB."
844
+ rescue Insika::Error => e
845
+ warn "insika demo:seed: #{e.message}"
846
+ exit 2
847
+ end
848
+
261
849
  # -- helpers -------------------------------------------------------
262
850
 
851
+ def mcp_store
852
+ backend = Insika::Wiring::Graph.backend_from_env
853
+ Insika::McpStore.new(config_store: Insika::ConfigStore.new(store: backend))
854
+ end
855
+
856
+ def mcp_registry
857
+ Insika::McpToolRegistry.new(mcp_store: mcp_store)
858
+ end
859
+
263
860
  def baseline_store
264
861
  backend = Insika::Wiring::Graph.backend_from_env
265
862
  Insika::BaselineStore.new(config_store: Insika::ConfigStore.new(store: backend))
@@ -296,6 +893,250 @@ end
296
893
  }.compact, transport: :cli))
297
894
  end
298
895
 
896
+ # insika harvest --agent ID [--last-sessions N] [--since ISO8601] [--full] [--json]
897
+ #
898
+ # mines one window of the agent's finished traffic and records
899
+ # the candidates. Writes nothing but the run + candidates + markers (D2's
900
+ # fork discipline); a cron or the operator's own habit runs it.
901
+ def harvest(argv)
902
+ opts = { json: false, full: false }
903
+ OptionParser.new do |o|
904
+ o.banner = "Usage: insika harvest --agent ID [options]"
905
+ o.on("--agent ID", "agent whose traffic to mine (required)") { |v| opts[:agent] = v }
906
+ o.on("--last-sessions N", Integer, "mine the N most recent conversations") { |v| opts[:last_sessions] = v }
907
+ o.on("--since ISO8601", "mine only turns from this instant on") { |v| opts[:since] = v }
908
+ o.on("--full", "ignore the mined markers and re-read the window") { opts[:full] = true }
909
+ o.on("--json", "emit the summary as JSON") { opts[:json] = true }
910
+ end.parse!(argv)
911
+
912
+ abort "insika harvest: --agent is required" if Insika::Coercion.blank?(opts[:agent])
913
+
914
+ result = begin
915
+ run_harvest(opts)
916
+ rescue Insika::Error, ArgumentError => e
917
+ warn "insika harvest: #{e.message}"
918
+ exit 2
919
+ end
920
+ if opts[:json]
921
+ require "json"
922
+ puts JSON.pretty_generate(result)
923
+ else
924
+ puts harvest_report(result)
925
+ end
926
+ exit(1) if result[:mined] && result[:run_id] && run_failed?(result)
927
+ end
928
+
929
+ # insika harvest:negative import --agent ID --file F
930
+ #
931
+ # the seed path: reads the VERSIONED rules file (parse! — loud), and
932
+ # writes the rules into the profile's harvest.negative_list via UpdateAgent
933
+ # (the operative, hot-editable list the engine applies).
934
+ def harvest_negative_import(argv)
935
+ opts = {}
936
+ OptionParser.new do |o|
937
+ o.banner = "Usage: insika harvest:negative import --agent ID --file F"
938
+ o.on("--agent ID", "agent whose profile receives the rules (required)") { |v| opts[:agent] = v }
939
+ o.on("--file F", "the negative-list rules file (required)") { |v| opts[:file] = v }
940
+ end.parse!(argv)
941
+
942
+ abort "insika harvest:negative import: --agent is required" if Insika::Coercion.blank?(opts[:agent])
943
+ abort "insika harvest:negative import: --file is required" if Insika::Coercion.blank?(opts[:file])
944
+
945
+ rules = begin
946
+ list = Insika::Harvest::NegativeList.parse!(File.read(opts[:file]))
947
+ list.rules.map { |r| { "rule" => r.rule, "pattern" => r.pattern, "note" => r.note } }
948
+ rescue Insika::Error, Errno::ENOENT => e
949
+ warn "insika harvest:negative import: #{e.message}"
950
+ exit 2
951
+ end
952
+
953
+ backend = Insika::Wiring::Graph.backend_from_env
954
+ config_store = Insika::ConfigStore.new(store: backend)
955
+ profiles = Insika::StoredProfileSource.new(config_store: config_store)
956
+ profile = profiles.fetch(opts[:agent]) ||
957
+ (abort "insika harvest:negative import: agent '#{opts[:agent]}' not configured")
958
+ handler = Insika::Commands::UpdateAgent.new(profile_source: profiles,
959
+ event_stream: Insika::EventStream.new)
960
+ merged = (profile.harvest || {}).merge("negative_list" => rules)
961
+ handler.call(Insika::Command.build(:update_agent,
962
+ { id: opts[:agent], harvest: merged }, transport: :cli))
963
+ puts "imported #{rules.size} rule(s) into #{opts[:agent]}.harvest.negative_list"
964
+ end
965
+
966
+ # insika harvest:criterion check --file F
967
+ #
968
+ # strict-loads the frozen conversion criterion — a missing key, an unknown
969
+ # key or an absent file exits non-zero. The hook before any promotion is
970
+ # allowed.
971
+ def harvest_criterion_check(argv)
972
+ opts = {}
973
+ OptionParser.new do |o|
974
+ o.banner = "Usage: insika harvest:criterion check --file F"
975
+ o.on("--file F", "the frozen criterion (required)") { |v| opts[:file] = v }
976
+ end.parse!(argv)
977
+
978
+ abort "insika harvest:criterion check: --file is required" if Insika::Coercion.blank?(opts[:file])
979
+
980
+ criterion = Insika::Harvest::Criterion.load(opts[:file])
981
+ puts "criterion ok: #{criterion.rule.metric} / #{criterion.rule.window} " \
982
+ "threshold #{criterion.rule.threshold} min_span #{criterion.rule.min_span} " \
983
+ "(#{criterion.sha})"
984
+ 0
985
+ rescue Insika::Error => e
986
+ warn "insika harvest:criterion check: #{e.message}"
987
+ exit 1
988
+ end
989
+
990
+ # insika knowledge:backfill --agent ID [--since ISO8601] [--json]
991
+ #
992
+ # the recovery path (RFC's "re-scan, not a queue"): replays the agent's
993
+ # stored sessions through the SAME extractor a live turn's terminal hook
994
+ # uses. Writes nothing but the concepts themselves.
995
+ def knowledge_backfill(argv)
996
+ opts = { json: false }
997
+ OptionParser.new do |o|
998
+ o.banner = "Usage: insika knowledge:backfill --agent ID [options]"
999
+ o.on("--agent ID", "agent whose sessions to replay (required)") { |v| opts[:agent] = v }
1000
+ o.on("--since ISO8601", "replay only sessions with turns from this instant on") { |v| opts[:since] = v }
1001
+ o.on("--json", "emit the summary as JSON") { opts[:json] = true }
1002
+ end.parse!(argv)
1003
+
1004
+ abort "insika knowledge:backfill: --agent is required" if Insika::Coercion.blank?(opts[:agent])
1005
+
1006
+ result = begin
1007
+ run_knowledge_backfill(opts)
1008
+ rescue Insika::Error, ArgumentError => e
1009
+ warn "insika knowledge:backfill: #{e.message}"
1010
+ exit 2
1011
+ end
1012
+ if opts[:json]
1013
+ require "json"
1014
+ puts JSON.pretty_generate(result)
1015
+ else
1016
+ puts knowledge_backfill_report(result)
1017
+ end
1018
+ end
1019
+
1020
+ # Builds the handler over the real durable backend — same "no app boot"
1021
+ # rule as `doctor`/`harvest`: no DEEPSEEK, no seeding, no reactor.
1022
+ def run_knowledge_backfill(opts)
1023
+ configure_cli_llm!
1024
+ backend = Insika::Wiring::Graph.backend_from_env
1025
+ config_store = Insika::ConfigStore.new(store: backend)
1026
+ profiles = Insika::StoredProfileSource.new(config_store: config_store)
1027
+ handler = Insika::Commands::BackfillKnowledge.new(
1028
+ profiles: profiles,
1029
+ knowledge_store: Insika::KnowledgeStore.new(store: backend),
1030
+ session_store: Insika::SessionStore.new(store: backend),
1031
+ task_store: Insika::TaskStore.new(store: backend),
1032
+ settings_store: Insika::SettingsStore.new(config_store: config_store),
1033
+ event_stream: Insika::EventStream.new
1034
+ )
1035
+ handler.call(Insika::Command.build(:backfill_knowledge,
1036
+ { agent: opts[:agent], since: opts[:since] }.compact, transport: :cli))
1037
+ end
1038
+
1039
+ def knowledge_backfill_report(result)
1040
+ return "knowledge:backfill: skipped (#{result[:skipped]})" unless result[:backfilled]
1041
+
1042
+ head = "knowledge:backfill: #{result[:concepts]} concept(s) learned from #{result[:sessions]} session(s)"
1043
+ head += " — #{result[:conflicts]} conflict(s), needs a human" if result[:conflicts].to_i.positive?
1044
+ dropped = result[:dropped].reject { |_k, v| v.to_i.zero? }
1045
+ dropped.empty? ? head : "#{head} — dropped: #{dropped.map { |k, v| "#{k}: #{v}" }.join(', ')}"
1046
+ end
1047
+
1048
+ # insika knowledge:export --agent ID [--out DIR] [--tenant T]
1049
+ #
1050
+ # writes one <name>.md per concept — the storage format IS the export
1051
+ # format, so this is a dump, not a converter, for okf-gem/graphify to
1052
+ # read directly. Safe to re-run: unlike evals:export's YAML.dump, there
1053
+ # is nothing lossy here to guard with --force.
1054
+ KNOWLEDGE_EXPORT_FORMATS = %w[md graphml].freeze
1055
+
1056
+ def knowledge_export(argv)
1057
+ opts = { out: ".", format: "md" }
1058
+ OptionParser.new do |o|
1059
+ o.banner = "Usage: insika knowledge:export --agent ID [options]"
1060
+ o.on("--agent ID", "agent whose concepts to export (required)") { |v| opts[:agent] = v }
1061
+ o.on("--out DIR", "destination directory (default: current directory)") { |v| opts[:out] = v }
1062
+ o.on("--tenant T", "explicit tenant scope (default: none)") { |v| opts[:tenant] = v }
1063
+ o.on("--format FORMAT", "md (default, one file per concept) | graphml (one graph file)") { |v| opts[:format] = v }
1064
+ end.parse!(argv)
1065
+
1066
+ abort "insika knowledge:export: --agent is required" if Insika::Coercion.blank?(opts[:agent])
1067
+ unless KNOWLEDGE_EXPORT_FORMATS.include?(opts[:format])
1068
+ abort "insika knowledge:export: --format must be #{KNOWLEDGE_EXPORT_FORMATS.join(' or ')} (got #{opts[:format].inspect})"
1069
+ end
1070
+
1071
+ backend = Insika::Wiring::Graph.backend_from_env
1072
+ store = Insika::KnowledgeStore.new(store: backend)
1073
+ tenant = Insika::Coercion.presence(opts[:tenant])
1074
+
1075
+ if opts[:format] == "graphml"
1076
+ FileUtils.mkdir_p(opts[:out])
1077
+ path = File.join(opts[:out], "knowledge.graphml")
1078
+ File.write(path, store.export_graphml(opts[:agent], tenant: tenant))
1079
+ puts "exported the concept graph to #{path}"
1080
+ else
1081
+ paths = store.export_dir(opts[:agent], opts[:out], tenant: tenant)
1082
+ puts "exported #{paths.size} concept(s) to #{opts[:out]}"
1083
+ end
1084
+ end
1085
+
1086
+ def run_failed?(result)
1087
+ backend = Insika::Wiring::Graph.backend_from_env
1088
+ run = Insika::HarvestStore.new(store: backend).find_run(result[:run_id])
1089
+ run && run.status == "failed"
1090
+ end
1091
+
1092
+ # Builds the handler over the real durable backend — same "no app boot" rule as
1093
+ # `doctor`: no DEEPSEEK, no seeding, no reactor.
1094
+ def run_harvest(opts)
1095
+ configure_cli_llm!
1096
+ backend = Insika::Wiring::Graph.backend_from_env
1097
+ config_store = Insika::ConfigStore.new(store: backend)
1098
+ profiles = Insika::StoredProfileSource.new(config_store: config_store)
1099
+ handler = Insika::Commands::RunHarvest.new(
1100
+ profiles: profiles,
1101
+ harvest_store: Insika::HarvestStore.new(store: backend),
1102
+ session_store: Insika::SessionStore.new(store: backend),
1103
+ task_store: Insika::TaskStore.new(store: backend),
1104
+ skill_store: Insika::SkillStore.new(config_store: config_store),
1105
+ settings_store: Insika::SettingsStore.new(config_store: config_store),
1106
+ negative_list: begin
1107
+ path = Insika::EnvSchema.read("INSIKA_HARVEST_NEGATIVE")
1108
+ path && Insika::Harvest::NegativeList.parse(File.read(path))
1109
+ rescue Errno::ENOENT
1110
+ nil
1111
+ end,
1112
+ miner_factory: nil,
1113
+ event_stream: Insika::EventStream.new
1114
+ )
1115
+ handler.call(Insika::Command.build(:run_harvest, {
1116
+ agent: opts[:agent], since: opts[:since],
1117
+ last_sessions: opts[:last_sessions], full: opts[:full]
1118
+ }.compact, transport: :cli))
1119
+ end
1120
+
1121
+ def harvest_report(result)
1122
+ head = if result[:mined]
1123
+ "harvest: #{result[:candidates]} candidate(s) — #{harvest_rejected(result[:rejected])}"
1124
+ else
1125
+ "harvest: skipped (#{result[:skipped]})"
1126
+ end
1127
+ if result[:cost] && result[:cost]["spent"].to_i.positive?
1128
+ head += " — #{result[:cost]['spent']} tokens" \
1129
+ "#{result[:cost]['cached'].to_i.positive? ? " (#{result[:cost]['cached']} cached)" : ''}"
1130
+ end
1131
+ head
1132
+ end
1133
+
1134
+ def harvest_rejected(rejected)
1135
+ return "nothing rejected" if rejected.nil? || rejected.values.all?(&:zero?)
1136
+
1137
+ rejected.reject { |_k, v| v.to_i.zero? }.map { |k, v| "#{k}: #{v}" }.join(", ")
1138
+ end
1139
+
299
1140
  INDENT = " " * 23 # aligns the provenance line under the finding's title
300
1141
 
301
1142
  def refine_report(run)
@@ -337,10 +1178,56 @@ end
337
1178
  # a skill and the agents holding it, so neither half alone is enough. The
338
1179
  # catalog brings the DISK seeds into the sweep (deploy/skills in a deployment,
339
1180
  # skills/ in the minimal wiring — whichever exists here); without it a stale
340
- # `eager:` or a Natura-in-shared-body sitting in a seed pack is invisible.
1181
+ # `eager:` or a store-specific phrase sitting in a shared seed pack is invisible.
341
1182
  skill_store: Insika::SkillStore.new(config_store: config_store),
342
1183
  skill_catalog: Insika::SkillCatalog.new(skill_roots),
343
1184
  profile_source: Insika::StoredProfileSource.new(config_store: config_store),
1185
+ # the shadow-parity check reads the pair store over the same
1186
+ # runtime backend (pairs hold raw customer text — the check reports their
1187
+ # age and the criterion's loadability before boot does).
1188
+ shadow_pair_store: Insika::ShadowPairStore.new(store: backend),
1189
+ # the cache-layers check — the BUILTIN set as classes (the
1190
+ # check reads the layer declaration off the class, no instances needed).
1191
+ context_providers: [
1192
+ Insika::Context::Providers::Request, Insika::Context::Providers::Prompt,
1193
+ Insika::Context::Providers::Skill, Insika::Context::Providers::SkillTrigger,
1194
+ Insika::Context::Providers::ToolSearch, Insika::Context::Providers::Memory,
1195
+ Insika::Context::Providers::Session
1196
+ ],
1197
+ # the memory-scopes check reads the cells over the same
1198
+ # runtime backend; `agent_ids` excuses the agent-memory tab's cells
1199
+ # (a bare cell named like an agent is a profile, not a customer —
1200
+ # the profile source is right here, so the CLI names them).
1201
+ memory_store: Insika::MemoryStore.new(store: backend),
1202
+ agent_ids: Insika::StoredProfileSource.new(config_store: config_store).all.map(&:id),
1203
+ # the outcome-funnel check reads the fold's cells and the
1204
+ # outcomes over the same runtime backend — a profile with outcomes but
1205
+ # no funnel warns, and a frozen baseline is confirmed before it is
1206
+ # relied on.
1207
+ funnel_store: Insika::FunnelStore.new(store: backend),
1208
+ outcome_store: Insika::OutcomeStore.new(store: backend),
1209
+ # the follow-up check reads the schedule records and the
1210
+ # contact cells over the same runtime backend (nil collaborators = the
1211
+ # CLI reports declarations only).
1212
+ followup_store: Insika::FollowupStore.new(store: backend),
1213
+ contact_store: Insika::ContactStore.new(store: backend),
1214
+ # the schedules check reads the rows over the same runtime
1215
+ # backend (nil = the CLI reports declarations only).
1216
+ schedule_store: Insika::ScheduleStore.new(store: backend),
1217
+ # the distillation check reads the proposal store over
1218
+ # the same runtime backend (nil = the CLI reports declarations only).
1219
+ proposal_store: Insika::ProposalStore.new(store: backend),
1220
+ # the harvest check reads the candidates over the same
1221
+ # runtime backend + the frozen criterion (nil = declarations only).
1222
+ harvest_store: Insika::HarvestStore.new(store: backend),
1223
+ harvest_criterion: begin
1224
+ path = Insika::EnvSchema.read("INSIKA_HARVEST_CRITERION")
1225
+ path && Insika::Harvest::Criterion.load(path)
1226
+ rescue Insika::ConfigError, Insika::ValidationError
1227
+ nil
1228
+ end,
1229
+ # the mcp check reads the instances over the same runtime backend.
1230
+ mcp_store: Insika::McpStore.new(config_store: config_store),
344
1231
  backend: backend
345
1232
  )
346
1233
  end