insika 0.2.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (300) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +325 -17
  3. data/README.md +52 -11
  4. data/bin/insika +895 -8
  5. data/bin/insika-router +87 -0
  6. data/docs/AGENTS.md +229 -163
  7. data/docs/API.md +56 -0
  8. data/docs/ARCHITECTURE.md +3 -2
  9. data/docs/ARTIFACTS.md +95 -0
  10. data/docs/BENCHMARK.md +2 -2
  11. data/docs/CHANNELS.md +111 -21
  12. data/docs/CONTEXT.md +132 -19
  13. data/docs/DEMO.md +80 -0
  14. data/docs/DEPLOY.md +75 -4
  15. data/docs/EMBEDDING.md +1 -1
  16. data/docs/EVALS.md +146 -2
  17. data/docs/FACTS.md +135 -0
  18. data/docs/HARVEST.md +116 -0
  19. data/docs/KNOWLEDGE.md +290 -0
  20. data/docs/LOADTEST.md +16 -10
  21. data/docs/MEDIA.md +128 -0
  22. data/docs/OBSERVABILITY.md +65 -6
  23. data/docs/OUTCOMES.md +137 -0
  24. data/docs/PLUGINS.md +51 -6
  25. data/docs/POLICY.md +216 -0
  26. data/docs/REFINEMENT.md +17 -12
  27. data/docs/RELEASING.md +27 -0
  28. data/docs/ROUTER.md +213 -0
  29. data/docs/RUNNING-LOCAL.md +6 -6
  30. data/docs/SCHEDULING.md +121 -0
  31. data/docs/SECURITY.md +79 -2
  32. data/docs/SKILLS.md +13 -4
  33. data/docs/SOAK.md +127 -0
  34. data/docs/TEMPLATES.md +134 -0
  35. data/docs/TOOLS.md +220 -27
  36. data/docs/WHY.md +1 -1
  37. data/docs/WORKFLOWS.md +2 -2
  38. data/docs/_includes/head_custom.html +5 -0
  39. data/docs/_includes/title.html +13 -0
  40. data/docs/_sass/color_schemes/insika.scss +32 -0
  41. data/docs/_sass/custom/custom.scss +199 -0
  42. data/docs/_sass/custom/setup.scss +26 -0
  43. data/docs/assets/img/favicon.svg +7 -0
  44. data/docs/assets/img/insika-mark.svg +7 -0
  45. data/docs/core-concepts.md +21 -0
  46. data/docs/domain.md +115 -0
  47. data/docs/improve.md +20 -0
  48. data/docs/index.md +9 -6
  49. data/docs/integrate.md +20 -0
  50. data/docs/operate.md +13 -6
  51. data/docs/prompts/ADD-TOOL.md +118 -0
  52. data/docs/prompts/DIAGNOSE-TURN.md +65 -0
  53. data/docs/prompts/GO-LIVE.md +138 -0
  54. data/docs/prompts/RUN-EXAMPLES.md +70 -0
  55. data/docs/reference.md +19 -0
  56. data/docs/ship.md +10 -2
  57. data/docs/start-here.md +18 -0
  58. data/lib/insika/agent_profile.rb +202 -10
  59. data/lib/insika/artifact_signing.rb +82 -0
  60. data/lib/insika/artifact_store.rb +160 -0
  61. data/lib/insika/balloon_splitter.rb +102 -0
  62. data/lib/insika/budget_ledger.rb +34 -3
  63. data/lib/insika/cache_series_store.rb +49 -0
  64. data/lib/insika/channel_delivery.rb +119 -14
  65. data/lib/insika/channels/relay.rb +77 -3
  66. data/lib/insika/channels/web.rb +2 -2
  67. data/lib/insika/chat_builder.rb +105 -2
  68. data/lib/insika/checkpoint_store.rb +16 -0
  69. data/lib/insika/commands/agent_payload.rb +3 -3
  70. data/lib/insika/commands/backfill_knowledge.rb +145 -0
  71. data/lib/insika/commands/cancel_followup.rb +49 -0
  72. data/lib/insika/commands/delete_artifact.rb +35 -0
  73. data/lib/insika/commands/delete_concept.rb +34 -0
  74. data/lib/insika/commands/delete_mcp.rb +6 -2
  75. data/lib/insika/commands/delete_tenant_data.rb +107 -0
  76. data/lib/insika/commands/export_customer_memory.rb +48 -0
  77. data/lib/insika/commands/forget_customer.rb +117 -0
  78. data/lib/insika/commands/freeze_funnel_baseline.rb +113 -0
  79. data/lib/insika/commands/gate_harvest.rb +138 -0
  80. data/lib/insika/commands/gate_refinement.rb +1 -1
  81. data/lib/insika/commands/judge_shadow_pairs.rb +124 -0
  82. data/lib/insika/commands/memory_forget_fact.rb +20 -4
  83. data/lib/insika/commands/memory_put_fact.rb +23 -4
  84. data/lib/insika/commands/promote_harvest.rb +130 -0
  85. data/lib/insika/commands/record_outcome.rb +46 -0
  86. data/lib/insika/commands/record_shadow_reply.rb +68 -0
  87. data/lib/insika/commands/refresh_mcp_tools.rb +47 -0
  88. data/lib/insika/commands/reject_harvest.rb +38 -0
  89. data/lib/insika/commands/resolve_proposal.rb +108 -0
  90. data/lib/insika/commands/restore_concept.rb +34 -0
  91. data/lib/insika/commands/revoke_contact.rb +49 -0
  92. data/lib/insika/commands/rollback_harvest.rb +86 -0
  93. data/lib/insika/commands/run_distillation.rb +186 -0
  94. data/lib/insika/commands/run_harvest.rb +393 -0
  95. data/lib/insika/commands/seed_demo_data.rb +31 -0
  96. data/lib/insika/commands/send_message.rb +103 -6
  97. data/lib/insika/commands/session_purge.rb +67 -0
  98. data/lib/insika/commands/upsert_mcp.rb +6 -3
  99. data/lib/insika/commands/write_concept.rb +57 -0
  100. data/lib/insika/contact_store.rb +183 -0
  101. data/lib/insika/context/builder.rb +21 -3
  102. data/lib/insika/context/fragment.rb +7 -3
  103. data/lib/insika/context/priority.rb +5 -0
  104. data/lib/insika/context/provider.rb +17 -3
  105. data/lib/insika/context/providers/briefing.rb +96 -0
  106. data/lib/insika/context/providers/knowledge.rb +108 -0
  107. data/lib/insika/context/providers/memory.rb +16 -7
  108. data/lib/insika/context/providers/prompt.rb +57 -23
  109. data/lib/insika/context/providers/skill.rb +2 -0
  110. data/lib/insika/context/providers/tool_search.rb +2 -0
  111. data/lib/insika/context_trace_store.rb +38 -2
  112. data/lib/insika/cron.rb +189 -0
  113. data/lib/insika/demo/agent_attrs.rb +43 -0
  114. data/lib/insika/demo/golden_cases.rb +81 -0
  115. data/lib/insika/demo/seeder.rb +336 -0
  116. data/lib/insika/distill.rb +224 -0
  117. data/lib/insika/distill_engine.rb +169 -0
  118. data/lib/insika/doctor.rb +889 -11
  119. data/lib/insika/dsl/definition.rb +3 -2
  120. data/lib/insika/dsl/runtime.rb +69 -82
  121. data/lib/insika/dsl/server_boot.rb +92 -1
  122. data/lib/insika/dsl/system.rb +10 -2
  123. data/lib/insika/dsl.rb +186 -5
  124. data/lib/insika/edge_limiter.rb +28 -7
  125. data/lib/insika/env_schema.rb +25 -3
  126. data/lib/insika/errors.rb +11 -0
  127. data/lib/insika/evals/assertions.rb +3 -2
  128. data/lib/insika/evals/golden.rb +41 -4
  129. data/lib/insika/evals/judge.rb +47 -2
  130. data/lib/insika/evals/pairwise.rb +29 -6
  131. data/lib/insika/evals/persona.rb +98 -0
  132. data/lib/insika/evals/runner.rb +9 -0
  133. data/lib/insika/evals/simulator.rb +225 -0
  134. data/lib/insika/evals/transport.rb +83 -1
  135. data/lib/insika/event_stream.rb +10 -0
  136. data/lib/insika/evidence.rb +183 -0
  137. data/lib/insika/executor.rb +869 -76
  138. data/lib/insika/followup_engine.rb +207 -0
  139. data/lib/insika/followup_policy.rb +198 -0
  140. data/lib/insika/followup_store.rb +306 -0
  141. data/lib/insika/funnel_declaration.rb +106 -0
  142. data/lib/insika/funnel_fold.rb +179 -0
  143. data/lib/insika/funnel_store.rb +163 -0
  144. data/lib/insika/golden_store.rb +17 -2
  145. data/lib/insika/grounding/matcher.rb +69 -0
  146. data/lib/insika/grounding.rb +44 -0
  147. data/lib/insika/harvest/conversion_gate.rb +159 -0
  148. data/lib/insika/harvest/criterion.rb +98 -0
  149. data/lib/insika/harvest/gate.rb +194 -0
  150. data/lib/insika/harvest/negative_list.rb +199 -0
  151. data/lib/insika/harvest.rb +241 -0
  152. data/lib/insika/harvest_engine.rb +193 -0
  153. data/lib/insika/harvest_store.rb +548 -0
  154. data/lib/insika/knowledge.rb +680 -0
  155. data/lib/insika/knowledge_store.rb +140 -0
  156. data/lib/insika/mcp_client.rb +94 -0
  157. data/lib/insika/mcp_json.rb +74 -0
  158. data/lib/insika/mcp_live_tool.rb +43 -0
  159. data/lib/insika/mcp_store.rb +98 -26
  160. data/lib/insika/mcp_tool_ingestor.rb +30 -8
  161. data/lib/insika/mcp_tool_registry.rb +100 -0
  162. data/lib/insika/media.rb +382 -0
  163. data/lib/insika/memory_audit_store.rb +85 -0
  164. data/lib/insika/memory_store.rb +264 -23
  165. data/lib/insika/message_origin.rb +7 -2
  166. data/lib/insika/middleware.rb +9 -0
  167. data/lib/insika/model_visible.rb +87 -0
  168. data/lib/insika/model_visible_trace_store.rb +66 -0
  169. data/lib/insika/onboarding.rb +22 -1
  170. data/lib/insika/outbox_store.rb +42 -4
  171. data/lib/insika/outcome_store.rb +147 -0
  172. data/lib/insika/overlay_tool_registry.rb +37 -17
  173. data/lib/insika/packaging.rb +163 -0
  174. data/lib/insika/parity/criterion.rb +79 -0
  175. data/lib/insika/parity/verdict.rb +318 -0
  176. data/lib/insika/prefix_fingerprint.rb +58 -0
  177. data/lib/insika/profile_source.rb +29 -1
  178. data/lib/insika/prompt_catalog.rb +10 -0
  179. data/lib/insika/proposal_store.rb +271 -0
  180. data/lib/insika/queue_policy.rb +4 -1
  181. data/lib/insika/refinement/proposer.rb +1 -1
  182. data/lib/insika/reliability.rb +32 -6
  183. data/lib/insika/retention.rb +316 -0
  184. data/lib/insika/router/app.rb +157 -0
  185. data/lib/insika/router/backend_pool.rb +98 -0
  186. data/lib/insika/router/hash_ring.rb +55 -0
  187. data/lib/insika/router/proxy_body.rb +34 -0
  188. data/lib/insika/router/session_key.rb +54 -0
  189. data/lib/insika/router.rb +18 -0
  190. data/lib/insika/routing.rb +101 -0
  191. data/lib/insika/safety/config.rb +44 -4
  192. data/lib/insika/safety/corpus.rb +255 -0
  193. data/lib/insika/safety/detectors.rb +32 -113
  194. data/lib/insika/safety/factory.rb +15 -2
  195. data/lib/insika/safety/grounding_enforcer.rb +59 -0
  196. data/lib/insika/safety/grounding_validator.rb +49 -0
  197. data/lib/insika/safety/input_guardrail.rb +1 -1
  198. data/lib/insika/safety/moderator.rb +1 -1
  199. data/lib/insika/safety/output_filter.rb +9 -5
  200. data/lib/insika/safety/output_validator.rb +11 -5
  201. data/lib/insika/schedule.rb +177 -0
  202. data/lib/insika/schedule_engine.rb +314 -0
  203. data/lib/insika/schedule_store.rb +208 -0
  204. data/lib/insika/schema_guard.rb +35 -0
  205. data/lib/insika/server/app.rb +247 -17
  206. data/lib/insika/server/rack_app.rb +21 -1
  207. data/lib/insika/server/responses.rb +40 -1
  208. data/lib/insika/session_actor.rb +8 -4
  209. data/lib/insika/session_store.rb +65 -2
  210. data/lib/insika/settings_store.rb +10 -0
  211. data/lib/insika/shadow_pair_store.rb +258 -0
  212. data/lib/insika/skill_catalog.rb +16 -0
  213. data/lib/insika/soak/envelope.rb +140 -0
  214. data/lib/insika/soak/report.rb +392 -0
  215. data/lib/insika/soak/runner.rb +554 -0
  216. data/lib/insika/steer_injector.rb +21 -10
  217. data/lib/insika/store.rb +10 -1
  218. data/lib/insika/stores/memory.rb +6 -0
  219. data/lib/insika/stores/sqlite.rb +8 -0
  220. data/lib/insika/studio/app.rb +1437 -54
  221. data/lib/insika/studio/assets/dist/application.css +1 -1
  222. data/lib/insika/studio/assets/dist/application.js +23 -23
  223. data/lib/insika/studio/assets/dist/favicon.svg +6 -0
  224. data/lib/insika/studio/forms.rb +308 -10
  225. data/lib/insika/studio/nav_icons.rb +19 -1
  226. data/lib/insika/studio/views/_agent_tab_cache.erb +25 -0
  227. data/lib/insika/studio/views/_agent_tab_config.erb +514 -0
  228. data/lib/insika/studio/views/_agent_tab_history.erb +24 -0
  229. data/lib/insika/studio/views/_agent_tab_loops.erb +54 -0
  230. data/lib/insika/studio/views/_agent_tab_memory.erb +51 -0
  231. data/lib/insika/studio/views/_agent_tab_outcomes.erb +31 -0
  232. data/lib/insika/studio/views/_agent_tab_prompts.erb +108 -0
  233. data/lib/insika/studio/views/_agent_tab_skills.erb +38 -0
  234. data/lib/insika/studio/views/_agents_master.erb +44 -0
  235. data/lib/insika/studio/views/_message.erb +49 -32
  236. data/lib/insika/studio/views/agent_detail.erb +62 -278
  237. data/lib/insika/studio/views/agents.erb +71 -54
  238. data/lib/insika/studio/views/approvals.erb +4 -1
  239. data/lib/insika/studio/views/artifact.erb +23 -0
  240. data/lib/insika/studio/views/artifacts.erb +59 -0
  241. data/lib/insika/studio/views/chats.erb +4 -1
  242. data/lib/insika/studio/views/customer.erb +94 -0
  243. data/lib/insika/studio/views/customers.erb +32 -0
  244. data/lib/insika/studio/views/evals.erb +6 -3
  245. data/lib/insika/studio/views/facts.erb +133 -0
  246. data/lib/insika/studio/views/followups.erb +125 -0
  247. data/lib/insika/studio/views/funnel.erb +106 -0
  248. data/lib/insika/studio/views/harvest.erb +234 -0
  249. data/lib/insika/studio/views/home.erb +106 -66
  250. data/lib/insika/studio/views/knowledge.erb +123 -0
  251. data/lib/insika/studio/views/layout.erb +15 -11
  252. data/lib/insika/studio/views/mcp.erb +174 -80
  253. data/lib/insika/studio/views/parity.erb +147 -0
  254. data/lib/insika/studio/views/playground.erb +7 -1
  255. data/lib/insika/studio/views/session.erb +233 -124
  256. data/lib/insika/studio/views/settings.erb +41 -1
  257. data/lib/insika/studio/views/skills.erb +1 -2
  258. data/lib/insika/studio/views/system_files.erb +1 -1
  259. data/lib/insika/studio/views/task.erb +13 -0
  260. data/lib/insika/studio/views/tasks.erb +4 -1
  261. data/lib/insika/studio/views/tools.erb +24 -10
  262. data/lib/insika/task_store.rb +21 -1
  263. data/lib/insika/templates/browser-agent/README.md +36 -0
  264. data/lib/insika/templates/browser-agent/agent.rb +49 -0
  265. data/lib/insika/templates/daily-digest/README.md +38 -0
  266. data/lib/insika/templates/daily-digest/agent.rb +77 -0
  267. data/lib/insika/templates/repo-explorer/README.md +36 -0
  268. data/lib/insika/templates/repo-explorer/agent.rb +45 -0
  269. data/lib/insika/templates/research-analyst/README.md +26 -0
  270. data/lib/insika/templates/research-analyst/agent.rb +58 -0
  271. data/lib/insika/templates/review-panel/README.md +20 -0
  272. data/lib/insika/templates/review-panel/agent.rb +50 -0
  273. data/lib/insika/templates/travel-planner/README.md +35 -0
  274. data/lib/insika/templates/travel-planner/agent.rb +87 -0
  275. data/lib/insika/templates.rb +112 -0
  276. data/lib/insika/testing/store_contract.rb +27 -6
  277. data/lib/insika/tick.rb +47 -11
  278. data/lib/insika/timezone.rb +45 -0
  279. data/lib/insika/tool_definition.rb +17 -7
  280. data/lib/insika/tool_envelope.rb +69 -0
  281. data/lib/insika/tool_manifest.rb +5 -1
  282. data/lib/insika/tools/data_defined_tool.rb +10 -0
  283. data/lib/insika/tools/generate_image.rb +89 -0
  284. data/lib/insika/tools/load_knowledge.rb +74 -0
  285. data/lib/insika/tools/run_persona_eval.rb +328 -0
  286. data/lib/insika/tools/save_artifact.rb +95 -0
  287. data/lib/insika/tools/schedule_followup.rb +164 -0
  288. data/lib/insika/tools/tts.rb +47 -0
  289. data/lib/insika/tools/update_briefing.rb +126 -0
  290. data/lib/insika/turn_output.rb +1 -1
  291. data/lib/insika/turn_state.rb +49 -1
  292. data/lib/insika/turn_timing.rb +22 -2
  293. data/lib/insika/version.rb +1 -1
  294. data/lib/insika/vitals.rb +84 -0
  295. data/lib/insika/wiring/graph.rb +472 -13
  296. data/lib/insika/wiring/graph_chat.rb +102 -0
  297. data/lib/insika.rb +149 -3
  298. metadata +181 -5
  299. data/docs/build.md +0 -14
  300. data/docs/understand.md +0 -10
@@ -0,0 +1,140 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "fileutils"
4
+ require "time"
5
+
6
+ module Insika
7
+ # LEARNED concepts, one per agent (+ optional tenant). Holds the complete
8
+ # concept markdown (frontmatter + body) in the durable Store, the same
9
+ # content/updated_at/bounded-history record shape `SkillStore` uses for
10
+ # SKILL.md — so the Studio's editor and undo work identically.
11
+ #
12
+ # Scoped like `MemoryStore` (agent/tenant baked into the scope string, not a
13
+ # second constructor argument): a concept is engine-written, per-deployment
14
+ # knowledge, never shared across agents the way a skill can be.
15
+ #
16
+ # scope: "knowledge:<agent_id>" # default
17
+ # scope: "knowledge:<agent_id>:<tenant>" # explicit tenant (multi-merchant)
18
+ # key: "concept:<name>"
19
+ #
20
+ # Record shape, one per concept:
21
+ # { "content" => "<entire concept markdown>", "updated_at" => iso8601,
22
+ # "history" => [ { "content" =>, "at" => }, ... ] }
23
+ #
24
+ # Layer 1 only: this is a plain upsert (last-write-wins, old version pushed
25
+ # into history). Dedup/merge/conflict detection across sightings is layer 2,
26
+ # not this store's job.
27
+ class KnowledgeStore
28
+ SCOPE_PREFIX = "knowledge"
29
+ KEY_PREFIX = "concept:"
30
+ HISTORY_MAX = 20
31
+
32
+ def initialize(store:)
33
+ @store = store
34
+ end
35
+
36
+ # -> String | nil (the complete concept markdown).
37
+ def get(agent_id, name, tenant: nil)
38
+ record(agent_id, name, tenant)&.fetch("content", nil)
39
+ end
40
+
41
+ # -> {"content" =>, "updated_at" =>} | nil. Cheap by design (no YAML
42
+ # frontmatter parse) — `updated_at` is the raw record's OWN timestamp,
43
+ # written on every upsert, so a caller can use it as a memoization key
44
+ # (Index::Scan's read cache) without re-parsing content that has not
45
+ # changed since the last read.
46
+ def meta(agent_id, name, tenant: nil)
47
+ rec = record(agent_id, name, tenant)
48
+ rec && { "content" => rec["content"], "updated_at" => rec["updated_at"] }
49
+ end
50
+
51
+ # -> [String] concept names for the scope, lexicographic order.
52
+ def names(agent_id, tenant: nil)
53
+ @store.list(scope_for(agent_id, tenant), KEY_PREFIX).map { |k| k.delete_prefix(KEY_PREFIX) }
54
+ end
55
+
56
+ # -> { name => content } of every concept in the scope.
57
+ def all(agent_id, tenant: nil)
58
+ names(agent_id, tenant: tenant).each_with_object({}) { |n, acc| acc[n] = get(agent_id, n, tenant: tenant) }
59
+ end
60
+
61
+ # Upsert. -> Hash (the stored record).
62
+ def write(agent_id, name, content, tenant: nil)
63
+ key = name.to_s
64
+ current = record(agent_id, key, tenant)
65
+ rec = build_record(content.to_s, current)
66
+ @store.set(scope_for(agent_id, tenant), KEY_PREFIX + key, rec)
67
+ rec
68
+ end
69
+
70
+ # -> bool (did it exist?).
71
+ def delete(agent_id, name, tenant: nil)
72
+ @store.delete(scope_for(agent_id, tenant), KEY_PREFIX + name.to_s)
73
+ end
74
+
75
+ # -> [ { "content" =>, "at" => } ] most recent first.
76
+ def versions(agent_id, name, tenant: nil)
77
+ record(agent_id, name, tenant)&.fetch("history", []) || []
78
+ end
79
+
80
+ # Restores version `index` as the current content (a new write). -> Hash.
81
+ def restore(agent_id, name, index, tenant: nil)
82
+ hist = versions(agent_id, name, tenant: tenant)
83
+ i = Integer(index)
84
+ raise Insika::NotFoundError, "concept '#{name}' not found" unless record(agent_id, name.to_s, tenant)
85
+ raise Insika::ValidationError, "version #{index} does not exist" if i.negative? || i >= hist.length
86
+
87
+ write(agent_id, name, hist[i]["content"], tenant: tenant)
88
+ end
89
+
90
+ # Writes one `<dir>/<name>.md` per concept — the storage format IS the
91
+ # export format, so this is a dump, not a converter: each file is the
92
+ # concept's content, byte for byte, directly consumable by okf-gem
93
+ # (`OKF::Bundle`) or graphify. Unlike a lossy re-serialization (YAML.dump
94
+ # on a curated corpus, say), re-exporting the same store is idempotent —
95
+ # no `force` guard needed, there is nothing here to lose. -> [paths].
96
+ def export_dir(agent_id, dir, tenant: nil)
97
+ FileUtils.mkdir_p(dir)
98
+ names(agent_id, tenant: tenant).map do |name|
99
+ path = File.join(dir, "#{name}.md")
100
+ File.write(path, get(agent_id, name, tenant: tenant))
101
+ path
102
+ end
103
+ end
104
+
105
+ # -> String (one GraphML document — the whole scope as a graph, §5's
106
+ # follow-up export shape). A record that fails to parse (a hand edit
107
+ # gone wrong) is skipped rather than breaking the whole export.
108
+ def export_graphml(agent_id, tenant: nil)
109
+ concepts = names(agent_id, tenant: tenant).filter_map do |name|
110
+ Knowledge::Concept.parse(get(agent_id, name, tenant: tenant))
111
+ end
112
+ Knowledge::GraphmlExport.build(concepts)
113
+ end
114
+
115
+ private
116
+
117
+ def scope_for(agent_id, tenant)
118
+ t = Coercion.presence(tenant)
119
+ t ? "#{SCOPE_PREFIX}:#{agent_id}:#{t}" : "#{SCOPE_PREFIX}:#{agent_id}"
120
+ end
121
+
122
+ def record(agent_id, name, tenant)
123
+ @store.get(scope_for(agent_id, tenant), KEY_PREFIX + name.to_s)
124
+ end
125
+
126
+ def build_record(content, current)
127
+ history = current ? current.fetch("history", []) : []
128
+ if current
129
+ history = [{ "content" => current["content"], "at" => current["updated_at"] }] + history
130
+ history = history.first(HISTORY_MAX)
131
+ end
132
+ # Microsecond precision ON PURPOSE (MemoryStore's own rule): `updated_at`
133
+ # is Index::Scan's cache-invalidation key — second precision collides
134
+ # for two writes in the same second (a backfill, a rapid consolidation),
135
+ # and a collision there means a stale cached concept survives a real
136
+ # write until the next search that lands in a different second.
137
+ { "content" => content, "updated_at" => Time.now.utc.iso8601(6), "history" => history }
138
+ end
139
+ end
140
+ end
@@ -0,0 +1,94 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Insika
4
+ # Builds a live MCP client for an McpStore record. The ONE seam
5
+ # between Insika and the transport gem: `ruby_llm-mcp` speaks stdio,
6
+ # Streamable HTTP and SSE with the full lifecycle (initialize handshake,
7
+ # session-id, notifications) — Insika stops maintaining its own protocol
8
+ # client. Swapping the gem later (e.g. to modelcontextprotocol/ruby-sdk)
9
+ # touches only this file.
10
+ #
11
+ # `require "ruby_llm/mcp"` is LAZY, done here rather than at `require
12
+ # "insika"` time (spec/insika/load_guard_spec.rb): the gem drags in
13
+ # ruby_llm + httpx, the same opt-in-cost discipline as every other provider
14
+ # surface.
15
+ #
16
+ # Client LIFECYCLE (start/stop, registering the child process with
17
+ # Insika::Shutdown so it dies on SIGTERM) belongs to whoever holds the
18
+ # client across calls — the live tool registry, later work. `for` itself
19
+ # returns an unstarted client (`start: false`); building one is cheap and
20
+ # side-effect-free.
21
+ module McpClient
22
+ # A stdio instance is arbitrary command execution by config — refused
23
+ # unless the operator opted in (config-over-convention, same pattern as
24
+ # the egress envs).
25
+ class StdioDisabled < Insika::Error; end
26
+
27
+ module_function
28
+
29
+ # `record` — the RAW McpStore record (get_raw/all_raw; never the masked
30
+ # one — a masked record's credentials are the __OCULTO__ sentinel, not
31
+ # something a client can connect with).
32
+ # -> RubyLLM::MCP::Client, not started. Raises StdioDisabled (stdio, gate
33
+ # off), Insika::Error (http/sse, egress-blocked url) or
34
+ # Insika::ValidationError (unknown transport).
35
+ def for(record, env_reader: Insika::EnvSchema, egress: Insika::EgressGuard)
36
+ require "ruby_llm/mcp"
37
+ apply_origin_header_fix!
38
+
39
+ case record["transport"].to_s
40
+ when "stdio" then stdio_client(record, env_reader: env_reader)
41
+ when "http" then http_client(record, :streamable_http, egress: egress)
42
+ when "sse" then http_client(record, :sse, egress: egress)
43
+ else
44
+ raise Insika::ValidationError,
45
+ "MCP instance '#{record["name"]}' has an unknown transport: #{record["transport"].inspect}"
46
+ end
47
+ end
48
+
49
+ def stdio_client(record, env_reader:)
50
+ unless env_reader.truthy?(env_reader.read("INSIKA_MCP_STDIO"))
51
+ raise StdioDisabled,
52
+ "MCP instance '#{record["name"]}' is stdio (arbitrary command execution by config) — " \
53
+ "set INSIKA_MCP_STDIO=1 to allow it to start"
54
+ end
55
+
56
+ RubyLLM::MCP.client(
57
+ name: record["name"], transport_type: :stdio, start: false,
58
+ config: { command: record["command"], args: Array(record["args"]), env: record["env"] || {} }
59
+ )
60
+ end
61
+
62
+ def http_client(record, transport_type, egress:)
63
+ url = record["url"]
64
+ reason = egress.violation(url)
65
+ raise Insika::Error, "MCP target blocked: #{reason}" if reason
66
+
67
+ RubyLLM::MCP.client(
68
+ name: record["name"], transport_type: transport_type, start: false,
69
+ config: { url: url, headers: record["headers"] || {} }
70
+ )
71
+ end
72
+
73
+ # https://github.com/patvice/ruby_llm-mcp/issues/140 (open, unfixed as of
74
+ # 1.0.1): StreamableHTTP#build_common_headers sets `Origin` to the MCP
75
+ # endpoint's OWN url (path included) on every request. That's invalid per
76
+ # RFC 6454 §7 (Origin is scheme+host[+port], never a path) and unneeded —
77
+ # a server-to-server client has no browser Origin to report, and the
78
+ # reference TS/Python SDKs never send one. No allowlist can ever match a
79
+ # value that carries a path, so any server enforcing the spec's
80
+ # DNS-rebinding Origin check (GitHub's remote MCP, Grafana, Metabase, ...)
81
+ # rejects every request. Same prepend the issue's own author ships in
82
+ # production. `Module#prepend` no-ops if the module is already in the
83
+ # ancestor chain, so calling this on every `for` is cheap and safe.
84
+ def apply_origin_header_fix!
85
+ RubyLLM::MCP::Native::Transports::StreamableHTTP.prepend(OriginHeaderFix)
86
+ end
87
+
88
+ module OriginHeaderFix
89
+ def build_common_headers
90
+ super.except("Origin")
91
+ end
92
+ end
93
+ end
94
+ end
@@ -0,0 +1,74 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+
5
+ module Insika
6
+ # The de-facto `mcpServers` JSON format every MCP client (Claude Desktop,
7
+ # Cursor, ...) already uses — one parser shared by all three PR3 config
8
+ # surfaces (CLI `insika mcp import`, the API `/v1/mcp` PUT, Studio's "Import
9
+ # JSON" textarea).
10
+ #
11
+ # {
12
+ # "mcpServers": {
13
+ # "tavily": { "url": "https://mcp.tavily.com/mcp", "headers": {...} },
14
+ # "filesystem": { "command": "npx", "args": ["-y", "@modelcontextprotocol/server-filesystem", "/tmp"] }
15
+ # }
16
+ # }
17
+ #
18
+ # A server entry with `command` and no `url` is stdio; one with `url` and no
19
+ # `command` is `http` (Streamable HTTP, the modern default) unless it names
20
+ # `"transport": "sse"` explicitly — the bare format has no other way to spell
21
+ # SSE. `export` always writes `transport` back so a round-trip is lossless.
22
+ module McpJson
23
+ module_function
24
+
25
+ # `json` — a JSON string or an already-parsed Hash (either key type).
26
+ # Upserts every entry via `mcp_store` (per-key secret reconciliation, same
27
+ # as any other upsert — re-importing an export's `__OCULTO__` sentinel
28
+ # preserves whatever is already stored, never wipes it).
29
+ # -> [Hash] the masked upserted records, in the document's key order.
30
+ def import(json, mcp_store:)
31
+ data = json.is_a?(String) ? JSON.parse(json) : stringify(json)
32
+ servers = stringify(data["mcpServers"] || {})
33
+ servers.map { |name, cfg| mcp_store.upsert(attrs_from(name, stringify(cfg))) }
34
+ end
35
+
36
+ # -> { "mcpServers" => { name => {...} } }, secrets masked (never plaintext
37
+ # — `mcp_store.all` already masks, this only reshapes).
38
+ def export(mcp_store:)
39
+ { "mcpServers" => mcp_store.all.each_with_object({}) { |record, acc| acc[record["name"]] = server_from(record) } }
40
+ end
41
+
42
+ def attrs_from(name, cfg)
43
+ {
44
+ name: name.to_s,
45
+ transport: presence(cfg["transport"]) || (presence(cfg["command"]) ? "stdio" : "http"),
46
+ command: cfg["command"], args: cfg["args"], url: cfg["url"],
47
+ headers: cfg["headers"], env: cfg["env"],
48
+ description: cfg["description"], enabled: cfg.fetch("enabled", true)
49
+ }
50
+ end
51
+ private_class_method :attrs_from
52
+
53
+ def server_from(record)
54
+ body = record["transport"] == "stdio" ? { "command" => record["command"], "args" => record["args"], "env" => record["env"] }
55
+ : { "url" => record["url"], "headers" => record["headers"] }
56
+ body.merge(
57
+ "transport" => record["transport"],
58
+ "description" => record["description"],
59
+ "enabled" => record["enabled"]
60
+ ).compact
61
+ end
62
+ private_class_method :server_from
63
+
64
+ def stringify(obj)
65
+ return {} unless obj.is_a?(Hash)
66
+
67
+ obj.each_with_object({}) { |(k, v), acc| acc[k.to_s] = v }
68
+ end
69
+ private_class_method :stringify
70
+
71
+ def presence(value) = Insika::Coercion.presence(value)
72
+ private_class_method :presence
73
+ end
74
+ end
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "ruby_llm"
4
+
5
+ module Insika
6
+ # One live MCP tool. `name`/`description`/`params_schema`
7
+ # come from the CACHED descriptor (McpStore#tools_cache — no I/O to build
8
+ # this instance, same cost as any other tool the registry hands out).
9
+ # `#execute` is the only thing that touches the network, calling through
10
+ # `client_for` (Insika::McpToolRegistry's memoized, started client — this
11
+ # class never builds or starts one itself) into the gem's own
12
+ # `RubyLLM::MCP::Tool#execute`, which already unwraps `content[].text` and
13
+ # turns `isError` into `{error:}`.
14
+ #
15
+ # Like every tool in this codebase, `#execute` NEVER raises: a connection
16
+ # or protocol failure becomes `{error:}`, the model's normal failure path
17
+ # (see Tools::DataDefinedTool's own rule).
18
+ class McpLiveTool < RubyLLM::Tool
19
+ def initialize(instance_name:, tool:, client_for:)
20
+ @instance_name = instance_name
21
+ @tool = tool
22
+ @client_for = client_for
23
+ super()
24
+ end
25
+
26
+ def name = @tool["name"]
27
+ def description = @tool["description"].to_s
28
+
29
+ def params_schema
30
+ schema = @tool["inputSchema"]
31
+ schema.nil? || schema.empty? ? { "type" => "object", "properties" => {} } : schema
32
+ end
33
+
34
+ def execute(**params)
35
+ live = @client_for.call.tool(@tool["name"])
36
+ raise Insika::NotFoundError, "tool '#{@tool["name"]}' no longer offered" if live.nil?
37
+
38
+ live.execute(**params)
39
+ rescue StandardError => e
40
+ { error: "MCP instance '#{@instance_name}' tool '#{@tool["name"]}' failed: #{e.message}" }
41
+ end
42
+ end
43
+ end
@@ -3,19 +3,27 @@
3
3
  module Insika
4
4
  # MCP instances authored at runtime. One
5
5
  # record per instance in the ConfigStore (scope "mcp"), keyed by the name slug
6
- # (`tavily`, `github`, ...). Holds transport/command/url, the `enabled` flag and
7
- # a Hash of `env` credentials (tokens/keys the instance injects into the server).
6
+ # (`tavily`, `github`, ...). Holds transport/command/args/url, the `enabled`
7
+ # flag, and two credential Hashes: `env` (stdio child-process environment) and
8
+ # `headers` (http/sse request headers). `env` used to double as
9
+ # "HTTP headers" for http/sse instances; a record written under that old
10
+ # meaning is migrated ON READ (see `migrate_legacy_headers!`), never rewritten
11
+ # silently (`insika doctor` flags it so the operator re-saves it explicitly).
8
12
  #
9
- # The credentials (`env`) NEVER leave here in plaintext to the UI: the display
10
- # reads (`get`/`all`) mask EACH value with the `__OCULTO__` sentinel. Only
11
- # `get_raw`/`all_raw` (consumed by an MCP client, never by the screen) return
12
- # the real values. On write, the sentinel coming back preserves the value; a new
13
- # string replaces it; "" clears it (see Insika::SecretMasking, the same pattern
14
- # as the LLM api_keys).
13
+ # Credentials (`env`/`headers`) NEVER leave here in plaintext to the UI: the
14
+ # display reads (`get`/`all`) mask EACH value with the `__OCULTO__` sentinel.
15
+ # Only `get_raw`/`all_raw` (consumed by Insika::McpClient, never by the
16
+ # screen) return the real values. On write, the sentinel coming back
17
+ # preserves the value; a new string replaces it; "" clears it (see
18
+ # Insika::SecretMasking, the same pattern as the LLM api_keys).
15
19
  #
16
- # Current scope: durable config CRUD (the instances UI). Running an MCP client
17
- # against these instances is later runtime work — the store is the editable
18
- # source from now on.
20
+ # Also holds `tools_cache` the last live discovery result
21
+ # (Insika::McpToolRegistry#refresh writes it; `entries`/Studio/doctor read
22
+ # it for cheap display; tool EXECUTION never depends on it.
23
+ # Not a credential -> never masked.
24
+ #
25
+ # Durable config CRUD (the instances UI) + the shape
26
+ # Insika::McpClient.for/Insika::McpToolRegistry read.
19
27
  class McpStore
20
28
  include Coercion
21
29
 
@@ -48,10 +56,25 @@ module Insika
48
56
  names.filter_map { |n| raw(n) }
49
57
  end
50
58
 
51
- # Upsert with per-env-key secret reconciliation. `attrs`
59
+ # System-written cache of the instance's discovered tools:
60
+ # Insika::McpToolRegistry#refresh connects live and writes the result here
61
+ # for cheap display (Studio/doctor/`entries`) — execution never reads it
62
+ # back, only the live client. NOT a credential -> never masked. -> the
63
+ # MASKED record (mirrors upsert's return).
64
+ def set_tools_cache(name, tools)
65
+ existing = raw(name)
66
+ raise Insika::NotFoundError, "MCP instance '#{name}' not found" if existing.nil?
67
+
68
+ record = existing.merge("tools_cache" => normalize_tools_cache(tools))
69
+ @cs.put(SCOPE, name, record)
70
+ mask(record)
71
+ end
72
+
73
+ # Upsert with per-key secret reconciliation. `attrs`
52
74
  # (string|symbol keys):
53
- # name (required), transport, command, url, description,
54
- # enabled (bool), env ({ "KEY" => value|sentinel|"" })
75
+ # name (required), transport, command, args ([String, ...]), url,
76
+ # description, enabled (bool), env ({ "KEY" => value|sentinel|"" }),
77
+ # headers ({ "Header-Name" => value|sentinel|"" })
55
78
  # -> MASKED Hash (the stored record).
56
79
  def upsert(attrs)
57
80
  h = symbolize(attrs)
@@ -63,10 +86,15 @@ module Insika
63
86
  "name" => name,
64
87
  "transport" => presence(h[:transport]) || "stdio",
65
88
  "command" => presence(h[:command]),
89
+ "args" => Array(h[:args]).map(&:to_s).reject(&:empty?),
66
90
  "url" => presence(h[:url]),
67
91
  "description" => presence(h[:description]),
68
92
  "enabled" => h.fetch(:enabled, true) ? true : false,
69
- "env" => reconcile_env(h[:env], existing && existing["env"])
93
+ "env" => reconcile_hash(h[:env], existing && existing["env"]),
94
+ "headers" => reconcile_hash(h[:headers], existing && existing["headers"]),
95
+ # editing an instance's connection details doesn't change its discovered
96
+ # tools -> preserved across upsert; only set_tools_cache writes it.
97
+ "tools_cache" => (existing && existing["tools_cache"]) || []
70
98
  }
71
99
  @cs.put(SCOPE, name, record)
72
100
  mask(record)
@@ -75,25 +103,69 @@ module Insika
75
103
  # -> bool (did it exist?).
76
104
  def delete(name) = @cs.delete(SCOPE, name.to_s)
77
105
 
106
+ # -> [String] names of http/sse instances still stored under the pre-
107
+ # Meaning of `env` (`insika doctor`'s "mcp" check; `raw`/`all_raw`
108
+ # already read them correctly — this is only to flag the ones that still
109
+ # need a re-save so the stored record itself catches up).
110
+ def legacy_header_names
111
+ names.select { |n| needs_header_migration?(@cs.get(SCOPE, n)) }
112
+ end
113
+
78
114
  private
79
115
 
80
- def raw(name) = @cs.get(SCOPE, name.to_s)
116
+ def raw(name) = migrate_legacy_headers(@cs.get(SCOPE, name.to_s))
117
+
118
+ # A record written before used `env` as HTTP headers for an
119
+ # http/sse instance. On READ ONLY (never rewritten silently — `insika
120
+ # doctor` flags it via `legacy_header_names` so the operator re-saves it
121
+ # explicitly), an http/sse record with `env` set and no `headers` yet is
122
+ # read as if `env` had been `headers` all along. A stdio record's `env` is
123
+ # untouched — it always meant process environment.
124
+ def migrate_legacy_headers(record)
125
+ return record unless needs_header_migration?(record)
126
+
127
+ record.merge("headers" => record["env"], "env" => {})
128
+ end
81
129
 
82
- # Each env value becomes the sentinel (or disappears if empty) — never leaks plaintext.
130
+ def needs_header_migration?(record)
131
+ return false if record.nil? || !http_like?(record["transport"])
132
+
133
+ (record["env"] || {}).any? && (record["headers"] || {}).empty?
134
+ end
135
+
136
+ def http_like?(transport) = %w[http sse].include?(transport.to_s)
137
+
138
+ # -> [{"name","description","inputSchema"}] string-keyed, dropping any
139
+ # entry without a name (nothing to register a Registry::Entry under).
140
+ def normalize_tools_cache(tools)
141
+ Array(tools).filter_map do |t|
142
+ h = stringify_hash(t)
143
+ name = presence(h["name"])
144
+ next nil if name.nil?
145
+
146
+ { "name" => name, "description" => h["description"].to_s, "inputSchema" => h["inputSchema"] || {} }
147
+ end
148
+ end
149
+
150
+ # Each env/headers value becomes the sentinel (or disappears if empty) — never leaks plaintext.
83
151
  def mask(record)
84
152
  return nil if record.nil?
85
153
 
86
- env = (record["env"] || {}).each_with_object({}) do |(k, v), acc|
87
- acc[k] = SecretMasking.mask(v)
88
- end
89
- record.merge("env" => env)
154
+ record.merge(
155
+ "env" => mask_hash(record["env"]),
156
+ "headers" => mask_hash(record["headers"])
157
+ )
158
+ end
159
+
160
+ def mask_hash(hash)
161
+ (hash || {}).each_with_object({}) { |(k, v), acc| acc[k] = SecretMasking.mask(v) }
90
162
  end
91
163
 
92
- # Reconciles the env received from the form against the stored one, key by key: a
93
- # key that came as the sentinel is preserved; a new string replaces it; "" (or missing
94
- # from the submission) clears it. NEW keys are added; old keys absent from the form
95
- # are removed (the form sends the complete set of keys).
96
- def reconcile_env(incoming, existing)
164
+ # Reconciles a credential Hash received from the form against the stored one, key by
165
+ # key: a key that came as the sentinel is preserved; a new string replaces it; "" (or
166
+ # missing from the submission) clears it. NEW keys are added; old keys absent from the
167
+ # form are removed (the form sends the complete set of keys).
168
+ def reconcile_hash(incoming, existing)
97
169
  inc = stringify_hash(incoming)
98
170
  old = existing || {}
99
171
  inc.each_with_object({}) do |(k, v), acc|
@@ -31,12 +31,19 @@ module Insika
31
31
  # stdio has no HTTP endpoint -> raises a clear error (later work).
32
32
  # - MCP session lifecycle (initialize/negotiation/session-id/notifications) and the
33
33
  # UNWRAP of the `tools/call` response (`{content:[{type,text}]}`) — the binding
34
- # makes a stateless POST and returns the raw body (extract body_raw).
35
- # - Credential injection (the instance `env`) as an auth header in the HTTP
36
- # binding: the `env` is consumed by a real MCP client (deferred), not mapped
37
- # to a header here.
34
+ # makes a stateless POST and returns the raw body (extract body_raw). A server
35
+ # that requires the handshake before answering `tools/list`/`tools/call` is not
36
+ # reachable yet that IS the real transport, not this minimal client.
38
37
  # - Tools whose name/top-level property is outside the ToolDefinition NAME_RE
39
38
  # (uppercase/hyphens) are ISOLATED into `errors[]` by the import (R4).
39
+ #
40
+ # CREDENTIAL INJECTION: the instance `env` (Hash) is sent verbatim as HTTP
41
+ # headers, on BOTH the discovery request (`tools/list`, this file) and every
42
+ # ingested tool's `tools/call` binding (`build_manifest`'s `defaults.headers`)
43
+ # — literal header-name -> value, the same convention as a data-tool's own
44
+ # `secret_headers` (the operator types the full value, "Bearer xxx" included;
45
+ # no magic prefixing). An instance with no `env` sends the bare
46
+ # `Content-Type` header only, byte-for-byte what shipped before this.
40
47
  class McpToolIngestor
41
48
  def initialize(mcp_store:, import_tools:, client_factory: nil)
42
49
  @mcp_store = mcp_store
@@ -69,17 +76,32 @@ module Insika
69
76
  end
70
77
 
71
78
  tools = Array((client || @client_factory.call(record)).list_tools)
72
- build_manifest(name.to_s, url, tools)
79
+ build_manifest(name.to_s, url, tools, env_headers(record))
73
80
  end
74
81
 
75
82
  private
76
83
 
77
- def build_manifest(name, url, tools)
84
+ # `headers` (an http/sse instance's credentials — `env` reads
85
+ # as `headers` for these transports via McpStore's migration-on-read, see
86
+ # McpStore#migrate_legacy_headers) -> literal HTTP headers (header-NAME ->
87
+ # value; the value carries its own scheme, e.g. "Bearer xxx" — no magic
88
+ # prefixing). NOT `secret_headers`: ToolManifest's own rule for a secret
89
+ # header requires a `{{secret.*}}` reference, which is the opposite of what
90
+ # a literal credential is — so an MCP instance's headers are NOT masked in
91
+ # a tool's edit page in /studio/tools, unlike a hand-authored data-tool's
92
+ # Authorization header. Acceptable for now (Studio is already
93
+ # operator-gated); a real fix routes this through the deployment's own
94
+ # {{secret.*}}/{{env.*}} resolution instead — later work.
95
+ def env_headers(record)
96
+ (record["headers"] || {}).each_with_object({}) { |(k, v), acc| acc[k.to_s] = v.to_s }
97
+ end
98
+
99
+ def build_manifest(name, url, tools, headers)
78
100
  {
79
101
  "version" => 1,
80
102
  "defaults" => {
81
103
  "method" => "POST",
82
- "headers" => { "Content-Type" => "application/json" },
104
+ "headers" => { "Content-Type" => "application/json" }.merge(headers),
83
105
  "response" => { "extract" => "body_raw" },
84
106
  "group" => "mcp:#{name}"
85
107
  },
@@ -127,7 +149,7 @@ module Insika
127
149
  end
128
150
 
129
151
  def default_client(record)
130
- Insika::McpHttpClient.new(url: record["url"])
152
+ Insika::McpHttpClient.new(url: record["url"], headers: env_headers(record))
131
153
  end
132
154
 
133
155
  def presence(str) = Insika::Coercion.presence(str)