insika 0.2.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (300) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +325 -17
  3. data/README.md +52 -11
  4. data/bin/insika +895 -8
  5. data/bin/insika-router +87 -0
  6. data/docs/AGENTS.md +229 -163
  7. data/docs/API.md +56 -0
  8. data/docs/ARCHITECTURE.md +3 -2
  9. data/docs/ARTIFACTS.md +95 -0
  10. data/docs/BENCHMARK.md +2 -2
  11. data/docs/CHANNELS.md +111 -21
  12. data/docs/CONTEXT.md +132 -19
  13. data/docs/DEMO.md +80 -0
  14. data/docs/DEPLOY.md +75 -4
  15. data/docs/EMBEDDING.md +1 -1
  16. data/docs/EVALS.md +146 -2
  17. data/docs/FACTS.md +135 -0
  18. data/docs/HARVEST.md +116 -0
  19. data/docs/KNOWLEDGE.md +290 -0
  20. data/docs/LOADTEST.md +16 -10
  21. data/docs/MEDIA.md +128 -0
  22. data/docs/OBSERVABILITY.md +65 -6
  23. data/docs/OUTCOMES.md +137 -0
  24. data/docs/PLUGINS.md +51 -6
  25. data/docs/POLICY.md +216 -0
  26. data/docs/REFINEMENT.md +17 -12
  27. data/docs/RELEASING.md +27 -0
  28. data/docs/ROUTER.md +213 -0
  29. data/docs/RUNNING-LOCAL.md +6 -6
  30. data/docs/SCHEDULING.md +121 -0
  31. data/docs/SECURITY.md +79 -2
  32. data/docs/SKILLS.md +13 -4
  33. data/docs/SOAK.md +127 -0
  34. data/docs/TEMPLATES.md +134 -0
  35. data/docs/TOOLS.md +220 -27
  36. data/docs/WHY.md +1 -1
  37. data/docs/WORKFLOWS.md +2 -2
  38. data/docs/_includes/head_custom.html +5 -0
  39. data/docs/_includes/title.html +13 -0
  40. data/docs/_sass/color_schemes/insika.scss +32 -0
  41. data/docs/_sass/custom/custom.scss +199 -0
  42. data/docs/_sass/custom/setup.scss +26 -0
  43. data/docs/assets/img/favicon.svg +7 -0
  44. data/docs/assets/img/insika-mark.svg +7 -0
  45. data/docs/core-concepts.md +21 -0
  46. data/docs/domain.md +115 -0
  47. data/docs/improve.md +20 -0
  48. data/docs/index.md +9 -6
  49. data/docs/integrate.md +20 -0
  50. data/docs/operate.md +13 -6
  51. data/docs/prompts/ADD-TOOL.md +118 -0
  52. data/docs/prompts/DIAGNOSE-TURN.md +65 -0
  53. data/docs/prompts/GO-LIVE.md +138 -0
  54. data/docs/prompts/RUN-EXAMPLES.md +70 -0
  55. data/docs/reference.md +19 -0
  56. data/docs/ship.md +10 -2
  57. data/docs/start-here.md +18 -0
  58. data/lib/insika/agent_profile.rb +202 -10
  59. data/lib/insika/artifact_signing.rb +82 -0
  60. data/lib/insika/artifact_store.rb +160 -0
  61. data/lib/insika/balloon_splitter.rb +102 -0
  62. data/lib/insika/budget_ledger.rb +34 -3
  63. data/lib/insika/cache_series_store.rb +49 -0
  64. data/lib/insika/channel_delivery.rb +119 -14
  65. data/lib/insika/channels/relay.rb +77 -3
  66. data/lib/insika/channels/web.rb +2 -2
  67. data/lib/insika/chat_builder.rb +105 -2
  68. data/lib/insika/checkpoint_store.rb +16 -0
  69. data/lib/insika/commands/agent_payload.rb +3 -3
  70. data/lib/insika/commands/backfill_knowledge.rb +145 -0
  71. data/lib/insika/commands/cancel_followup.rb +49 -0
  72. data/lib/insika/commands/delete_artifact.rb +35 -0
  73. data/lib/insika/commands/delete_concept.rb +34 -0
  74. data/lib/insika/commands/delete_mcp.rb +6 -2
  75. data/lib/insika/commands/delete_tenant_data.rb +107 -0
  76. data/lib/insika/commands/export_customer_memory.rb +48 -0
  77. data/lib/insika/commands/forget_customer.rb +117 -0
  78. data/lib/insika/commands/freeze_funnel_baseline.rb +113 -0
  79. data/lib/insika/commands/gate_harvest.rb +138 -0
  80. data/lib/insika/commands/gate_refinement.rb +1 -1
  81. data/lib/insika/commands/judge_shadow_pairs.rb +124 -0
  82. data/lib/insika/commands/memory_forget_fact.rb +20 -4
  83. data/lib/insika/commands/memory_put_fact.rb +23 -4
  84. data/lib/insika/commands/promote_harvest.rb +130 -0
  85. data/lib/insika/commands/record_outcome.rb +46 -0
  86. data/lib/insika/commands/record_shadow_reply.rb +68 -0
  87. data/lib/insika/commands/refresh_mcp_tools.rb +47 -0
  88. data/lib/insika/commands/reject_harvest.rb +38 -0
  89. data/lib/insika/commands/resolve_proposal.rb +108 -0
  90. data/lib/insika/commands/restore_concept.rb +34 -0
  91. data/lib/insika/commands/revoke_contact.rb +49 -0
  92. data/lib/insika/commands/rollback_harvest.rb +86 -0
  93. data/lib/insika/commands/run_distillation.rb +186 -0
  94. data/lib/insika/commands/run_harvest.rb +393 -0
  95. data/lib/insika/commands/seed_demo_data.rb +31 -0
  96. data/lib/insika/commands/send_message.rb +103 -6
  97. data/lib/insika/commands/session_purge.rb +67 -0
  98. data/lib/insika/commands/upsert_mcp.rb +6 -3
  99. data/lib/insika/commands/write_concept.rb +57 -0
  100. data/lib/insika/contact_store.rb +183 -0
  101. data/lib/insika/context/builder.rb +21 -3
  102. data/lib/insika/context/fragment.rb +7 -3
  103. data/lib/insika/context/priority.rb +5 -0
  104. data/lib/insika/context/provider.rb +17 -3
  105. data/lib/insika/context/providers/briefing.rb +96 -0
  106. data/lib/insika/context/providers/knowledge.rb +108 -0
  107. data/lib/insika/context/providers/memory.rb +16 -7
  108. data/lib/insika/context/providers/prompt.rb +57 -23
  109. data/lib/insika/context/providers/skill.rb +2 -0
  110. data/lib/insika/context/providers/tool_search.rb +2 -0
  111. data/lib/insika/context_trace_store.rb +38 -2
  112. data/lib/insika/cron.rb +189 -0
  113. data/lib/insika/demo/agent_attrs.rb +43 -0
  114. data/lib/insika/demo/golden_cases.rb +81 -0
  115. data/lib/insika/demo/seeder.rb +336 -0
  116. data/lib/insika/distill.rb +224 -0
  117. data/lib/insika/distill_engine.rb +169 -0
  118. data/lib/insika/doctor.rb +889 -11
  119. data/lib/insika/dsl/definition.rb +3 -2
  120. data/lib/insika/dsl/runtime.rb +69 -82
  121. data/lib/insika/dsl/server_boot.rb +92 -1
  122. data/lib/insika/dsl/system.rb +10 -2
  123. data/lib/insika/dsl.rb +186 -5
  124. data/lib/insika/edge_limiter.rb +28 -7
  125. data/lib/insika/env_schema.rb +25 -3
  126. data/lib/insika/errors.rb +11 -0
  127. data/lib/insika/evals/assertions.rb +3 -2
  128. data/lib/insika/evals/golden.rb +41 -4
  129. data/lib/insika/evals/judge.rb +47 -2
  130. data/lib/insika/evals/pairwise.rb +29 -6
  131. data/lib/insika/evals/persona.rb +98 -0
  132. data/lib/insika/evals/runner.rb +9 -0
  133. data/lib/insika/evals/simulator.rb +225 -0
  134. data/lib/insika/evals/transport.rb +83 -1
  135. data/lib/insika/event_stream.rb +10 -0
  136. data/lib/insika/evidence.rb +183 -0
  137. data/lib/insika/executor.rb +869 -76
  138. data/lib/insika/followup_engine.rb +207 -0
  139. data/lib/insika/followup_policy.rb +198 -0
  140. data/lib/insika/followup_store.rb +306 -0
  141. data/lib/insika/funnel_declaration.rb +106 -0
  142. data/lib/insika/funnel_fold.rb +179 -0
  143. data/lib/insika/funnel_store.rb +163 -0
  144. data/lib/insika/golden_store.rb +17 -2
  145. data/lib/insika/grounding/matcher.rb +69 -0
  146. data/lib/insika/grounding.rb +44 -0
  147. data/lib/insika/harvest/conversion_gate.rb +159 -0
  148. data/lib/insika/harvest/criterion.rb +98 -0
  149. data/lib/insika/harvest/gate.rb +194 -0
  150. data/lib/insika/harvest/negative_list.rb +199 -0
  151. data/lib/insika/harvest.rb +241 -0
  152. data/lib/insika/harvest_engine.rb +193 -0
  153. data/lib/insika/harvest_store.rb +548 -0
  154. data/lib/insika/knowledge.rb +680 -0
  155. data/lib/insika/knowledge_store.rb +140 -0
  156. data/lib/insika/mcp_client.rb +94 -0
  157. data/lib/insika/mcp_json.rb +74 -0
  158. data/lib/insika/mcp_live_tool.rb +43 -0
  159. data/lib/insika/mcp_store.rb +98 -26
  160. data/lib/insika/mcp_tool_ingestor.rb +30 -8
  161. data/lib/insika/mcp_tool_registry.rb +100 -0
  162. data/lib/insika/media.rb +382 -0
  163. data/lib/insika/memory_audit_store.rb +85 -0
  164. data/lib/insika/memory_store.rb +264 -23
  165. data/lib/insika/message_origin.rb +7 -2
  166. data/lib/insika/middleware.rb +9 -0
  167. data/lib/insika/model_visible.rb +87 -0
  168. data/lib/insika/model_visible_trace_store.rb +66 -0
  169. data/lib/insika/onboarding.rb +22 -1
  170. data/lib/insika/outbox_store.rb +42 -4
  171. data/lib/insika/outcome_store.rb +147 -0
  172. data/lib/insika/overlay_tool_registry.rb +37 -17
  173. data/lib/insika/packaging.rb +163 -0
  174. data/lib/insika/parity/criterion.rb +79 -0
  175. data/lib/insika/parity/verdict.rb +318 -0
  176. data/lib/insika/prefix_fingerprint.rb +58 -0
  177. data/lib/insika/profile_source.rb +29 -1
  178. data/lib/insika/prompt_catalog.rb +10 -0
  179. data/lib/insika/proposal_store.rb +271 -0
  180. data/lib/insika/queue_policy.rb +4 -1
  181. data/lib/insika/refinement/proposer.rb +1 -1
  182. data/lib/insika/reliability.rb +32 -6
  183. data/lib/insika/retention.rb +316 -0
  184. data/lib/insika/router/app.rb +157 -0
  185. data/lib/insika/router/backend_pool.rb +98 -0
  186. data/lib/insika/router/hash_ring.rb +55 -0
  187. data/lib/insika/router/proxy_body.rb +34 -0
  188. data/lib/insika/router/session_key.rb +54 -0
  189. data/lib/insika/router.rb +18 -0
  190. data/lib/insika/routing.rb +101 -0
  191. data/lib/insika/safety/config.rb +44 -4
  192. data/lib/insika/safety/corpus.rb +255 -0
  193. data/lib/insika/safety/detectors.rb +32 -113
  194. data/lib/insika/safety/factory.rb +15 -2
  195. data/lib/insika/safety/grounding_enforcer.rb +59 -0
  196. data/lib/insika/safety/grounding_validator.rb +49 -0
  197. data/lib/insika/safety/input_guardrail.rb +1 -1
  198. data/lib/insika/safety/moderator.rb +1 -1
  199. data/lib/insika/safety/output_filter.rb +9 -5
  200. data/lib/insika/safety/output_validator.rb +11 -5
  201. data/lib/insika/schedule.rb +177 -0
  202. data/lib/insika/schedule_engine.rb +314 -0
  203. data/lib/insika/schedule_store.rb +208 -0
  204. data/lib/insika/schema_guard.rb +35 -0
  205. data/lib/insika/server/app.rb +247 -17
  206. data/lib/insika/server/rack_app.rb +21 -1
  207. data/lib/insika/server/responses.rb +40 -1
  208. data/lib/insika/session_actor.rb +8 -4
  209. data/lib/insika/session_store.rb +65 -2
  210. data/lib/insika/settings_store.rb +10 -0
  211. data/lib/insika/shadow_pair_store.rb +258 -0
  212. data/lib/insika/skill_catalog.rb +16 -0
  213. data/lib/insika/soak/envelope.rb +140 -0
  214. data/lib/insika/soak/report.rb +392 -0
  215. data/lib/insika/soak/runner.rb +554 -0
  216. data/lib/insika/steer_injector.rb +21 -10
  217. data/lib/insika/store.rb +10 -1
  218. data/lib/insika/stores/memory.rb +6 -0
  219. data/lib/insika/stores/sqlite.rb +8 -0
  220. data/lib/insika/studio/app.rb +1437 -54
  221. data/lib/insika/studio/assets/dist/application.css +1 -1
  222. data/lib/insika/studio/assets/dist/application.js +23 -23
  223. data/lib/insika/studio/assets/dist/favicon.svg +6 -0
  224. data/lib/insika/studio/forms.rb +308 -10
  225. data/lib/insika/studio/nav_icons.rb +19 -1
  226. data/lib/insika/studio/views/_agent_tab_cache.erb +25 -0
  227. data/lib/insika/studio/views/_agent_tab_config.erb +514 -0
  228. data/lib/insika/studio/views/_agent_tab_history.erb +24 -0
  229. data/lib/insika/studio/views/_agent_tab_loops.erb +54 -0
  230. data/lib/insika/studio/views/_agent_tab_memory.erb +51 -0
  231. data/lib/insika/studio/views/_agent_tab_outcomes.erb +31 -0
  232. data/lib/insika/studio/views/_agent_tab_prompts.erb +108 -0
  233. data/lib/insika/studio/views/_agent_tab_skills.erb +38 -0
  234. data/lib/insika/studio/views/_agents_master.erb +44 -0
  235. data/lib/insika/studio/views/_message.erb +49 -32
  236. data/lib/insika/studio/views/agent_detail.erb +62 -278
  237. data/lib/insika/studio/views/agents.erb +71 -54
  238. data/lib/insika/studio/views/approvals.erb +4 -1
  239. data/lib/insika/studio/views/artifact.erb +23 -0
  240. data/lib/insika/studio/views/artifacts.erb +59 -0
  241. data/lib/insika/studio/views/chats.erb +4 -1
  242. data/lib/insika/studio/views/customer.erb +94 -0
  243. data/lib/insika/studio/views/customers.erb +32 -0
  244. data/lib/insika/studio/views/evals.erb +6 -3
  245. data/lib/insika/studio/views/facts.erb +133 -0
  246. data/lib/insika/studio/views/followups.erb +125 -0
  247. data/lib/insika/studio/views/funnel.erb +106 -0
  248. data/lib/insika/studio/views/harvest.erb +234 -0
  249. data/lib/insika/studio/views/home.erb +106 -66
  250. data/lib/insika/studio/views/knowledge.erb +123 -0
  251. data/lib/insika/studio/views/layout.erb +15 -11
  252. data/lib/insika/studio/views/mcp.erb +174 -80
  253. data/lib/insika/studio/views/parity.erb +147 -0
  254. data/lib/insika/studio/views/playground.erb +7 -1
  255. data/lib/insika/studio/views/session.erb +233 -124
  256. data/lib/insika/studio/views/settings.erb +41 -1
  257. data/lib/insika/studio/views/skills.erb +1 -2
  258. data/lib/insika/studio/views/system_files.erb +1 -1
  259. data/lib/insika/studio/views/task.erb +13 -0
  260. data/lib/insika/studio/views/tasks.erb +4 -1
  261. data/lib/insika/studio/views/tools.erb +24 -10
  262. data/lib/insika/task_store.rb +21 -1
  263. data/lib/insika/templates/browser-agent/README.md +36 -0
  264. data/lib/insika/templates/browser-agent/agent.rb +49 -0
  265. data/lib/insika/templates/daily-digest/README.md +38 -0
  266. data/lib/insika/templates/daily-digest/agent.rb +77 -0
  267. data/lib/insika/templates/repo-explorer/README.md +36 -0
  268. data/lib/insika/templates/repo-explorer/agent.rb +45 -0
  269. data/lib/insika/templates/research-analyst/README.md +26 -0
  270. data/lib/insika/templates/research-analyst/agent.rb +58 -0
  271. data/lib/insika/templates/review-panel/README.md +20 -0
  272. data/lib/insika/templates/review-panel/agent.rb +50 -0
  273. data/lib/insika/templates/travel-planner/README.md +35 -0
  274. data/lib/insika/templates/travel-planner/agent.rb +87 -0
  275. data/lib/insika/templates.rb +112 -0
  276. data/lib/insika/testing/store_contract.rb +27 -6
  277. data/lib/insika/tick.rb +47 -11
  278. data/lib/insika/timezone.rb +45 -0
  279. data/lib/insika/tool_definition.rb +17 -7
  280. data/lib/insika/tool_envelope.rb +69 -0
  281. data/lib/insika/tool_manifest.rb +5 -1
  282. data/lib/insika/tools/data_defined_tool.rb +10 -0
  283. data/lib/insika/tools/generate_image.rb +89 -0
  284. data/lib/insika/tools/load_knowledge.rb +74 -0
  285. data/lib/insika/tools/run_persona_eval.rb +328 -0
  286. data/lib/insika/tools/save_artifact.rb +95 -0
  287. data/lib/insika/tools/schedule_followup.rb +164 -0
  288. data/lib/insika/tools/tts.rb +47 -0
  289. data/lib/insika/tools/update_briefing.rb +126 -0
  290. data/lib/insika/turn_output.rb +1 -1
  291. data/lib/insika/turn_state.rb +49 -1
  292. data/lib/insika/turn_timing.rb +22 -2
  293. data/lib/insika/version.rb +1 -1
  294. data/lib/insika/vitals.rb +84 -0
  295. data/lib/insika/wiring/graph.rb +472 -13
  296. data/lib/insika/wiring/graph_chat.rb +102 -0
  297. data/lib/insika.rb +149 -3
  298. metadata +181 -5
  299. data/docs/build.md +0 -14
  300. data/docs/understand.md +0 -10
data/docs/TEMPLATES.md ADDED
@@ -0,0 +1,134 @@
1
+ ---
2
+ title: Templates
3
+ parent: Integrate
4
+ nav_order: 6
5
+ permalink: /templates/
6
+ ---
7
+
8
+ # Templates
9
+
10
+ Example agents shipped **inside the gem** — `lib/insika/templates/<name>/`,
11
+ one DSL file per template. `insika new <name>` copies it for you to run and
12
+ edit; the same file is what the Studio gallery evaluates to create the
13
+ agent from a click. One source of truth, two doors — never a parallel pack
14
+ format to drift.
15
+
16
+ ## The gallery
17
+
18
+ ```bash
19
+ insika new --list
20
+ ```
21
+
22
+ ```
23
+ travel-planner Starter Weather + currency data-tools against keyless public APIs …
24
+ research-analyst Advanced Insika.system fan-out — three specialist subagents research …
25
+ daily-digest Always-on A recurring schedule plus save_artifact build and publish …
26
+ review-panel Teams Two specialists reviewed in parallel by a synthesizing lead …
27
+ repo-explorer MCP Live MCP tool-loop over http — answers questions about any …
28
+ browser-agent MCP Live MCP tool-loop over stdio — navigates and summarizes …
29
+ ```
30
+
31
+ ```bash
32
+ insika new travel-planner # copies ./travel-planner/{agent.rb,README.md}
33
+ insika new travel-planner my-trip # ...into ./my-trip/ instead
34
+ ```
35
+
36
+ The CLI prints the exact run line, including any env the template needs
37
+ **set** (not just available as an override) — a stdio MCP template needs
38
+ `INSIKA_MCP_STDIO=1`, for instance. The generated script *is* the editing
39
+ surface: no Gemfile, no questionnaire, no placeholders to fill in.
40
+
41
+ The same roster appears as a "+ from template" gallery on the Studio
42
+ `/studio/agents` page — clicking **Create** dispatches the identical
43
+ `:create_agent` (and, for a system template, one per agent) the CLI-run
44
+ copy would produce. A template marked `studio: false` in its frontmatter
45
+ (none in wave 1) shows a "CLI-only for now" note instead of a button —
46
+ reserved for a template whose value is a durable workflow, until workflow
47
+ import into a running store exists.
48
+
49
+ ## The MCP trail: point it at your own server
50
+
51
+ `repo-explorer` (http) and `browser-agent` (stdio) are not showcases for
52
+ one MCP vendor — they demonstrate exactly how to plug **any** MCP server
53
+ into an agent. Each ships with a working, keyless default so
54
+ `insika new` + the run line works with zero setup, but the server is a
55
+ config value:
56
+
57
+ ```bash
58
+ MCP_URL=https://your-mcp-server/mcp DEEPSEEK_API_KEY=sk-... ruby repo-explorer/agent.rb "..."
59
+ MCP_COMMAND=your-mcp-server INSIKA_MCP_STDIO=1 DEEPSEEK_API_KEY=sk-... ruby browser-agent/agent.rb "..."
60
+ ```
61
+
62
+ Swap the env var, rewrite the instructions for the new server's tools —
63
+ nothing else in `agent.rb` changes.
64
+
65
+ ## Writing a template
66
+
67
+ A template is `lib/insika/templates/<name>/agent.rb` + `README.md`.
68
+
69
+ **The frontmatter contract** — a `# ---` … `# ---` comment block, YAML
70
+ inside, right after the standard `# frozen_string_literal: true` (that
71
+ magic comment is skipped automatically — a template doesn't have to break
72
+ the convention every other file in the gem follows):
73
+
74
+ ```ruby
75
+ # frozen_string_literal: true
76
+
77
+ # ---
78
+ # title: My Template
79
+ # trail: Starter | Advanced | Always-on | Teams | MCP
80
+ # description: one line, shown in the CLI list and the Studio card.
81
+ # capabilities: comma, separated, tags
82
+ # studio: true # optional, default true
83
+ # env: SOME_REQUIRED_VAR # optional — env the run line must SET, not just may override
84
+ # requires: Node.js and npm # optional — a local dependency beyond the gem + a provider key
85
+ # ---
86
+ ```
87
+
88
+ **The two-doors mechanics**, in the file itself:
89
+
90
+ 1. `require "insika"` — gem-style, never `require_relative` (the file gets
91
+ copied out of the gem into an arbitrary directory).
92
+ 2. Build the agent/system as a normal top-level local: `travel = Insika.agent(...) { ... }`.
93
+ 3. Guard the CLI demo footer: `if __FILE__ == $PROGRAM_NAME ... end`. False
94
+ whenever `Insika::Templates.evaluate` loads the file (never true from
95
+ inside the gem/Studio process), so the Studio door never makes a network
96
+ call, prints anything, or parses `ARGV`.
97
+ 4. End the file with the **bare** built value (`travel`, `panel`, `team`,
98
+ …) as its last expression — `evaluate` runs the file in an isolated
99
+ `instance_eval` and returns whatever that last expression is. No
100
+ registration call, no second format.
101
+ 5. **No top-level constants.** `instance_eval`'s isolation keeps local
102
+ variables and `def`s from leaking into the NEXT template evaluated in
103
+ the same process, but Ruby scopes a `CONST = ...` assignment lexically,
104
+ not by `self` — it would leak. Use a local variable (closures see it
105
+ fine from inside a `do...end` block) — every wave-1 template does.
106
+
107
+ **Engine-only rules** (enforced by the lint below):
108
+ provider-agnostic (one provider key), zero tenant/store data, external
109
+ calls only to keyless public APIs, every tool/mcp group covered by an
110
+ explicit allowlist.
111
+
112
+ **Declaring an `mcp` server** auto-adds `"mcp:<name>"` to that agent's
113
+ `tools_allow_groups` (`Insika::DSL::Builder#mcp`) — without it the agent
114
+ could never call the MCP tool it just declared, since `PackImporter`
115
+ forces `tools_allow: []` for a pack with no `data_tool`. A **system**-level
116
+ `mcp` (declared outside any member `agent { }` block) grants no agent
117
+ access by itself — declare it inside the specific agent that needs it.
118
+
119
+ ## The E3 lint
120
+
121
+ `spec/insika/templates_spec.rb` iterates `Insika::Templates.all` for real —
122
+ one example per template name, so a broken new template fails by name, not
123
+ a generic loop assertion. It checks, per template:
124
+
125
+ - evaluates cleanly to schema-valid pack(s) (`id`/`model` present);
126
+ - every `data_tool` it declares is in that SAME pack's `tools_allow`;
127
+ - every `mcp` instance's group is granted by SOME agent in the pack(s);
128
+ - every referenced host (`data_tool` URL, http/sse `mcp` URL) passes
129
+ `Insika::EgressGuard.violation` — public HTTPS only, same guard a live
130
+ turn would apply;
131
+ - no hardcoded secret-shaped literal (`sk-...`, a long `Bearer ...` token)
132
+ in the source.
133
+
134
+ Run it before adding a template: `bundle exec rspec spec/insika/templates_spec.rb`.
data/docs/TOOLS.md CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  title: Tools
3
- parent: Build an agent
4
- nav_order: 2
3
+ parent: Core concepts
4
+ nav_order: 3
5
5
  permalink: /tools/
6
6
  ---
7
7
 
@@ -12,15 +12,16 @@ kinds, and the distinction that matters is **who can change one at runtime**:
12
12
 
13
13
  | | **Code tool** | **Data tool** | **MCP tool** |
14
14
  |---|---|---|---|
15
- | What | a Ruby class (`< RubyLLM::Tool`) | an HTTP call described by config, no Ruby | an MCP server's tool, ingested |
16
- | Lives | in the deployment image | as a row in SQLite | as data-tool rows in SQLite |
17
- | Editable at runtime | no (shipped in the image) | **yes** (DSL / API / manifest / Studio) | **yes** (re-ingest) |
18
- | Reach for it when | logic must run in-process (file edit, shell, subagent) | calling an external HTTP API | adopting a whole MCP toolset at once |
15
+ | What | a Ruby class (`< RubyLLM::Tool`) | an HTTP call described by config, no Ruby | an MCP server's tool, called LIVE |
16
+ | Lives | in the deployment image | as a row in SQLite | on the MCP server, behind a live client |
17
+ | Editable at runtime | no (shipped in the image) | **yes** (DSL / API / manifest / Studio) | **yes** — enable/edit the *instance* (DSL / CLI / API / JSON import / Studio); the server owns its own tools |
18
+ | Reach for it when | logic must run in-process (file edit, shell, subagent) | calling an external HTTP API | adopting a whole external MCP server's toolset |
19
19
 
20
- **MCP tools are not a separate runtime type.** An MCP ingestor discovers an MCP
21
- server's tools and turns each into an HTTP **data tool** that posts a JSON-RPC
22
- `tools/call`, tagged with a `group` naming the source instance. (Only
23
- HTTP-transport MCP servers are ingestible; stdio is rejected.)
20
+ **MCP tools are not data tools.** Configuring an enabled MCP **instance** (any
21
+ surface below) is enough its tools appear automatically, tagged
22
+ `mcp:<instance>`, and each CALL goes straight to the server through a live,
23
+ held client (stdio process / Streamable HTTP / SSE, with the full protocol
24
+ handshake) — never a frozen snapshot. See [MCP servers](#mcp-servers) below.
24
25
 
25
26
  Code tools **win name collisions** — you cannot register a data tool whose name
26
27
  shadows a code tool.
@@ -49,6 +50,11 @@ the one you create and change without a rebuild. See
49
50
  }
50
51
  ```
51
52
 
53
+ `{{secret.api_token}}` above is only real coming through the **manifest**
54
+ write path (`POST /v1/tools/manifest`) — writing this same shape via the DSL
55
+ or Studio needs the literal header value instead; see
56
+ "[The one gotcha](#the-one-gotcha-envsecret-templating-is-manifest-only)" below.
57
+
52
58
  ### Parameters: the schema is the contract
53
59
 
54
60
  `parameters` is **JSON Schema**, and it reaches the provider verbatim — it is the only
@@ -87,15 +93,25 @@ allow never becomes a request: it returns an `{ error: … }` naming the path
87
93
  retries against. Structure is strict; a scalar may arrive in its lossless string form
88
94
  (`"2"`, `"true"`) and is never coerced — what the model sent is what the request carries.
89
95
 
90
- **Placeholders** are resolved at turn time:
96
+ **Placeholders**. Two of these resolve at turn time; `{{secret.*}}` resolves
97
+ once, at ingestion — see the gotcha below before reaching for it:
91
98
 
92
- - `{{param}}` — a declared top-level parameter, filled from the model's call.
99
+ - `{{param}}` — a declared top-level parameter, filled from the model's call,
100
+ every turn.
93
101
  - `{{ctx.*}}` — turn context set **server-side, never by the model**: a closed set
94
- of `chat_id`, `store_id`, `agent_id`, `tenant`. This is how a tool knows *which*
95
- session/agent it is acting for without trusting the model.
96
- - `{{secret.*}}` allowed **only** inside a header named in `secret_headers`.
97
- A secret placeholder anywhere else is rejected (it would leak unmasked). The
98
- real secret value is injected at provision time and never lives on disk.
102
+ of `chat_id`, `store_id`, `agent_id`, `tenant`, `image_url`. This is how a tool knows *which*
103
+ session/agent it is acting for without trusting the model. `image_url` is the
104
+ first image part on the message (a photo for analysis outside the prompt);
105
+ absent when the turn carried none. Resolved every turn, like `{{param}}`.
106
+ - `{{secret.*}}` **only resolved on the manifest ingestion path**
107
+ (`POST /v1/tools/manifest`; see "[The one gotcha](#the-one-gotcha-envsecret-templating-is-manifest-only)"
108
+ below), and only once — the resolved value is what gets stored, the token
109
+ itself never lives on disk and is never re-read per turn. Allowed **only**
110
+ inside a header named in `secret_headers`. Written any other way — DSL,
111
+ Studio, or anywhere outside a `secret_headers` header — a
112
+ `{{secret.*}}` is not a credential the engine knows how to fill; it is an
113
+ undeclared parameter, and tool registration refuses it exactly like it
114
+ refuses any other unknown placeholder.
99
115
 
100
116
  **Validation** happens on ingestion. Common rejections:
101
117
 
@@ -181,6 +197,72 @@ backend, not of whoever calls it. Every agent sharing the tool gets the same val
181
197
  > there **preserves** it — the form carries the stored values through instead of
182
198
  > replacing the record with only what it shows.
183
199
 
200
+ ## Evidence: the lean envelope and grounding
201
+
202
+ A catalog tool returns products; the model should only ever quote the ones the tool
203
+ actually returned — the store dies of a SKU the model invented. `evidence` is the
204
+ declaration that makes "no claim without a tool ID" an engine rule instead of a
205
+ prompt convention. One declaration does **both** jobs: the engine strips the result
206
+ down to what the model sees (the lean envelope) **and** records every returned id on
207
+ the session's evidence ledger. There is no "lean but not evidence" mode.
208
+
209
+ ```jsonc
210
+ { "name": "search_products",
211
+ "response": { "extract": "evidence_envelope" },
212
+ "evidence": "products" } // bare kind
213
+
214
+ { "evidence": { "kind": "products", // full form
215
+ "items": "results", // non-default paths
216
+ "attachments": "cards" } }
217
+ ```
218
+
219
+ - `evidence_envelope` is the canonical extract: the raw response body arrives under
220
+ an engine-only key, the envelope parses `items`/`attachments` out of it, and
221
+ **nothing re-fattens** — the transcript and the tool trace record only the lean
222
+ result. It **requires** the `evidence` declaration (refused at load otherwise).
223
+ - **Wire contract** — the lean result the model sees is always
224
+ `{ "items": [ { "id": "…", "line": "…" } ] }` (≤ 16 items; `line` truncated to
225
+ 200 chars). A tool whose result has no valid items yields `{ "items": [] }`,
226
+ never a null. A malformed evidence result becomes `{ "error": … }` back to the
227
+ model — a correctable tool answer, exactly like a malformed call.
228
+ - **Attachments** are the optional second half: `[{ "type": "card"|"image",
229
+ "url": "…", "caption": "…" }]` (≤ 16, url ≤ 500 chars, malformed dropped). They
230
+ **never** reach the model context or the transcript — they ride the channel
231
+ delivery as an additive `attachments` key on the outbox payload, and the channel
232
+ (or its consumer) decides what a card looks like.
233
+ - A **code tool** opts in the same way: it either returns `{ items, attachments }`
234
+ directly and declares `evidence` in its registry metadata, or exposes an
235
+ `evidence` reader. No declaration = today's tool behavior, byte for byte.
236
+
237
+ ### Grounding: policing claims against the ledger
238
+
239
+ With the ledger fed, the pack declares how claims are policed — data on the agent,
240
+ not a separate code path (see [Agents](AGENTS.md)):
241
+
242
+ ```ruby
243
+ grounding mode: :flag, matcher: { sku: '\b[A-Z]{2,4}\d{4,8}\b' }
244
+ ```
245
+
246
+ - `mode` is `flag` (the default — audit), `enforce` (cut), or `off`. Absent = off.
247
+ - `matcher.sku` is a regex for the store's SKU shape, applied to the final answer;
248
+ every match that is **not** in the evidence ledger is an ungrounded claim.
249
+ Grounding is **SKU-only** by design: a name-based half cannot flag anything
250
+ without a "this is a product name" signal, so the ledger grounds ids, and the
251
+ model quoting a returned product by its *name* is simply outside the check
252
+ (the SKU path is the claim detector). A `sku` that does not compile is refused
253
+ at build; a matcher with no `sku` builds but matches nothing — `insika doctor`
254
+ warns about it.
255
+ - **`flag`** appends an `:ungrounded` flag (category `ungrounded`, source
256
+ `evidence`) to the existing `:guardrail_flagged` event — audit after the fact,
257
+ like every other output flag.
258
+ - **`enforce`** *cuts the sentence* containing an ungrounded claim from the content
259
+ the turn persists and delivers, and the flag carries `action: "cut"` so the audit
260
+ can tell a cut from a flag. It is honest about streaming: on a streaming surface
261
+ the already-streamed bytes are the channel's reality, which is exactly why the
262
+ default is `flag` — ship `enforce` only after a matcher audit proves precision.
263
+ - Grounding is **independent of the guardrails opt-in**: an agent with guardrails
264
+ off and `grounding.mode: :flag` still gets the check.
265
+
184
266
  ## Registering a tool
185
267
 
186
268
  A tool appears in the Studio panel and enters an agent's tool-loop when it is
@@ -193,16 +275,117 @@ reload, no restart):
193
275
  3. **Manifest** — `POST /v1/tools/manifest`. Partial failure is isolated: one
194
276
  malformed tool becomes an `errors[]` entry; only a structural manifest error
195
277
  fails the whole request. The response reports `{ version, created, updated, errors }`.
196
- 4. **MCP ingestion**import a server; each of its tools becomes a data tool.
197
-
198
- ### The one gotcha: env templating is manifest-only
199
-
200
- `{{env.*}}` (and `{{secret.*}}`) are substituted **at ingestion, on the manifest
201
- path**. Other write paths do **not** resolve `{{env.*}}` a literal
202
- `{{env.API_URL}}` there fails the `http`/`https` URL check and 422s. Rule:
203
- **manifest tools may template the URL with `{{env.*}}`; tools written any other
204
- way must ship a literal URL.** `{{ctx.*}}` and `{{param}}` work everywhere (they
205
- resolve at turn time, not ingestion).
278
+ 4. ~~MCP ingestion~~retired. An MCP server's tools are no
279
+ longer written into this store at all; see [MCP servers](#mcp-servers).
280
+
281
+ ### The one gotcha: env/secret templating is manifest-only
282
+
283
+ `{{env.*}}` and `{{secret.*}}` are substituted **at ingestion, on the manifest
284
+ path, once** the resolved literal is what gets stored; the token itself
285
+ never survives to a turn. Other write paths (DSL, Studio) do
286
+ **not** resolve either: a literal `{{env.API_URL}}` in a URL fails the
287
+ `http`/`https` check and 422s; a literal `{{secret.X}}` anywhere — including
288
+ inside a header named in `secret_headers` — fails tool registration the same
289
+ way an unknown parameter would (`ToolDefinition.build`'s placeholder check
290
+ does not special-case it). Rule: **manifest tools may template a URL with
291
+ `{{env.*}}` and a `secret_headers` header with `{{secret.*}}`; tools written
292
+ any other way must ship literal values** — a real URL, and a real (masked on
293
+ read) header value. `{{ctx.*}}` and `{{param}}` work everywhere (they resolve
294
+ at turn time, not ingestion).
295
+
296
+ ## MCP servers
297
+
298
+ An MCP **instance** is durable config — transport, target, credentials, an
299
+ `enabled` flag — held in its own store, separate from data tools. Once an
300
+ instance is enabled, its tools appear in the catalog automatically (group
301
+ `mcp:<instance>`, `side_effect: true`), and every call goes straight to the
302
+ server through a live, held client — the runtime never converts an MCP tool
303
+ into a stored data tool.
304
+
305
+ **Three transports**, picked by `transport:`:
306
+
307
+ | Transport | Target | Notes |
308
+ |---|---|---|
309
+ | `stdio` | `command` + `args`, run as a child process, `env` is its process environment | requires `INSIKA_MCP_STDIO=1` — see below |
310
+ | `http` | `url` + `headers` (Streamable HTTP, the modern default) | egress-guarded like any outbound URL |
311
+ | `sse` | `url` + `headers` | same egress guard as `http` |
312
+
313
+ **The stdio gate.** A stdio instance is arbitrary command execution by
314
+ config — it saves, but refuses to start ("stdio disabled by env") until the
315
+ operator sets `INSIKA_MCP_STDIO=1` (config-over-convention, the same pattern
316
+ as the egress envs). `http`/`sse` need no such gate; their URL is checked by
317
+ the normal egress allowlist instead.
318
+
319
+ **Credentials are never visible in plaintext.** `env` (stdio) and `headers`
320
+ (http/sse) mask every value as `__OCULTO__` on read, everywhere (CLI, API,
321
+ Studio). On write, sending the sentinel back **preserves** the stored value; a
322
+ new string **replaces** it; `""` (or omitting the key) **clears** it — the
323
+ same per-key reconciliation `llm_providers` api keys use.
324
+
325
+ **Discovery vs execution.** `insika mcp refresh <name>` (or `POST
326
+ /v1/mcp/:name/import`, kept as that action's route since before the live
327
+ registry) connects live, lists the server's tools, and caches the result
328
+ (`tools_cache`) purely for display — the Studio panel and `insika doctor`.
329
+ **Execution never reads that cache**: a live turn always goes through the
330
+ held client, which does its own discovery on first use regardless of whether
331
+ `refresh` ever ran.
332
+
333
+ ### Configuring an instance
334
+
335
+ 1. **DSL** — inside `Insika.system { … }` or a single `Insika.agent { … }`:
336
+
337
+ ```ruby
338
+ mcp "tavily", transport: :http, url: "https://mcp.tavily.com/mcp",
339
+ headers: { "Authorization" => "Bearer #{ENV['TAVILY_KEY']}" }
340
+ mcp "filesystem", transport: :stdio, command: "npx",
341
+ args: ["-y", "@modelcontextprotocol/server-filesystem", "/tmp"]
342
+ ```
343
+
344
+ Code is the **template**: transport/command/args/url/description always
345
+ follow the declaration on every boot. But once the instance exists, its
346
+ `enabled` flag and its credentials are the **operator's** — a Studio/CLI/API
347
+ edit made after boot is never clobbered back by the next restart.
348
+
349
+ 2. **CLI** — `insika mcp list | add | remove | import <file.json> | test <name> |
350
+ refresh <name>`. `add` takes `--name`, `--transport`, `--command`/`--arg`
351
+ (repeatable) or `--url`/`--header "Name: value"` (repeatable)/`--env
352
+ "KEY=value"` (repeatable), `--description`, `--disabled`. `test` connects
353
+ live and prints the discovered tools (or the error) without any special
354
+ setup; `refresh` does the same and additionally updates `tools_cache`.
355
+
356
+ 3. **JSON import/export** — the same `mcpServers` shape every MCP client
357
+ (Claude Desktop, Cursor, …) already uses:
358
+
359
+ ```jsonc
360
+ {
361
+ "mcpServers": {
362
+ "tavily": { "url": "https://mcp.tavily.com/mcp", "headers": { "Authorization": "Bearer …" } },
363
+ "filesystem": { "command": "npx", "args": ["-y", "@modelcontextprotocol/server-filesystem", "/tmp"] }
364
+ }
365
+ }
366
+ ```
367
+
368
+ `insika mcp import FILE.json` upserts every entry (a bare `command` implies
369
+ `stdio`; a bare `url` implies `http`; add `"transport": "sse"` explicitly
370
+ for SSE — the bare format has no other way to spell it). The same parser
371
+ backs `PUT /v1/mcp` and Studio's "Import JSON" box; `export` produces the
372
+ document back with secrets masked as `__OCULTO__`, so round-tripping an
373
+ export never wipes a stored credential.
374
+
375
+ 4. **HTTP API** (operator-only, gateway Bearer):
376
+ - `GET /v1/mcp` — every instance, masked.
377
+ - `GET /v1/mcp/:name` — one instance, masked.
378
+ - `PUT /v1/mcp` — upsert (body = the instance attrs, `name` required).
379
+ - `DELETE /v1/mcp/:name` — remove (idempotent).
380
+ - `POST /v1/mcp/:name/import` — refresh (connect live, list tools, cache).
381
+
382
+ 5. **Studio** — the `/studio/mcp` panel (create/edit/delete). The form is
383
+ transport-aware (stdio shows command/args/env, http/sse shows
384
+ url/headers); each instance shows a status chip ("N tool(s)", "untested",
385
+ "stdio disabled", or "off") and its discovered tools from `tools_cache`; a
386
+ "Test connection" button dispatches the same `refresh_mcp_tools` seam as
387
+ `insika mcp test`; an "Import JSON" box takes a `mcpServers` document and
388
+ fans it out into one `upsert_mcp` per entry.
206
389
 
207
390
  ## Making it appear — and enter the tool-loop
208
391
 
@@ -293,9 +476,19 @@ Work down this checklist:
293
476
  4. **URL literal?** For non-manifest tools, an unresolved `{{env.*}}` would have
294
477
  422'd at import — re-check the definition.
295
478
 
479
+ ## The `save_artifact` built-in
480
+
481
+ `save_artifact` is a **registry tool** — it obeys the same per-agent
482
+ `tools_allow` as any data tool, and an agent that did not name it cannot call
483
+ it (`tools_allow: %w[save_artifact]`). The agent hands in `title` + `content`
484
+ and gets the URL back; the tenant is bound from the turn, never a parameter the
485
+ model types. See [Artifacts](ARTIFACTS.md) for the tool contract, the serving
486
+ routes, the signed link and the retention/LGPD reach.
487
+
296
488
  ## See also
297
489
 
298
490
  - [Agents](AGENTS.md) — allowlists, groups, and per-agent tool exposure.
491
+ - [Artifacts](ARTIFACTS.md) — the report destination: the tool, the routes, the signed link.
299
492
  - [Plugins](PLUGINS.md) — where a code tool comes from, and how to package one.
300
493
  - [Security](SECURITY.md) — egress, sandbox, and approval gating together.
301
494
  - [Architecture](ARCHITECTURE.md) — the tool-loop and side-effect checkpointing.
data/docs/WHY.md CHANGED
@@ -1,6 +1,6 @@
1
1
  ---
2
2
  title: Why Insika
3
- parent: Understand the idea
3
+ parent: Start here
4
4
  nav_order: 1
5
5
  permalink: /why/
6
6
  ---
data/docs/WORKFLOWS.md CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  title: Workflows
3
- parent: Build an agent
4
- nav_order: 5
3
+ parent: Core concepts
4
+ nav_order: 6
5
5
  permalink: /workflows/
6
6
  ---
7
7
 
@@ -0,0 +1,5 @@
1
+ {%- comment -%}
2
+ Appended inside <head> on every page by the theme. The theme's own favicon
3
+ include only looks for a legacy /favicon.ico, so the SVG icon is declared here.
4
+ {%- endcomment -%}
5
+ <link rel="icon" href="{{ '/assets/img/favicon.svg' | relative_url }}" type="image/svg+xml">
@@ -0,0 +1,13 @@
1
+ {%- comment -%}
2
+ Overrides the theme's title.html so the sidebar shows the pillar mark next to
3
+ the wordmark. The theme's own `site.logo` path swaps the title for a single
4
+ background image, which would drop the text — and the text is what the
5
+ browser tab, the skip link and screen readers rely on.
6
+ {%- endcomment -%}
7
+ <span class="site-title-mark" aria-hidden="true">
8
+ <svg viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" focusable="false">
9
+ <rect x="3" y="3" width="18" height="3.6" rx="1"/>
10
+ <rect x="8.7" y="7.6" width="6.6" height="8.8" class="shaft"/>
11
+ <rect x="3" y="17.4" width="18" height="3.6" rx="1"/>
12
+ </svg>
13
+ </span>{{ site.title }}
@@ -0,0 +1,32 @@
1
+ // Insika's colour scheme for Just the Docs.
2
+ //
3
+ // Selected by `color_scheme: insika` in _config.yml; the theme picks the file up
4
+ // by name and needs no other wiring. This file only maps the palette tokens from
5
+ // _sass/custom/setup.scss onto the theme's own `!default` variables — anything
6
+ // that is not a theme variable belongs in _sass/custom/custom.scss.
7
+ //
8
+ // The palette is warm stone with a terracotta accent: the name is Zulu for the
9
+ // pillar that carries a structure, and the docs should read like one — quiet
10
+ // neutrals, a single load-bearing colour. It is deliberately unlike the blue and
11
+ // purple that every other Ruby docs site defaults to.
12
+
13
+ $color-scheme: insika;
14
+
15
+ $body-background-color: $white;
16
+ $body-heading-color: $ink-900;
17
+ $body-text-color: $ink-700;
18
+ $link-color: $terracotta-100;
19
+ $nav-child-link-color: $ink-700;
20
+ $sidebar-color: $stone-050;
21
+ $border-color: $stone-200;
22
+ $base-button-color: $stone-100;
23
+ $btn-primary-color: $terracotta-100;
24
+ $code-background-color: $stone-100;
25
+ $feedback-color: darken($sidebar-color, 3%);
26
+ $table-background-color: $white;
27
+ $search-background-color: $white;
28
+ $search-result-preview-color: $ink-500;
29
+
30
+ // The theme ships accessible-pygments; github-light is the one that sits calmly
31
+ // on a warm background instead of fighting it.
32
+ @import "./vendor/accessible-pygments/github-light";