insika 0.3.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (190) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +180 -0
  3. data/README.md +45 -10
  4. data/bin/insika +684 -0
  5. data/bin/insika-router +87 -0
  6. data/docs/AGENTS.md +94 -403
  7. data/docs/API.md +5 -5
  8. data/docs/ARCHITECTURE.md +3 -2
  9. data/docs/ARTIFACTS.md +95 -0
  10. data/docs/BENCHMARK.md +2 -2
  11. data/docs/CHANNELS.md +14 -14
  12. data/docs/CONTEXT.md +9 -7
  13. data/docs/DEMO.md +80 -0
  14. data/docs/DEPLOY.md +71 -3
  15. data/docs/EMBEDDING.md +1 -1
  16. data/docs/EVALS.md +128 -3
  17. data/docs/FACTS.md +3 -3
  18. data/docs/HARVEST.md +5 -6
  19. data/docs/KNOWLEDGE.md +290 -0
  20. data/docs/LOADTEST.md +2 -2
  21. data/docs/MEDIA.md +128 -0
  22. data/docs/OBSERVABILITY.md +15 -10
  23. data/docs/OUTCOMES.md +137 -0
  24. data/docs/PLUGINS.md +51 -6
  25. data/docs/POLICY.md +216 -0
  26. data/docs/REFINEMENT.md +14 -9
  27. data/docs/RELEASING.md +4 -4
  28. data/docs/ROUTER.md +213 -0
  29. data/docs/RUNNING-LOCAL.md +3 -3
  30. data/docs/SCHEDULING.md +121 -0
  31. data/docs/SECURITY.md +22 -6
  32. data/docs/SKILLS.md +11 -2
  33. data/docs/SOAK.md +2 -2
  34. data/docs/TEMPLATES.md +134 -0
  35. data/docs/TOOLS.md +152 -27
  36. data/docs/WHY.md +1 -1
  37. data/docs/WORKFLOWS.md +2 -2
  38. data/docs/_includes/head_custom.html +5 -0
  39. data/docs/_includes/title.html +13 -0
  40. data/docs/_sass/color_schemes/insika.scss +32 -0
  41. data/docs/_sass/custom/custom.scss +199 -0
  42. data/docs/_sass/custom/setup.scss +26 -0
  43. data/docs/assets/img/favicon.svg +7 -0
  44. data/docs/assets/img/insika-mark.svg +7 -0
  45. data/docs/core-concepts.md +21 -0
  46. data/docs/domain.md +4 -4
  47. data/docs/improve.md +20 -0
  48. data/docs/index.md +8 -5
  49. data/docs/integrate.md +20 -0
  50. data/docs/operate.md +13 -6
  51. data/docs/prompts/ADD-TOOL.md +118 -0
  52. data/docs/prompts/DIAGNOSE-TURN.md +65 -0
  53. data/docs/prompts/GO-LIVE.md +138 -0
  54. data/docs/prompts/RUN-EXAMPLES.md +70 -0
  55. data/docs/reference.md +19 -0
  56. data/docs/ship.md +10 -2
  57. data/docs/start-here.md +18 -0
  58. data/lib/insika/agent_profile.rb +73 -16
  59. data/lib/insika/artifact_signing.rb +82 -0
  60. data/lib/insika/artifact_store.rb +160 -0
  61. data/lib/insika/channel_delivery.rb +1 -1
  62. data/lib/insika/chat_builder.rb +22 -2
  63. data/lib/insika/commands/agent_payload.rb +2 -2
  64. data/lib/insika/commands/backfill_knowledge.rb +145 -0
  65. data/lib/insika/commands/delete_artifact.rb +35 -0
  66. data/lib/insika/commands/delete_concept.rb +34 -0
  67. data/lib/insika/commands/delete_mcp.rb +6 -2
  68. data/lib/insika/commands/delete_tenant_data.rb +15 -3
  69. data/lib/insika/commands/gate_refinement.rb +1 -1
  70. data/lib/insika/commands/refresh_mcp_tools.rb +47 -0
  71. data/lib/insika/commands/restore_concept.rb +34 -0
  72. data/lib/insika/commands/seed_demo_data.rb +31 -0
  73. data/lib/insika/commands/upsert_mcp.rb +6 -3
  74. data/lib/insika/commands/write_concept.rb +57 -0
  75. data/lib/insika/context/priority.rb +2 -0
  76. data/lib/insika/context/providers/knowledge.rb +108 -0
  77. data/lib/insika/context/providers/prompt.rb +30 -24
  78. data/lib/insika/cron.rb +189 -0
  79. data/lib/insika/demo/agent_attrs.rb +43 -0
  80. data/lib/insika/demo/golden_cases.rb +81 -0
  81. data/lib/insika/demo/seeder.rb +336 -0
  82. data/lib/insika/doctor.rb +176 -8
  83. data/lib/insika/dsl/definition.rb +3 -2
  84. data/lib/insika/dsl/runtime.rb +60 -79
  85. data/lib/insika/dsl/server_boot.rb +23 -1
  86. data/lib/insika/dsl/system.rb +10 -2
  87. data/lib/insika/dsl.rb +103 -2
  88. data/lib/insika/env_schema.rb +16 -1
  89. data/lib/insika/evals/golden.rb +41 -4
  90. data/lib/insika/evals/judge.rb +47 -2
  91. data/lib/insika/evals/pairwise.rb +11 -0
  92. data/lib/insika/evals/persona.rb +98 -0
  93. data/lib/insika/evals/runner.rb +9 -0
  94. data/lib/insika/evals/simulator.rb +225 -0
  95. data/lib/insika/evals/transport.rb +83 -1
  96. data/lib/insika/event_stream.rb +10 -0
  97. data/lib/insika/executor.rb +231 -55
  98. data/lib/insika/followup_policy.rb +2 -25
  99. data/lib/insika/golden_store.rb +16 -1
  100. data/lib/insika/grounding/matcher.rb +1 -1
  101. data/lib/insika/knowledge.rb +680 -0
  102. data/lib/insika/knowledge_store.rb +140 -0
  103. data/lib/insika/mcp_client.rb +94 -0
  104. data/lib/insika/mcp_json.rb +74 -0
  105. data/lib/insika/mcp_live_tool.rb +43 -0
  106. data/lib/insika/mcp_store.rb +98 -26
  107. data/lib/insika/mcp_tool_ingestor.rb +30 -8
  108. data/lib/insika/mcp_tool_registry.rb +100 -0
  109. data/lib/insika/media.rb +115 -31
  110. data/lib/insika/message_origin.rb +1 -1
  111. data/lib/insika/middleware.rb +9 -0
  112. data/lib/insika/onboarding.rb +17 -1
  113. data/lib/insika/outcome_store.rb +1 -1
  114. data/lib/insika/overlay_tool_registry.rb +37 -17
  115. data/lib/insika/packaging.rb +2 -2
  116. data/lib/insika/profile_source.rb +8 -1
  117. data/lib/insika/prompt_catalog.rb +10 -0
  118. data/lib/insika/retention.rb +36 -1
  119. data/lib/insika/router/app.rb +157 -0
  120. data/lib/insika/router/backend_pool.rb +98 -0
  121. data/lib/insika/router/hash_ring.rb +55 -0
  122. data/lib/insika/router/proxy_body.rb +34 -0
  123. data/lib/insika/router/session_key.rb +54 -0
  124. data/lib/insika/router.rb +18 -0
  125. data/lib/insika/schedule.rb +177 -0
  126. data/lib/insika/schedule_engine.rb +314 -0
  127. data/lib/insika/schedule_store.rb +208 -0
  128. data/lib/insika/server/app.rb +105 -15
  129. data/lib/insika/server/rack_app.rb +5 -1
  130. data/lib/insika/server/responses.rb +1 -1
  131. data/lib/insika/skill_catalog.rb +12 -0
  132. data/lib/insika/steer_injector.rb +21 -10
  133. data/lib/insika/studio/app.rb +567 -45
  134. data/lib/insika/studio/assets/dist/application.css +1 -1
  135. data/lib/insika/studio/assets/dist/application.js +21 -21
  136. data/lib/insika/studio/forms.rb +46 -5
  137. data/lib/insika/studio/nav_icons.rb +14 -1
  138. data/lib/insika/studio/views/_agent_tab_cache.erb +25 -0
  139. data/lib/insika/studio/views/_agent_tab_config.erb +514 -0
  140. data/lib/insika/studio/views/_agent_tab_history.erb +24 -0
  141. data/lib/insika/studio/views/_agent_tab_loops.erb +54 -0
  142. data/lib/insika/studio/views/_agent_tab_memory.erb +51 -0
  143. data/lib/insika/studio/views/_agent_tab_outcomes.erb +31 -0
  144. data/lib/insika/studio/views/_agent_tab_prompts.erb +108 -0
  145. data/lib/insika/studio/views/_agent_tab_skills.erb +38 -0
  146. data/lib/insika/studio/views/_agents_master.erb +44 -0
  147. data/lib/insika/studio/views/_message.erb +49 -32
  148. data/lib/insika/studio/views/agent_detail.erb +61 -820
  149. data/lib/insika/studio/views/agents.erb +70 -57
  150. data/lib/insika/studio/views/artifact.erb +23 -0
  151. data/lib/insika/studio/views/artifacts.erb +59 -0
  152. data/lib/insika/studio/views/evals.erb +2 -2
  153. data/lib/insika/studio/views/facts.erb +1 -1
  154. data/lib/insika/studio/views/funnel.erb +1 -1
  155. data/lib/insika/studio/views/home.erb +106 -67
  156. data/lib/insika/studio/views/knowledge.erb +123 -0
  157. data/lib/insika/studio/views/layout.erb +14 -11
  158. data/lib/insika/studio/views/mcp.erb +174 -80
  159. data/lib/insika/studio/views/session.erb +231 -177
  160. data/lib/insika/studio/views/settings.erb +39 -1
  161. data/lib/insika/studio/views/skills.erb +1 -1
  162. data/lib/insika/studio/views/tools.erb +24 -9
  163. data/lib/insika/templates/browser-agent/README.md +36 -0
  164. data/lib/insika/templates/browser-agent/agent.rb +49 -0
  165. data/lib/insika/templates/daily-digest/README.md +38 -0
  166. data/lib/insika/templates/daily-digest/agent.rb +77 -0
  167. data/lib/insika/templates/repo-explorer/README.md +36 -0
  168. data/lib/insika/templates/repo-explorer/agent.rb +45 -0
  169. data/lib/insika/templates/research-analyst/README.md +26 -0
  170. data/lib/insika/templates/research-analyst/agent.rb +58 -0
  171. data/lib/insika/templates/review-panel/README.md +20 -0
  172. data/lib/insika/templates/review-panel/agent.rb +50 -0
  173. data/lib/insika/templates/travel-planner/README.md +35 -0
  174. data/lib/insika/templates/travel-planner/agent.rb +87 -0
  175. data/lib/insika/templates.rb +112 -0
  176. data/lib/insika/tick.rb +24 -12
  177. data/lib/insika/timezone.rb +45 -0
  178. data/lib/insika/tools/generate_image.rb +52 -7
  179. data/lib/insika/tools/load_knowledge.rb +74 -0
  180. data/lib/insika/tools/run_persona_eval.rb +328 -0
  181. data/lib/insika/tools/save_artifact.rb +95 -0
  182. data/lib/insika/turn_output.rb +1 -1
  183. data/lib/insika/turn_state.rb +15 -4
  184. data/lib/insika/version.rb +1 -1
  185. data/lib/insika/wiring/graph.rb +184 -12
  186. data/lib/insika/wiring/graph_chat.rb +102 -0
  187. data/lib/insika.rb +57 -0
  188. metadata +105 -5
  189. data/docs/build.md +0 -14
  190. data/docs/understand.md +0 -10
@@ -0,0 +1,121 @@
1
+ ---
2
+ title: Schedules
3
+ parent: Operate
4
+ nav_order: 2
5
+ permalink: /schedules/
6
+ ---
7
+
8
+ # Schedules — recurring turns the engine fires
9
+
10
+ A schedule is a turn nobody has to remember to send: a daily report at 22:00,
11
+ an eval sweep every night, a heartbeat every hour. The engine fires it on its
12
+ own periodic tick — no cron on some other box pointing at an authenticated
13
+ route. (That route still works, if you want it; the built-in trigger just
14
+ removes the homework.)
15
+
16
+ A schedule is **declared on the agent** — pack data, like `followup:` or the
17
+ budget — in one of the same three places every profile field is edited: the
18
+ DSL at import, `POST /v1/agents` in the pack, or the Studio's config form
19
+ (the **Schedules** group on the agent page). Edits are hot: the next pass
20
+ sees them.
21
+
22
+ ## The declaration
23
+
24
+ ```ruby
25
+ agent = Insika.agent("reporter") do
26
+ schedule "daily_report", cron: "0 22 * * *", tz: "America/Sao_Paulo",
27
+ message: "Run the daily report now.",
28
+ overrides: { turn_timeout: 900, max_tool_calls: 200 }
29
+ schedule "heartbeat", every: 3600, message: "Say you are alive."
30
+ end
31
+ ```
32
+
33
+ | Key | Meaning |
34
+ |---|---|
35
+ | `id` | the schedule's name (the argument). Lowercase, `[a-z][a-z0-9_-]*` |
36
+ | `cron` | a five-field expression (`minute hour day-of-month month day-of-week`), or |
37
+ | `every` | a plain interval in seconds — the two are **exclusive** |
38
+ | `tz` | IANA zone for **cron** materialization (default `Etc/UTC`). `every` never needs it — every comparison runs in UTC |
39
+ | `message` | the synthetic inbound that kicks each run — what the agent "hears" |
40
+ | `session_mode` | `"new"` (default) — a fresh session per run, the report shape; `"fixed"` — one standing session, the "standing assistant" shape |
41
+ | `session_id` | for `fixed` sessions: the standing session (created on first run when missing) |
42
+ | `overrides` | per-run ceilings: `turn_timeout`, `max_tool_calls`, `model` — a report needs a bigger ceiling than a chat turn; the base profile is untouched |
43
+ | `enabled` | `false` pauses the schedule (the Studio toggle / the JSON field) |
44
+
45
+ > **The cadence floor.** Firing rides one claim window per pass — a schedule
46
+ > fires **at most once per window**. That is the true cadence ceiling: an
47
+ > `every: 60` does not fire sixty times a minute, it fires once per window.
48
+ > `doctor` warns when a declared `every` is shorter than the claim window.
49
+
50
+ The Studio renders the schedules section as a JSON array of the same
51
+ declarations plus a read-only card: each schedule's next fire, last run and —
52
+ when a window was skipped — the skip reason. `doctor` parses every
53
+ declaration with the engine's own parser: an invalid cron, an unknown
54
+ runtime zone, a schedule with neither trigger, an unknown override key —
55
+ each is named, per agent, as an error finding.
56
+
57
+ ## The engine's triggers: cron subset
58
+
59
+ Five fields, whitespace-separated. Per field: `*` (or `?`), a single value, a
60
+ range (`N-M`), a step (`*/N`, `N-M/N`, `N/N`), or a comma list of those.
61
+ Day-of-week is `0-7` with `7` = Sunday; when **both** day fields are
62
+ restricted the date matches on **either** (standard cron OR semantics).
63
+ `L`, `W`, `#` and month/day names are refused loudly at creation — the engine
64
+ will not silently ignore a cron that only some dates understand.
65
+
66
+ ```text
67
+ minute hour day-of-month month day-of-week
68
+ 0 22 * * *
69
+ ```
70
+
71
+ ## Firing: one turn per window, no catch-up
72
+
73
+ Firing rides the tick, gated by its own claim window (one scheduler per
74
+ window across `N` workers — the same claim the outbox and recovery sweeps
75
+ use). Each due schedule is claimed transactionally: the task and the
76
+ schedule's state (last run, last task id, next fire) commit together, so two
77
+ workers racing serialize on the backend's lock and **exactly one fires per
78
+ window**.
79
+
80
+ Three skip rules are part of the contract, all recorded on the schedule and
81
+ shown in the Studio — never silent, never queued:
82
+
83
+ - **late** — the no-catch-up policy. A window more than one claim window in
84
+ the past is **missed, not replayed**: a deploy that was down over 22:00 does
85
+ not fire a 22:00 report at 06:00 the next morning. The schedule's lattice
86
+ advances to the next window and the skip is recorded.
87
+ - **overlap** — the previous run's task is still live (`queued`/`running`/
88
+ `waiting`/`paused`). The window is skipped, recorded, and the next one
89
+ fires. There is no queue buildup: one scheduled run at a time per schedule.
90
+ - **budget** — a **hard** calendar budget (`budget daily: …` on the profile)
91
+ already at/over its cap. The edge would fail the turn anyway; the engine
92
+ refuses to even queue it. A `soft` budget crosses and runs — the ledger
93
+ warns as usual.
94
+
95
+ A bounded run also costs what it costs: the run's usage lands on the same
96
+ `BudgetLedger` the edge enforces, and `billed = total + cached +
97
+ cache_creation` — a long report is cache-heavy, measure the caps against it.
98
+
99
+ ## What a schedule run is
100
+
101
+ A first-class turn, stamped `origin: "scheduled"` so a refinement read can
102
+ never mistake the engine's kick for a customer speaking. It enters through the
103
+ pipeline directly (never through the message edge, so no rate-limit/token
104
+ ceiling gate applies — same as a resume), is charged to the ledger like any
105
+ turn, and its result is delivered wherever the agent's outputs go: a channel
106
+ answer, or – for the report shape – nothing at all, when the run publishes an
107
+ artifact instead. The Studio's schedule card links the last run's task.
108
+
109
+ No customer ever receives anything from a schedule unless your agent sends a
110
+ message in reply — the engine contacts no one.
111
+
112
+ ## The boundaries
113
+
114
+ - **Not a job queue.** No priorities, no fan-out, no retries of a failed run
115
+ beyond what Recovery already does for any task.
116
+ - **Not the follow-up feature.** `schedule_followup` is a one-shot,
117
+ customer-facing, consent-gated contact from inside a conversation — and it
118
+ keeps being that. These schedules are operator-declared recurring internal
119
+ triggers: no contact policy, no consent, no customer.
120
+ - **Not a replacement for your cron.** The external route stays; this is the
121
+ built-in one.
data/docs/SECURITY.md CHANGED
@@ -171,7 +171,7 @@ LLM calls**. A *resumed* turn is never re-counted.
171
171
  > `0` to explicitly disable. A malformed value in the Studio raises a validation
172
172
  > error rather than silently disabling a production limit.
173
173
 
174
- See [Agents §Layer 4](AGENTS.md#layer-4-edge-limits-flood-and-spend-control).
174
+ See [Agents §Layer 4](POLICY.md#layer-4-edge-limits-flood-and-spend-control).
175
175
 
176
176
  ## Guardrails
177
177
 
@@ -203,7 +203,7 @@ Strictness selects the detector categories (`low` = injection only; `medium`
203
203
  (default) and `high` add sexual and abuse). Safe-reply lookup falls back per
204
204
  category: the agent's category reply → the agent's default → the builtin
205
205
  category → the builtin default. All of it is editable in the Studio Configuration
206
- form. See [Agents §Layer 3](AGENTS.md#layer-3-guardrails-content-safety).
206
+ form. See [Agents §Layer 3](POLICY.md#layer-3-guardrails-content-safety).
207
207
 
208
208
  ## Human approval
209
209
 
@@ -367,7 +367,7 @@ rotated key or a typo never takes the whole service down. The `insika doctor`
367
367
  command runs the same checks on demand against a live database. See
368
368
  [Deploy](DEPLOY.md#strict-config-and-insika-doctor).
369
369
 
370
- ## Memory and the right to be forgotten (LGPD, RFC-0031)
370
+ ## Memory and the right to be forgotten (LGPD)
371
371
 
372
372
  Memory is scoped per **`(tenant, customer)`** — the cell `"memory:<tenant>:<customer>"`
373
373
  is the isolation boundary. A query against one tenant never touches another's
@@ -394,7 +394,7 @@ forgotten or aged out. Three operations enforce the right to be forgotten:
394
394
  outbox deliveries). The operator-mutation audit store records a digest-free line
395
395
  ("a purge happened, with N records") — content-free by construction.
396
396
 
397
- ### Distilled facts are personal data (RFC-0034)
397
+ ### Distilled facts are personal data
398
398
 
399
399
  The distillation loop ([Facts](FACTS.md)) writes **proposals** — a distilled
400
400
  fact's name and value are personal data, and they are treated like the rest of
@@ -402,7 +402,7 @@ the memory footprint: `forget_customer` deletes the person's proposals (every
402
402
  status), `delete_tenant_data` deletes a tenant's, and the `retention_days`
403
403
  sweep ages them out with the transcripts they were distilled from. Provenance
404
404
  holds: an approved fact is written with `origin: "distilled:<session_ref>"`
405
- (the RFC-0031 closed set gains one spelling, never an open string). Events and
405
+ (the closed set gains one spelling, never an open string). Events and
406
406
  audit carry ids and counts only — a fact value never enters the stream, the
407
407
  ledger or a log; the evidence excerpt is a link read from the transcript at
408
408
  request time, never a copy.
@@ -412,7 +412,7 @@ deployment (in `single_tenant` the bare cell is the designed customer shape), an
412
412
  never the session-marked cells. See [Context](CONTEXT.md#memory) and
413
413
  [Deploy](DEPLOY.md#strict-config-and-insika-doctor).
414
414
 
415
- ### Harvest candidates are derived data (RFC-0035)
415
+ ### Harvest candidates are derived data
416
416
 
417
417
  The harvest loop ([Harvest](HARVEST.md)) writes **candidates** — a mined skill
418
418
  proposal is behavior instructions, the same trust level as any skill content,
@@ -428,6 +428,22 @@ product claim the origin sessions' evidence ledger did not see. Events and the
428
428
  promotion log carry ids, refs and verdicts only — a skill body never enters
429
429
  the stream.
430
430
 
431
+ ### Learned knowledge is redacted before it is stored
432
+
433
+ The [Knowledge](KNOWLEDGE.md) loop writes **concepts** extracted from finished
434
+ conversations — store-scoped, not customer-scoped, but written from real
435
+ transcript text, so the write path is deliberately conservative: a concept's
436
+ body goes through the same PII/secret redactor the output guardrail uses
437
+ before it is ever persisted, and `sources` holds session ids only — never
438
+ message content, never a customer identifier. This is the one write path in
439
+ the engine where a model-authored field is rejected outright rather than
440
+ merely validated: `provenance`, `confidence`, `sources` and the timestamps are
441
+ stamped by the engine, so a model cannot self-assign trust it did not earn or
442
+ smuggle a scope into a store-wide record. A repeat sighting that contradicts
443
+ what's on record is never silently merged — the conservative default when
444
+ the engine cannot tell is to flag it for a human, not to guess. Events carry
445
+ names and counts only — a concept's content never enters the stream.
446
+
431
447
  ## See also
432
448
 
433
449
  - [Agents](AGENTS.md) — the five access layers per agent.
data/docs/SKILLS.md CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  title: Skills
3
- parent: Build an agent
4
- nav_order: 3
3
+ parent: Core concepts
4
+ nav_order: 4
5
5
  permalink: /skills/
6
6
  ---
7
7
 
@@ -16,6 +16,14 @@ while paying for the text of only the ones it opens.
16
16
 
17
17
  See [`examples/skills/`](https://github.com/guizaols/insika/tree/main/examples/skills/) for a runnable one.
18
18
 
19
+ **Skills vs. Knowledge.** They share a format — YAML frontmatter over a
20
+ Markdown body, progressive loading — which makes them easy to confuse. A
21
+ skill is **curated**: a human writes it, and it is canonical until a human
22
+ changes it. A [Knowledge](KNOWLEDGE.md) concept is **learned**: the engine
23
+ extracts it from finished conversations, it is `provenance: observed`, and it
24
+ sits below skills in the context priority ladder — earned trust, not
25
+ authored trust.
26
+
19
27
  ## Format
20
28
 
21
29
  ```markdown
@@ -281,4 +289,5 @@ reading the doctor:
281
289
  - [Agents](AGENTS.md) — the skills allowlist.
282
290
  - [Tools](TOOLS.md) — `load_skill` and deferred-tool progressive disclosure.
283
291
  - [Plugins](PLUGINS.md) — shipping skills inside a plugin, and the two extension tiers.
292
+ - [Knowledge](KNOWLEDGE.md) — the learned counterpart: engine-extracted concepts, same format, earned trust.
284
293
  - [`examples/skills/`](https://github.com/guizaols/insika/tree/main/examples/skills/) — progressive loading, runnable.
data/docs/SOAK.md CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  title: Soak
3
- parent: Operate & prove it
4
- nav_order: 4
3
+ parent: Operate
4
+ nav_order: 5
5
5
  permalink: /soak/
6
6
  ---
7
7
 
data/docs/TEMPLATES.md ADDED
@@ -0,0 +1,134 @@
1
+ ---
2
+ title: Templates
3
+ parent: Integrate
4
+ nav_order: 6
5
+ permalink: /templates/
6
+ ---
7
+
8
+ # Templates
9
+
10
+ Example agents shipped **inside the gem** — `lib/insika/templates/<name>/`,
11
+ one DSL file per template. `insika new <name>` copies it for you to run and
12
+ edit; the same file is what the Studio gallery evaluates to create the
13
+ agent from a click. One source of truth, two doors — never a parallel pack
14
+ format to drift.
15
+
16
+ ## The gallery
17
+
18
+ ```bash
19
+ insika new --list
20
+ ```
21
+
22
+ ```
23
+ travel-planner Starter Weather + currency data-tools against keyless public APIs …
24
+ research-analyst Advanced Insika.system fan-out — three specialist subagents research …
25
+ daily-digest Always-on A recurring schedule plus save_artifact build and publish …
26
+ review-panel Teams Two specialists reviewed in parallel by a synthesizing lead …
27
+ repo-explorer MCP Live MCP tool-loop over http — answers questions about any …
28
+ browser-agent MCP Live MCP tool-loop over stdio — navigates and summarizes …
29
+ ```
30
+
31
+ ```bash
32
+ insika new travel-planner # copies ./travel-planner/{agent.rb,README.md}
33
+ insika new travel-planner my-trip # ...into ./my-trip/ instead
34
+ ```
35
+
36
+ The CLI prints the exact run line, including any env the template needs
37
+ **set** (not just available as an override) — a stdio MCP template needs
38
+ `INSIKA_MCP_STDIO=1`, for instance. The generated script *is* the editing
39
+ surface: no Gemfile, no questionnaire, no placeholders to fill in.
40
+
41
+ The same roster appears as a "+ from template" gallery on the Studio
42
+ `/studio/agents` page — clicking **Create** dispatches the identical
43
+ `:create_agent` (and, for a system template, one per agent) the CLI-run
44
+ copy would produce. A template marked `studio: false` in its frontmatter
45
+ (none in wave 1) shows a "CLI-only for now" note instead of a button —
46
+ reserved for a template whose value is a durable workflow, until workflow
47
+ import into a running store exists.
48
+
49
+ ## The MCP trail: point it at your own server
50
+
51
+ `repo-explorer` (http) and `browser-agent` (stdio) are not showcases for
52
+ one MCP vendor — they demonstrate exactly how to plug **any** MCP server
53
+ into an agent. Each ships with a working, keyless default so
54
+ `insika new` + the run line works with zero setup, but the server is a
55
+ config value:
56
+
57
+ ```bash
58
+ MCP_URL=https://your-mcp-server/mcp DEEPSEEK_API_KEY=sk-... ruby repo-explorer/agent.rb "..."
59
+ MCP_COMMAND=your-mcp-server INSIKA_MCP_STDIO=1 DEEPSEEK_API_KEY=sk-... ruby browser-agent/agent.rb "..."
60
+ ```
61
+
62
+ Swap the env var, rewrite the instructions for the new server's tools —
63
+ nothing else in `agent.rb` changes.
64
+
65
+ ## Writing a template
66
+
67
+ A template is `lib/insika/templates/<name>/agent.rb` + `README.md`.
68
+
69
+ **The frontmatter contract** — a `# ---` … `# ---` comment block, YAML
70
+ inside, right after the standard `# frozen_string_literal: true` (that
71
+ magic comment is skipped automatically — a template doesn't have to break
72
+ the convention every other file in the gem follows):
73
+
74
+ ```ruby
75
+ # frozen_string_literal: true
76
+
77
+ # ---
78
+ # title: My Template
79
+ # trail: Starter | Advanced | Always-on | Teams | MCP
80
+ # description: one line, shown in the CLI list and the Studio card.
81
+ # capabilities: comma, separated, tags
82
+ # studio: true # optional, default true
83
+ # env: SOME_REQUIRED_VAR # optional — env the run line must SET, not just may override
84
+ # requires: Node.js and npm # optional — a local dependency beyond the gem + a provider key
85
+ # ---
86
+ ```
87
+
88
+ **The two-doors mechanics**, in the file itself:
89
+
90
+ 1. `require "insika"` — gem-style, never `require_relative` (the file gets
91
+ copied out of the gem into an arbitrary directory).
92
+ 2. Build the agent/system as a normal top-level local: `travel = Insika.agent(...) { ... }`.
93
+ 3. Guard the CLI demo footer: `if __FILE__ == $PROGRAM_NAME ... end`. False
94
+ whenever `Insika::Templates.evaluate` loads the file (never true from
95
+ inside the gem/Studio process), so the Studio door never makes a network
96
+ call, prints anything, or parses `ARGV`.
97
+ 4. End the file with the **bare** built value (`travel`, `panel`, `team`,
98
+ …) as its last expression — `evaluate` runs the file in an isolated
99
+ `instance_eval` and returns whatever that last expression is. No
100
+ registration call, no second format.
101
+ 5. **No top-level constants.** `instance_eval`'s isolation keeps local
102
+ variables and `def`s from leaking into the NEXT template evaluated in
103
+ the same process, but Ruby scopes a `CONST = ...` assignment lexically,
104
+ not by `self` — it would leak. Use a local variable (closures see it
105
+ fine from inside a `do...end` block) — every wave-1 template does.
106
+
107
+ **Engine-only rules** (enforced by the lint below):
108
+ provider-agnostic (one provider key), zero tenant/store data, external
109
+ calls only to keyless public APIs, every tool/mcp group covered by an
110
+ explicit allowlist.
111
+
112
+ **Declaring an `mcp` server** auto-adds `"mcp:<name>"` to that agent's
113
+ `tools_allow_groups` (`Insika::DSL::Builder#mcp`) — without it the agent
114
+ could never call the MCP tool it just declared, since `PackImporter`
115
+ forces `tools_allow: []` for a pack with no `data_tool`. A **system**-level
116
+ `mcp` (declared outside any member `agent { }` block) grants no agent
117
+ access by itself — declare it inside the specific agent that needs it.
118
+
119
+ ## The E3 lint
120
+
121
+ `spec/insika/templates_spec.rb` iterates `Insika::Templates.all` for real —
122
+ one example per template name, so a broken new template fails by name, not
123
+ a generic loop assertion. It checks, per template:
124
+
125
+ - evaluates cleanly to schema-valid pack(s) (`id`/`model` present);
126
+ - every `data_tool` it declares is in that SAME pack's `tools_allow`;
127
+ - every `mcp` instance's group is granted by SOME agent in the pack(s);
128
+ - every referenced host (`data_tool` URL, http/sse `mcp` URL) passes
129
+ `Insika::EgressGuard.violation` — public HTTPS only, same guard a live
130
+ turn would apply;
131
+ - no hardcoded secret-shaped literal (`sk-...`, a long `Bearer ...` token)
132
+ in the source.
133
+
134
+ Run it before adding a template: `bundle exec rspec spec/insika/templates_spec.rb`.
data/docs/TOOLS.md CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  title: Tools
3
- parent: Build an agent
4
- nav_order: 2
3
+ parent: Core concepts
4
+ nav_order: 3
5
5
  permalink: /tools/
6
6
  ---
7
7
 
@@ -12,15 +12,16 @@ kinds, and the distinction that matters is **who can change one at runtime**:
12
12
 
13
13
  | | **Code tool** | **Data tool** | **MCP tool** |
14
14
  |---|---|---|---|
15
- | What | a Ruby class (`< RubyLLM::Tool`) | an HTTP call described by config, no Ruby | an MCP server's tool, ingested |
16
- | Lives | in the deployment image | as a row in SQLite | as data-tool rows in SQLite |
17
- | Editable at runtime | no (shipped in the image) | **yes** (DSL / API / manifest / Studio) | **yes** (re-ingest) |
18
- | Reach for it when | logic must run in-process (file edit, shell, subagent) | calling an external HTTP API | adopting a whole MCP toolset at once |
15
+ | What | a Ruby class (`< RubyLLM::Tool`) | an HTTP call described by config, no Ruby | an MCP server's tool, called LIVE |
16
+ | Lives | in the deployment image | as a row in SQLite | on the MCP server, behind a live client |
17
+ | Editable at runtime | no (shipped in the image) | **yes** (DSL / API / manifest / Studio) | **yes** — enable/edit the *instance* (DSL / CLI / API / JSON import / Studio); the server owns its own tools |
18
+ | Reach for it when | logic must run in-process (file edit, shell, subagent) | calling an external HTTP API | adopting a whole external MCP server's toolset |
19
19
 
20
- **MCP tools are not a separate runtime type.** An MCP ingestor discovers an MCP
21
- server's tools and turns each into an HTTP **data tool** that posts a JSON-RPC
22
- `tools/call`, tagged with a `group` naming the source instance. (Only
23
- HTTP-transport MCP servers are ingestible; stdio is rejected.)
20
+ **MCP tools are not data tools.** Configuring an enabled MCP **instance** (any
21
+ surface below) is enough its tools appear automatically, tagged
22
+ `mcp:<instance>`, and each CALL goes straight to the server through a live,
23
+ held client (stdio process / Streamable HTTP / SSE, with the full protocol
24
+ handshake) — never a frozen snapshot. See [MCP servers](#mcp-servers) below.
24
25
 
25
26
  Code tools **win name collisions** — you cannot register a data tool whose name
26
27
  shadows a code tool.
@@ -49,6 +50,11 @@ the one you create and change without a rebuild. See
49
50
  }
50
51
  ```
51
52
 
53
+ `{{secret.api_token}}` above is only real coming through the **manifest**
54
+ write path (`POST /v1/tools/manifest`) — writing this same shape via the DSL
55
+ or Studio needs the literal header value instead; see
56
+ "[The one gotcha](#the-one-gotcha-envsecret-templating-is-manifest-only)" below.
57
+
52
58
  ### Parameters: the schema is the contract
53
59
 
54
60
  `parameters` is **JSON Schema**, and it reaches the provider verbatim — it is the only
@@ -87,17 +93,25 @@ allow never becomes a request: it returns an `{ error: … }` naming the path
87
93
  retries against. Structure is strict; a scalar may arrive in its lossless string form
88
94
  (`"2"`, `"true"`) and is never coerced — what the model sent is what the request carries.
89
95
 
90
- **Placeholders** are resolved at turn time:
96
+ **Placeholders**. Two of these resolve at turn time; `{{secret.*}}` resolves
97
+ once, at ingestion — see the gotcha below before reaching for it:
91
98
 
92
- - `{{param}}` — a declared top-level parameter, filled from the model's call.
99
+ - `{{param}}` — a declared top-level parameter, filled from the model's call,
100
+ every turn.
93
101
  - `{{ctx.*}}` — turn context set **server-side, never by the model**: a closed set
94
102
  of `chat_id`, `store_id`, `agent_id`, `tenant`, `image_url`. This is how a tool knows *which*
95
103
  session/agent it is acting for without trusting the model. `image_url` is the
96
104
  first image part on the message (a photo for analysis outside the prompt);
97
- absent when the turn carried none.
98
- - `{{secret.*}}` — allowed **only** inside a header named in `secret_headers`.
99
- A secret placeholder anywhere else is rejected (it would leak unmasked). The
100
- real secret value is injected at provision time and never lives on disk.
105
+ absent when the turn carried none. Resolved every turn, like `{{param}}`.
106
+ - `{{secret.*}}` — **only resolved on the manifest ingestion path**
107
+ (`POST /v1/tools/manifest`; see "[The one gotcha](#the-one-gotcha-envsecret-templating-is-manifest-only)"
108
+ below), and only once the resolved value is what gets stored, the token
109
+ itself never lives on disk and is never re-read per turn. Allowed **only**
110
+ inside a header named in `secret_headers`. Written any other way — DSL,
111
+ Studio, or anywhere outside a `secret_headers` header — a
112
+ `{{secret.*}}` is not a credential the engine knows how to fill; it is an
113
+ undeclared parameter, and tool registration refuses it exactly like it
114
+ refuses any other unknown placeholder.
101
115
 
102
116
  **Validation** happens on ingestion. Common rejections:
103
117
 
@@ -183,7 +197,7 @@ backend, not of whoever calls it. Every agent sharing the tool gets the same val
183
197
  > there **preserves** it — the form carries the stored values through instead of
184
198
  > replacing the record with only what it shows.
185
199
 
186
- ## Evidence: the lean envelope and grounding (RFC-0029)
200
+ ## Evidence: the lean envelope and grounding
187
201
 
188
202
  A catalog tool returns products; the model should only ever quote the ones the tool
189
203
  actually returned — the store dies of a SKU the model invented. `evidence` is the
@@ -261,16 +275,117 @@ reload, no restart):
261
275
  3. **Manifest** — `POST /v1/tools/manifest`. Partial failure is isolated: one
262
276
  malformed tool becomes an `errors[]` entry; only a structural manifest error
263
277
  fails the whole request. The response reports `{ version, created, updated, errors }`.
264
- 4. **MCP ingestion**import a server; each of its tools becomes a data tool.
265
-
266
- ### The one gotcha: env templating is manifest-only
267
-
268
- `{{env.*}}` (and `{{secret.*}}`) are substituted **at ingestion, on the manifest
269
- path**. Other write paths do **not** resolve `{{env.*}}` a literal
270
- `{{env.API_URL}}` there fails the `http`/`https` URL check and 422s. Rule:
271
- **manifest tools may template the URL with `{{env.*}}`; tools written any other
272
- way must ship a literal URL.** `{{ctx.*}}` and `{{param}}` work everywhere (they
273
- resolve at turn time, not ingestion).
278
+ 4. ~~MCP ingestion~~retired. An MCP server's tools are no
279
+ longer written into this store at all; see [MCP servers](#mcp-servers).
280
+
281
+ ### The one gotcha: env/secret templating is manifest-only
282
+
283
+ `{{env.*}}` and `{{secret.*}}` are substituted **at ingestion, on the manifest
284
+ path, once** the resolved literal is what gets stored; the token itself
285
+ never survives to a turn. Other write paths (DSL, Studio) do
286
+ **not** resolve either: a literal `{{env.API_URL}}` in a URL fails the
287
+ `http`/`https` check and 422s; a literal `{{secret.X}}` anywhere — including
288
+ inside a header named in `secret_headers` — fails tool registration the same
289
+ way an unknown parameter would (`ToolDefinition.build`'s placeholder check
290
+ does not special-case it). Rule: **manifest tools may template a URL with
291
+ `{{env.*}}` and a `secret_headers` header with `{{secret.*}}`; tools written
292
+ any other way must ship literal values** — a real URL, and a real (masked on
293
+ read) header value. `{{ctx.*}}` and `{{param}}` work everywhere (they resolve
294
+ at turn time, not ingestion).
295
+
296
+ ## MCP servers
297
+
298
+ An MCP **instance** is durable config — transport, target, credentials, an
299
+ `enabled` flag — held in its own store, separate from data tools. Once an
300
+ instance is enabled, its tools appear in the catalog automatically (group
301
+ `mcp:<instance>`, `side_effect: true`), and every call goes straight to the
302
+ server through a live, held client — the runtime never converts an MCP tool
303
+ into a stored data tool.
304
+
305
+ **Three transports**, picked by `transport:`:
306
+
307
+ | Transport | Target | Notes |
308
+ |---|---|---|
309
+ | `stdio` | `command` + `args`, run as a child process, `env` is its process environment | requires `INSIKA_MCP_STDIO=1` — see below |
310
+ | `http` | `url` + `headers` (Streamable HTTP, the modern default) | egress-guarded like any outbound URL |
311
+ | `sse` | `url` + `headers` | same egress guard as `http` |
312
+
313
+ **The stdio gate.** A stdio instance is arbitrary command execution by
314
+ config — it saves, but refuses to start ("stdio disabled by env") until the
315
+ operator sets `INSIKA_MCP_STDIO=1` (config-over-convention, the same pattern
316
+ as the egress envs). `http`/`sse` need no such gate; their URL is checked by
317
+ the normal egress allowlist instead.
318
+
319
+ **Credentials are never visible in plaintext.** `env` (stdio) and `headers`
320
+ (http/sse) mask every value as `__OCULTO__` on read, everywhere (CLI, API,
321
+ Studio). On write, sending the sentinel back **preserves** the stored value; a
322
+ new string **replaces** it; `""` (or omitting the key) **clears** it — the
323
+ same per-key reconciliation `llm_providers` api keys use.
324
+
325
+ **Discovery vs execution.** `insika mcp refresh <name>` (or `POST
326
+ /v1/mcp/:name/import`, kept as that action's route since before the live
327
+ registry) connects live, lists the server's tools, and caches the result
328
+ (`tools_cache`) purely for display — the Studio panel and `insika doctor`.
329
+ **Execution never reads that cache**: a live turn always goes through the
330
+ held client, which does its own discovery on first use regardless of whether
331
+ `refresh` ever ran.
332
+
333
+ ### Configuring an instance
334
+
335
+ 1. **DSL** — inside `Insika.system { … }` or a single `Insika.agent { … }`:
336
+
337
+ ```ruby
338
+ mcp "tavily", transport: :http, url: "https://mcp.tavily.com/mcp",
339
+ headers: { "Authorization" => "Bearer #{ENV['TAVILY_KEY']}" }
340
+ mcp "filesystem", transport: :stdio, command: "npx",
341
+ args: ["-y", "@modelcontextprotocol/server-filesystem", "/tmp"]
342
+ ```
343
+
344
+ Code is the **template**: transport/command/args/url/description always
345
+ follow the declaration on every boot. But once the instance exists, its
346
+ `enabled` flag and its credentials are the **operator's** — a Studio/CLI/API
347
+ edit made after boot is never clobbered back by the next restart.
348
+
349
+ 2. **CLI** — `insika mcp list | add | remove | import <file.json> | test <name> |
350
+ refresh <name>`. `add` takes `--name`, `--transport`, `--command`/`--arg`
351
+ (repeatable) or `--url`/`--header "Name: value"` (repeatable)/`--env
352
+ "KEY=value"` (repeatable), `--description`, `--disabled`. `test` connects
353
+ live and prints the discovered tools (or the error) without any special
354
+ setup; `refresh` does the same and additionally updates `tools_cache`.
355
+
356
+ 3. **JSON import/export** — the same `mcpServers` shape every MCP client
357
+ (Claude Desktop, Cursor, …) already uses:
358
+
359
+ ```jsonc
360
+ {
361
+ "mcpServers": {
362
+ "tavily": { "url": "https://mcp.tavily.com/mcp", "headers": { "Authorization": "Bearer …" } },
363
+ "filesystem": { "command": "npx", "args": ["-y", "@modelcontextprotocol/server-filesystem", "/tmp"] }
364
+ }
365
+ }
366
+ ```
367
+
368
+ `insika mcp import FILE.json` upserts every entry (a bare `command` implies
369
+ `stdio`; a bare `url` implies `http`; add `"transport": "sse"` explicitly
370
+ for SSE — the bare format has no other way to spell it). The same parser
371
+ backs `PUT /v1/mcp` and Studio's "Import JSON" box; `export` produces the
372
+ document back with secrets masked as `__OCULTO__`, so round-tripping an
373
+ export never wipes a stored credential.
374
+
375
+ 4. **HTTP API** (operator-only, gateway Bearer):
376
+ - `GET /v1/mcp` — every instance, masked.
377
+ - `GET /v1/mcp/:name` — one instance, masked.
378
+ - `PUT /v1/mcp` — upsert (body = the instance attrs, `name` required).
379
+ - `DELETE /v1/mcp/:name` — remove (idempotent).
380
+ - `POST /v1/mcp/:name/import` — refresh (connect live, list tools, cache).
381
+
382
+ 5. **Studio** — the `/studio/mcp` panel (create/edit/delete). The form is
383
+ transport-aware (stdio shows command/args/env, http/sse shows
384
+ url/headers); each instance shows a status chip ("N tool(s)", "untested",
385
+ "stdio disabled", or "off") and its discovered tools from `tools_cache`; a
386
+ "Test connection" button dispatches the same `refresh_mcp_tools` seam as
387
+ `insika mcp test`; an "Import JSON" box takes a `mcpServers` document and
388
+ fans it out into one `upsert_mcp` per entry.
274
389
 
275
390
  ## Making it appear — and enter the tool-loop
276
391
 
@@ -361,9 +476,19 @@ Work down this checklist:
361
476
  4. **URL literal?** For non-manifest tools, an unresolved `{{env.*}}` would have
362
477
  422'd at import — re-check the definition.
363
478
 
479
+ ## The `save_artifact` built-in
480
+
481
+ `save_artifact` is a **registry tool** — it obeys the same per-agent
482
+ `tools_allow` as any data tool, and an agent that did not name it cannot call
483
+ it (`tools_allow: %w[save_artifact]`). The agent hands in `title` + `content`
484
+ and gets the URL back; the tenant is bound from the turn, never a parameter the
485
+ model types. See [Artifacts](ARTIFACTS.md) for the tool contract, the serving
486
+ routes, the signed link and the retention/LGPD reach.
487
+
364
488
  ## See also
365
489
 
366
490
  - [Agents](AGENTS.md) — allowlists, groups, and per-agent tool exposure.
491
+ - [Artifacts](ARTIFACTS.md) — the report destination: the tool, the routes, the signed link.
367
492
  - [Plugins](PLUGINS.md) — where a code tool comes from, and how to package one.
368
493
  - [Security](SECURITY.md) — egress, sandbox, and approval gating together.
369
494
  - [Architecture](ARCHITECTURE.md) — the tool-loop and side-effect checkpointing.
data/docs/WHY.md CHANGED
@@ -1,6 +1,6 @@
1
1
  ---
2
2
  title: Why Insika
3
- parent: Understand the idea
3
+ parent: Start here
4
4
  nav_order: 1
5
5
  permalink: /why/
6
6
  ---
data/docs/WORKFLOWS.md CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  title: Workflows
3
- parent: Build an agent
4
- nav_order: 5
3
+ parent: Core concepts
4
+ nav_order: 6
5
5
  permalink: /workflows/
6
6
  ---
7
7
 
@@ -0,0 +1,5 @@
1
+ {%- comment -%}
2
+ Appended inside <head> on every page by the theme. The theme's own favicon
3
+ include only looks for a legacy /favicon.ico, so the SVG icon is declared here.
4
+ {%- endcomment -%}
5
+ <link rel="icon" href="{{ '/assets/img/favicon.svg' | relative_url }}" type="image/svg+xml">
@@ -0,0 +1,13 @@
1
+ {%- comment -%}
2
+ Overrides the theme's title.html so the sidebar shows the pillar mark next to
3
+ the wordmark. The theme's own `site.logo` path swaps the title for a single
4
+ background image, which would drop the text — and the text is what the
5
+ browser tab, the skip link and screen readers rely on.
6
+ {%- endcomment -%}
7
+ <span class="site-title-mark" aria-hidden="true">
8
+ <svg viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" focusable="false">
9
+ <rect x="3" y="3" width="18" height="3.6" rx="1"/>
10
+ <rect x="8.7" y="7.6" width="6.6" height="8.8" class="shaft"/>
11
+ <rect x="3" y="17.4" width="18" height="3.6" rx="1"/>
12
+ </svg>
13
+ </span>{{ site.title }}