insika 0.3.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (190) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +180 -0
  3. data/README.md +45 -10
  4. data/bin/insika +684 -0
  5. data/bin/insika-router +87 -0
  6. data/docs/AGENTS.md +94 -403
  7. data/docs/API.md +5 -5
  8. data/docs/ARCHITECTURE.md +3 -2
  9. data/docs/ARTIFACTS.md +95 -0
  10. data/docs/BENCHMARK.md +2 -2
  11. data/docs/CHANNELS.md +14 -14
  12. data/docs/CONTEXT.md +9 -7
  13. data/docs/DEMO.md +80 -0
  14. data/docs/DEPLOY.md +71 -3
  15. data/docs/EMBEDDING.md +1 -1
  16. data/docs/EVALS.md +128 -3
  17. data/docs/FACTS.md +3 -3
  18. data/docs/HARVEST.md +5 -6
  19. data/docs/KNOWLEDGE.md +290 -0
  20. data/docs/LOADTEST.md +2 -2
  21. data/docs/MEDIA.md +128 -0
  22. data/docs/OBSERVABILITY.md +15 -10
  23. data/docs/OUTCOMES.md +137 -0
  24. data/docs/PLUGINS.md +51 -6
  25. data/docs/POLICY.md +216 -0
  26. data/docs/REFINEMENT.md +14 -9
  27. data/docs/RELEASING.md +4 -4
  28. data/docs/ROUTER.md +213 -0
  29. data/docs/RUNNING-LOCAL.md +3 -3
  30. data/docs/SCHEDULING.md +121 -0
  31. data/docs/SECURITY.md +22 -6
  32. data/docs/SKILLS.md +11 -2
  33. data/docs/SOAK.md +2 -2
  34. data/docs/TEMPLATES.md +134 -0
  35. data/docs/TOOLS.md +152 -27
  36. data/docs/WHY.md +1 -1
  37. data/docs/WORKFLOWS.md +2 -2
  38. data/docs/_includes/head_custom.html +5 -0
  39. data/docs/_includes/title.html +13 -0
  40. data/docs/_sass/color_schemes/insika.scss +32 -0
  41. data/docs/_sass/custom/custom.scss +199 -0
  42. data/docs/_sass/custom/setup.scss +26 -0
  43. data/docs/assets/img/favicon.svg +7 -0
  44. data/docs/assets/img/insika-mark.svg +7 -0
  45. data/docs/core-concepts.md +21 -0
  46. data/docs/domain.md +4 -4
  47. data/docs/improve.md +20 -0
  48. data/docs/index.md +8 -5
  49. data/docs/integrate.md +20 -0
  50. data/docs/operate.md +13 -6
  51. data/docs/prompts/ADD-TOOL.md +118 -0
  52. data/docs/prompts/DIAGNOSE-TURN.md +65 -0
  53. data/docs/prompts/GO-LIVE.md +138 -0
  54. data/docs/prompts/RUN-EXAMPLES.md +70 -0
  55. data/docs/reference.md +19 -0
  56. data/docs/ship.md +10 -2
  57. data/docs/start-here.md +18 -0
  58. data/lib/insika/agent_profile.rb +73 -16
  59. data/lib/insika/artifact_signing.rb +82 -0
  60. data/lib/insika/artifact_store.rb +160 -0
  61. data/lib/insika/channel_delivery.rb +1 -1
  62. data/lib/insika/chat_builder.rb +22 -2
  63. data/lib/insika/commands/agent_payload.rb +2 -2
  64. data/lib/insika/commands/backfill_knowledge.rb +145 -0
  65. data/lib/insika/commands/delete_artifact.rb +35 -0
  66. data/lib/insika/commands/delete_concept.rb +34 -0
  67. data/lib/insika/commands/delete_mcp.rb +6 -2
  68. data/lib/insika/commands/delete_tenant_data.rb +15 -3
  69. data/lib/insika/commands/gate_refinement.rb +1 -1
  70. data/lib/insika/commands/refresh_mcp_tools.rb +47 -0
  71. data/lib/insika/commands/restore_concept.rb +34 -0
  72. data/lib/insika/commands/seed_demo_data.rb +31 -0
  73. data/lib/insika/commands/upsert_mcp.rb +6 -3
  74. data/lib/insika/commands/write_concept.rb +57 -0
  75. data/lib/insika/context/priority.rb +2 -0
  76. data/lib/insika/context/providers/knowledge.rb +108 -0
  77. data/lib/insika/context/providers/prompt.rb +30 -24
  78. data/lib/insika/cron.rb +189 -0
  79. data/lib/insika/demo/agent_attrs.rb +43 -0
  80. data/lib/insika/demo/golden_cases.rb +81 -0
  81. data/lib/insika/demo/seeder.rb +336 -0
  82. data/lib/insika/doctor.rb +176 -8
  83. data/lib/insika/dsl/definition.rb +3 -2
  84. data/lib/insika/dsl/runtime.rb +60 -79
  85. data/lib/insika/dsl/server_boot.rb +23 -1
  86. data/lib/insika/dsl/system.rb +10 -2
  87. data/lib/insika/dsl.rb +103 -2
  88. data/lib/insika/env_schema.rb +16 -1
  89. data/lib/insika/evals/golden.rb +41 -4
  90. data/lib/insika/evals/judge.rb +47 -2
  91. data/lib/insika/evals/pairwise.rb +11 -0
  92. data/lib/insika/evals/persona.rb +98 -0
  93. data/lib/insika/evals/runner.rb +9 -0
  94. data/lib/insika/evals/simulator.rb +225 -0
  95. data/lib/insika/evals/transport.rb +83 -1
  96. data/lib/insika/event_stream.rb +10 -0
  97. data/lib/insika/executor.rb +231 -55
  98. data/lib/insika/followup_policy.rb +2 -25
  99. data/lib/insika/golden_store.rb +16 -1
  100. data/lib/insika/grounding/matcher.rb +1 -1
  101. data/lib/insika/knowledge.rb +680 -0
  102. data/lib/insika/knowledge_store.rb +140 -0
  103. data/lib/insika/mcp_client.rb +94 -0
  104. data/lib/insika/mcp_json.rb +74 -0
  105. data/lib/insika/mcp_live_tool.rb +43 -0
  106. data/lib/insika/mcp_store.rb +98 -26
  107. data/lib/insika/mcp_tool_ingestor.rb +30 -8
  108. data/lib/insika/mcp_tool_registry.rb +100 -0
  109. data/lib/insika/media.rb +115 -31
  110. data/lib/insika/message_origin.rb +1 -1
  111. data/lib/insika/middleware.rb +9 -0
  112. data/lib/insika/onboarding.rb +17 -1
  113. data/lib/insika/outcome_store.rb +1 -1
  114. data/lib/insika/overlay_tool_registry.rb +37 -17
  115. data/lib/insika/packaging.rb +2 -2
  116. data/lib/insika/profile_source.rb +8 -1
  117. data/lib/insika/prompt_catalog.rb +10 -0
  118. data/lib/insika/retention.rb +36 -1
  119. data/lib/insika/router/app.rb +157 -0
  120. data/lib/insika/router/backend_pool.rb +98 -0
  121. data/lib/insika/router/hash_ring.rb +55 -0
  122. data/lib/insika/router/proxy_body.rb +34 -0
  123. data/lib/insika/router/session_key.rb +54 -0
  124. data/lib/insika/router.rb +18 -0
  125. data/lib/insika/schedule.rb +177 -0
  126. data/lib/insika/schedule_engine.rb +314 -0
  127. data/lib/insika/schedule_store.rb +208 -0
  128. data/lib/insika/server/app.rb +105 -15
  129. data/lib/insika/server/rack_app.rb +5 -1
  130. data/lib/insika/server/responses.rb +1 -1
  131. data/lib/insika/skill_catalog.rb +12 -0
  132. data/lib/insika/steer_injector.rb +21 -10
  133. data/lib/insika/studio/app.rb +567 -45
  134. data/lib/insika/studio/assets/dist/application.css +1 -1
  135. data/lib/insika/studio/assets/dist/application.js +21 -21
  136. data/lib/insika/studio/forms.rb +46 -5
  137. data/lib/insika/studio/nav_icons.rb +14 -1
  138. data/lib/insika/studio/views/_agent_tab_cache.erb +25 -0
  139. data/lib/insika/studio/views/_agent_tab_config.erb +514 -0
  140. data/lib/insika/studio/views/_agent_tab_history.erb +24 -0
  141. data/lib/insika/studio/views/_agent_tab_loops.erb +54 -0
  142. data/lib/insika/studio/views/_agent_tab_memory.erb +51 -0
  143. data/lib/insika/studio/views/_agent_tab_outcomes.erb +31 -0
  144. data/lib/insika/studio/views/_agent_tab_prompts.erb +108 -0
  145. data/lib/insika/studio/views/_agent_tab_skills.erb +38 -0
  146. data/lib/insika/studio/views/_agents_master.erb +44 -0
  147. data/lib/insika/studio/views/_message.erb +49 -32
  148. data/lib/insika/studio/views/agent_detail.erb +61 -820
  149. data/lib/insika/studio/views/agents.erb +70 -57
  150. data/lib/insika/studio/views/artifact.erb +23 -0
  151. data/lib/insika/studio/views/artifacts.erb +59 -0
  152. data/lib/insika/studio/views/evals.erb +2 -2
  153. data/lib/insika/studio/views/facts.erb +1 -1
  154. data/lib/insika/studio/views/funnel.erb +1 -1
  155. data/lib/insika/studio/views/home.erb +106 -67
  156. data/lib/insika/studio/views/knowledge.erb +123 -0
  157. data/lib/insika/studio/views/layout.erb +14 -11
  158. data/lib/insika/studio/views/mcp.erb +174 -80
  159. data/lib/insika/studio/views/session.erb +231 -177
  160. data/lib/insika/studio/views/settings.erb +39 -1
  161. data/lib/insika/studio/views/skills.erb +1 -1
  162. data/lib/insika/studio/views/tools.erb +24 -9
  163. data/lib/insika/templates/browser-agent/README.md +36 -0
  164. data/lib/insika/templates/browser-agent/agent.rb +49 -0
  165. data/lib/insika/templates/daily-digest/README.md +38 -0
  166. data/lib/insika/templates/daily-digest/agent.rb +77 -0
  167. data/lib/insika/templates/repo-explorer/README.md +36 -0
  168. data/lib/insika/templates/repo-explorer/agent.rb +45 -0
  169. data/lib/insika/templates/research-analyst/README.md +26 -0
  170. data/lib/insika/templates/research-analyst/agent.rb +58 -0
  171. data/lib/insika/templates/review-panel/README.md +20 -0
  172. data/lib/insika/templates/review-panel/agent.rb +50 -0
  173. data/lib/insika/templates/travel-planner/README.md +35 -0
  174. data/lib/insika/templates/travel-planner/agent.rb +87 -0
  175. data/lib/insika/templates.rb +112 -0
  176. data/lib/insika/tick.rb +24 -12
  177. data/lib/insika/timezone.rb +45 -0
  178. data/lib/insika/tools/generate_image.rb +52 -7
  179. data/lib/insika/tools/load_knowledge.rb +74 -0
  180. data/lib/insika/tools/run_persona_eval.rb +328 -0
  181. data/lib/insika/tools/save_artifact.rb +95 -0
  182. data/lib/insika/turn_output.rb +1 -1
  183. data/lib/insika/turn_state.rb +15 -4
  184. data/lib/insika/version.rb +1 -1
  185. data/lib/insika/wiring/graph.rb +184 -12
  186. data/lib/insika/wiring/graph_chat.rb +102 -0
  187. data/lib/insika.rb +57 -0
  188. metadata +105 -5
  189. data/docs/build.md +0 -14
  190. data/docs/understand.md +0 -10
@@ -0,0 +1,138 @@
1
+ ---
2
+ title: Prompt — go live
3
+ parent: Reference
4
+ nav_order: 5
5
+ permalink: /go-live/
6
+ layout: default
7
+ render_with_liquid: false
8
+ ---
9
+
10
+ # Take this agent to production
11
+
12
+ > **You are a coding agent** (Claude Code, Codex, Cursor, …) reading this because a
13
+ > developer pasted a prompt pointing here — something like *"deploy this"* or *"take
14
+ > it to production"*. Treat this file as a **skill**: follow the steps in order and
15
+ > apply the RULES literally. Production is where shortcuts become incidents.
16
+
17
+ Your job: get **one working local setup** running as **one production instance**,
18
+ verified end to end. The authoritative reference is
19
+ [`docs/DEPLOY.md`](../DEPLOY.md) (served at `GET /docs/deploy.md`); this file is the
20
+ ordered path through it.
21
+
22
+ ## Step 0 — Gather context (silently)
23
+
24
+ - **Repo or gem?** The reference deployment is a checkout of the insika repo
25
+ (`Dockerfile` + `config.ru` + `railway.json` already in it). An adopter's own app
26
+ consumes the gem instead — then the developer's repo needs its own image; the env
27
+ contract below is identical.
28
+ - **Which platform?** Railway is the documented path. Any Docker host works; the
29
+ Kubernetes caveats are in [`docs/DEPLOY.md`](../DEPLOY.md) § Kubernetes.
30
+ - **Does it work locally?** One green `reply()` or `serve` turn first. Do not debug an
31
+ agent and a deployment at the same time.
32
+ - Read [`docs/DEPLOY.md`](../DEPLOY.md) and
33
+ [`docs/SECURITY.md`](../SECURITY.md) before writing anything.
34
+
35
+ ## Step 1 — Mint the two secrets (RULES, not taste)
36
+
37
+ Two tokens, **two different values** — the fallback of one onto the other is a dev
38
+ convenience only:
39
+
40
+ | Token | Gates | Rotating it |
41
+ |---|---|---|
42
+ | `ADMIN_TOKEN` | `/studio` login (the operator — just you) | safe, independent |
43
+ | `OPENCLAW_GATEWAY_TOKEN` | Bearer for `/v1/responses` + `/v1/agents` (your API consumers) | both sides together, same step |
44
+
45
+ Generate each: `ruby -rsecurerandom -e 'puts SecureRandom.hex(24)'`. Set them as
46
+ platform env vars. **Never** write either into a file, a commit, or your own output.
47
+
48
+ ## Step 2 — The non-negotiable env
49
+
50
+ - **`INSIKA_DB` on a mounted volume** (the image defaults to `/data/insika.db` —
51
+ mount a volume at `/data`). No volume = SQLite is ephemeral and recovery resumes
52
+ nothing after a redeploy.
53
+ - **`WEB_CONCURRENCY` stays `1`.** It is a contract input, not a throughput knob:
54
+ N>1 without session-sticky routing in front is a guaranteed cross-session reply
55
+ leak, and `insika doctor` errors on it on Railway. The fix, when throughput is
56
+ actually needed, is [`insika-router`](../ROUTER.md) in front — not a bigger number.
57
+ - **Provider key** (`DEEPSEEK_API_KEY` for the demo provider) — without it the engine
58
+ still boots (`/up` green) but every turn fails until it is configured.
59
+ - **`INSIKA_EGRESS_HOSTS`** = exactly the hosts your data-tools call. A backend on the
60
+ developer's machine gets a public **https tunnel** + its host in this list — never
61
+ `INSIKA_EGRESS_ALLOW_HTTP`/`_ALLOW_PRIVATE` in cloud.
62
+ - On Railway also **`RAILWAY_DEPLOYMENT_DRAINING_SECONDS=30`**: the platform default
63
+ is 0 — SIGKILL right after SIGTERM — which cancels the graceful drain entirely.
64
+
65
+ ## Step 3 — Deploy
66
+
67
+ Railway (repo path — `railway.json` already sets builder, start command, `/up`
68
+ healthcheck, restart policy):
69
+
70
+ 1. Create the project/service from the repo (builder = Dockerfile).
71
+ 2. Mount the volume at `/data`.
72
+ 3. Set the vars from Steps 1–2.
73
+ 4. Deploy; the healthcheck must go green on `/up`.
74
+
75
+ Any Docker host, same contract:
76
+
77
+ ```bash
78
+ docker build -t insika .
79
+ docker run -p 9292:9292 -v insika-data:/data \
80
+ -e DEEPSEEK_API_KEY=... -e ADMIN_TOKEN=... -e OPENCLAW_GATEWAY_TOKEN=... \
81
+ insika
82
+ ```
83
+
84
+ ## Step 4 — Prove it with ONE real turn
85
+
86
+ In order, each with evidence:
87
+
88
+ 1. `curl https://<host>/up` → `{"status":"ok"}`.
89
+ 2. `bin/insika doctor` against the deployed volume (or via the platform's shell) —
90
+ relay its findings verbatim; fix errors before continuing.
91
+ 3. One authenticated turn:
92
+
93
+ ```bash
94
+ curl -N https://<host>/v1/responses \
95
+ -H "Authorization: Bearer $OPENCLAW_GATEWAY_TOKEN" \
96
+ -H "Content-Type: application/json" \
97
+ -d '{"model":"<agent-id>","user":"go-live-check","input":"hello"}'
98
+ ```
99
+
100
+ The reply must be real model output. 401 → token mismatch (Step 1); a provider error
101
+ → key/model id (Step 2); anything else → stop and diagnose with
102
+ [`docs/prompts/DIAGNOSE-TURN.md`](DIAGNOSE-TURN.md) before touching config.
103
+
104
+ 4. Log in to `/studio` with the new `ADMIN_TOKEN` and find the go-live-check session.
105
+
106
+ ## Step 5 — Close the total-loss hole (Litestream)
107
+
108
+ A single volume is the one point of total loss. Enable continuous replication by env
109
+ (off by default, zero code): set `LITESTREAM_REPLICA_URL` + credentials per
110
+ [`docs/DEPLOY.md`](../DEPLOY.md) § Backup / DR. Then **run the restore drill** — an
111
+ untested backup does not count:
112
+
113
+ ```bash
114
+ scripts/litestream-restore-drill.sh # local proof of the mechanism, or the
115
+ # production drill in DEPLOY.md § Restore drill
116
+ ```
117
+
118
+ If the developer declines Litestream, record that as an explicit accepted risk in
119
+ your report — do not silently skip it.
120
+
121
+ ## Step 6 — Self-check
122
+
123
+ - [ ] `/up` green, `doctor` clean, one real authenticated turn with model output.
124
+ - [ ] Two distinct tokens, both only in platform env; nothing secret in git or logs.
125
+ - [ ] Volume mounted; `WEB_CONCURRENCY=1`; drain buffer set (Railway).
126
+ - [ ] Egress allowlist names only the hosts the tools actually call.
127
+ - [ ] Litestream on **and** a restore exercised — or the risk explicitly accepted.
128
+
129
+ ## Hard constraints
130
+
131
+ - **Never raise `WEB_CONCURRENCY` to "fix" throughput.** The failure it causes is a
132
+ reply delivered to the wrong customer — read
133
+ [`docs/DEPLOY.md`](../DEPLOY.md) § The process model before proposing any scaling.
134
+ - **`_ALLOW_HTTP`/`_ALLOW_PRIVATE` never in cloud.** They exist for fully-local loops.
135
+ - **The onboarding surface (`/start.md`, `/docs`) is opt-in in production**
136
+ (`INSIKA_ONBOARDING=1`) — leaving it off is the default posture, not a bug.
137
+ - **Report every deviation.** A var you had to add, a check that failed and was
138
+ worked around, a step the platform made impossible — findings, not noise.
@@ -0,0 +1,70 @@
1
+ ---
2
+ title: Prompt — run every example
3
+ parent: Reference
4
+ nav_order: 2
5
+ permalink: /run-examples/
6
+ layout: default
7
+ render_with_liquid: false
8
+ ---
9
+
10
+ # Run every example
11
+
12
+ > **You are a coding agent** (Claude Code, Codex, Cursor, …) reading this because a
13
+ > developer pasted a prompt pointing here. Treat this file as a **skill**: follow the
14
+ > steps in order and apply the RULES literally. Do not improvise beyond them.
15
+
16
+ Your job: get every runnable example under `examples/` running — one at a time — and
17
+ explain to the developer what each one demonstrates. The authoritative list (and the
18
+ one-line capability per example) is
19
+ [`examples/README.md`](https://github.com/guizaols/insika/tree/main/examples/README.md)
20
+ (`examples/README.md` when the repo is checked out). Read it first.
21
+
22
+ ## Step 0 — Gather context (do this first, silently)
23
+
24
+ RULES — verify, do not assume:
25
+
26
+ - **Ruby ≥ 3.3.** Run `ruby -v`. If lower, stop and tell the developer; do not try to
27
+ upgrade Ruby for them.
28
+ - **Insika must be loadable** — `require "insika"` (installed gem) or the checked-out
29
+ repo's bundle. Do not copy source files around to "fix" a missing install.
30
+ - **A provider key comes from the environment** (`DEEPSEEK_API_KEY` for the demo).
31
+ None is set → **ask the developer**; never invent or hard-code one.
32
+ - **Read each example's own `README.md` before running it.** Some need more than one
33
+ terminal or extra env vars; the README is the contract.
34
+
35
+ ## Step 1 — Run in this order, ONE at a time
36
+
37
+ | # | Example | Command | Note |
38
+ |---|---------|---------|------|
39
+ | 1 | hello-agent | `ruby examples/hello-agent/hello.rb` | smallest agent; one turn |
40
+ | 2 | data-tool | `ruby examples/data-tool/currency_agent.rb` | declarative HTTP tool + the egress guard |
41
+ | 3 | skills | `ruby examples/skills/skill_agent.rb` | progressive skill loading |
42
+ | 4 | memory | `ruby examples/memory/memory_agent.rb` | cross-session `remember` |
43
+ | 5 | guardrails | `ruby examples/guardrails/guarded_agent.rb` | content-safety guardrails |
44
+ | 6 | agentic-workflows | `ruby examples/agentic-workflows/sequential.rb` | then routing/parallel/delegation/evaluator |
45
+ | 7 | scheduled-report | `ruby examples/scheduled-report/report_agent.rb` | runs one report turn inline; add `--serve` only if asked |
46
+ | 8 | relay-channel | see its README | TWO processes + env vars; skip unless asked |
47
+
48
+ For each one: read its README → run → quote what it printed → explain in ≤ 3 lines
49
+ what capability it demonstrated.
50
+
51
+ Skip unless the developer asks: `insika-code/` (a full deployment, not a one-file
52
+ script), `quickstart.rb` (hello-agent already covers it), and anything in `examples/`
53
+ the table above does not list.
54
+
55
+ ## Step 2 — When one fails
56
+
57
+ - **Auth or model error** → stop that example, report the exact error, ask for a valid
58
+ key/model id. Never retry with a guessed id.
59
+ - **A cause you can read from the output** (missing env var, port already bound…) →
60
+ say so and fix only with the developer's OK.
61
+ - **Never edit an example to make it pass silently.** An example that needed a change
62
+ to run is a finding, not noise.
63
+
64
+ ## Step 3 — Self-check before you report done
65
+
66
+ - [ ] Every example above either printed real model output or was reported blocked,
67
+ with the exact blocker.
68
+ - [ ] No provider key written into any file; no invented model ids.
69
+ - [ ] Each example got its ≤ 3-line "what this demonstrates".
70
+ - [ ] Nothing was edited to make a failure disappear.
data/docs/reference.md ADDED
@@ -0,0 +1,19 @@
1
+ ---
2
+ title: Reference
3
+ nav_order: 8
4
+ has_children: true
5
+ permalink: /reference/
6
+ ---
7
+
8
+ # Reference
9
+
10
+ The removability map, plus the paste-prompts that hand a journey to a coding
11
+ agent. A running instance serves each prompt at `GET /docs/<name>.md`, so you can
12
+ point Claude Code, Codex or Cursor at the URL instead of pasting the text.
13
+
14
+ - **[The domain-free core](domain.md)** — what ships in the gem, what a deployment declares, and how to clear it.
15
+ - **[Prompt — run every example](prompts/RUN-EXAMPLES.md)** — run `examples/` one at a time and explain what each proves.
16
+ - **[Prompt — add a tool or skill](prompts/ADD-TOOL.md)** — pick the right kind, wire the allowlist, prove it with one turn.
17
+ - **[Prompt — diagnose a failed turn](prompts/DIAGNOSE-TURN.md)** — symptom to mechanism, then fix one thing.
18
+ - **[Prompt — go live](prompts/GO-LIVE.md)** — tokens, volume, deploy, one authenticated turn, and the backup hole.
19
+ {: .card-grid }
data/docs/ship.md CHANGED
@@ -1,10 +1,18 @@
1
1
  ---
2
2
  title: Ship it
3
- nav_order: 4
3
+ nav_order: 5
4
4
  has_children: true
5
5
  permalink: /ship/
6
6
  ---
7
7
 
8
8
  # Ship it
9
9
 
10
- What stands between your agent and the open internet, and how to put it on a server.
10
+ What stands between your agent and the open internet, and how to put it on a
11
+ server without losing a turn to a restart.
12
+
13
+ - **[Security](SECURITY.md)** — guardrails, egress, approvals, secrets, and the privacy obligations that come with memory.
14
+ - **[Sandbox](SANDBOX.md)** — confined execution for tool code you did not write.
15
+ - **[Deploy](DEPLOY.md)** — the container, the durable volume, the process model, and the full environment table.
16
+ - **[Router](ROUTER.md)** — the session-sticky proxy that lets you run more than one worker.
17
+ - **[Releasing](RELEASING.md)** — how the gem is cut, and the install proof that runs before it is published.
18
+ {: .card-grid }
@@ -0,0 +1,18 @@
1
+ ---
2
+ title: Start here
3
+ nav_order: 2
4
+ has_children: true
5
+ permalink: /start-here/
6
+ ---
7
+
8
+ # Start here
9
+
10
+ Four pages, in order: what problem the runtime solves, how to get one running on
11
+ your machine, what actually happens inside a turn, and how to fill an empty
12
+ install with enough data to see every loop working.
13
+
14
+ - **[Why Insika](WHY.md)** — a runtime instead of a hand-rolled loop, an assembled framework, or a hosted gateway.
15
+ - **[Running locally](RUNNING-LOCAL.md)** — boot the engine, open the control UI, point a Responses client at it.
16
+ - **[Architecture](ARCHITECTURE.md)** — the turn pipeline, the tool-loop, checkpoint recovery, the concurrency model.
17
+ - **[Demo data](DEMO.md)** — seed a deployment so funnels, refinement and approvals have something to show.
18
+ {: .card-grid }
@@ -128,6 +128,12 @@ module Insika
128
128
  # event the consumer acts on). Same opt-in as
129
129
  # `memory`. What "stuck" MEANS is the consumer's call
130
130
  # (escalation via CRM/operator), never the engine's.
131
+ :stt_prompt, # STT vocabulary hint (WS9): domain words
132
+ # (product names, brand terms) the transcriber should
133
+ # expect on this agent's voice notes — passed straight
134
+ # through to the Whisper-family provider's `prompt:`.
135
+ # nil/absent = the deployment default (INSIKA_STT_PROMPT
136
+ # env) or nothing. OPERATOR config, never customer input.
131
137
  :outputs, # generated-media output policy (WS9, saída):
132
138
  # { "image" => { "model" => …, "size" => "1024x1024" },
133
139
  # "tts" => { "model" => "tts-1", "voice" => "alloy",
@@ -244,20 +250,50 @@ module Insika
244
250
  # off (parity, byte-identical engine). Shape-validated by
245
251
  # the command/engine, never here (the refinement precedent).
246
252
  # Deep-stringified like the other free-form hashes.
247
- :harvest # the gated-harvest declaration — pack data,
248
- # exactly like refinement/distill:
249
- # { "enabled" => bool,
250
- # "negative_list" => [ { "rule" => "…", "pattern" => "…",
251
- # "note" => "…" } ],
252
- # "miner" => { "model" => "<ref — absent = the platform
253
- # utility_model>", "window" => { "last_sessions" => N },
254
- # "max_proposals" => N, "budget" => { "tokens" => N } },
255
- # "idle_hours" => 24, "min_messages" => 3 }.
256
- # The ENGINE mines (reads sessions, asks the miner, filters
257
- # through the negative list + grounding), never authors a
258
- # rule (D4). nil/absent = the loop is off (parity).
259
- # Shape-validated by the command/engine/doctor, never here.
253
+ :harvest, # the gated-harvest declaration — pack data,
254
+ # exactly like refinement/distill:
255
+ # { "enabled" => bool,
256
+ # "negative_list" => [ { "rule" => "…", "pattern" => "…",
257
+ # "note" => "…" } ],
258
+ # "miner" => { "model" => "<ref — absent = the platform
259
+ # utility_model>", "window" => { "last_sessions" => N },
260
+ # "max_proposals" => N, "budget" => { "tokens" => N } },
261
+ # "idle_hours" => 24, "min_messages" => 3 }.
262
+ # The ENGINE mines (reads sessions, asks the miner, filters
263
+ # through the negative list + grounding), never authors a
264
+ # rule (D4). nil/absent = the loop is off (parity).
265
+ # Shape-validated by the command/engine/doctor, never here.
266
+ # Deep-stringified like the other free-form hashes.
267
+ :knowledge, # the post-turn learning declaration — pack
268
+ # data, exactly like distill/harvest:
269
+ # { "extract" => true, "retrieve" => true,
270
+ # "model" => "<ref — absent = the platform
271
+ # utility_model>", "top_k" => 5,
272
+ # "index" => "scan", "types" => ["fact", …] }.
273
+ # The ENGINE extracts concepts after the turn and
274
+ # stamps their provenance/confidence/sources;
275
+ # the model only names concepts (same D1
276
+ # discipline as distill). nil/absent = the
277
+ # loop is off (parity). Shape-validated by
278
+ # the extractor/doctor, never here.
260
279
  # Deep-stringified like the other free-form hashes.
280
+ :schedules # the recurring-schedule declarations — pack
281
+ # data, exactly like followup/distill:
282
+ # [ { "id" => "daily_report",
283
+ # "cron" | "every" => …,
284
+ # "tz" => "America/Sao_Paulo",
285
+ # "message" => "<the synthetic inbound>",
286
+ # "session_mode" => "new"|"fixed",
287
+ # "overrides" => { "turn_timeout" => N,
288
+ # "max_tool_calls" => N,
289
+ # "model" => … },
290
+ # "enabled" => bool }, … ].
291
+ # The ENGINE owns the firing (the
292
+ # ScheduleEngine, the tick's duty); the
293
+ # store rows are declared-derived.
294
+ # Shape-validated by Insika::Schedule,
295
+ # never here. nil/empty = the feature is
296
+ # off for that agent (parity).
261
297
  )
262
298
 
263
299
  # Reopened class (not a Data.define block): a constant assigned inside
@@ -289,8 +325,8 @@ module Insika
289
325
  params: {}, model_policy: nil, guardrails: nil, sandbox: nil,
290
326
  refinement: nil, capabilities_declared: nil, edge_stream: nil, metadata: {},
291
327
  budget: nil, reliability: nil, alerts: nil, routes: nil, stuck_signal: nil,
292
- outputs: nil, briefing_fields: nil, grounding: nil, funnel: nil,
293
- followup: nil, distill: nil, harvest: nil)
328
+ outputs: nil, stt_prompt: nil, briefing_fields: nil, grounding: nil, funnel: nil,
329
+ followup: nil, distill: nil, harvest: nil, knowledge: nil, schedules: nil)
294
330
  new(
295
331
  id: id, model: model, provider: provider, base_prompt: base_prompt,
296
332
  prompt_files: Array(prompt_files), tools_allow: tools_allow,
@@ -326,6 +362,9 @@ module Insika
326
362
  routes: Coercion.deep_stringify(routes),
327
363
  stuck_signal: stuck_signal,
328
364
  outputs: Coercion.deep_stringify(outputs),
365
+ # plain vocabulary string, like `base_prompt` — no deep_stringify (not
366
+ # a Hash/Array). "" round-trips as nil (Coercion.presence).
367
+ stt_prompt: Coercion.presence(stt_prompt),
329
368
  # Flat [String] — same discipline as capabilities_declared: a
330
369
  # symbol/string mix would be a silent miss in the provider's known-set.
331
370
  briefing_fields: normalize_briefing_fields(briefing_fields),
@@ -348,10 +387,28 @@ module Insika
348
387
  # harvest is profile DATA, deep-stringified like the other
349
388
  # free-form hashes; shape-validated by the command/engine/doctor
350
389
  # (never here — the refinement precedent). nil = off (parity).
351
- harvest: Coercion.deep_stringify(harvest)
390
+ harvest: Coercion.deep_stringify(harvest),
391
+ # knowledge is profile DATA, deep-stringified like the other
392
+ # free-form hashes; shape-validated by the extractor/doctor
393
+ # (never here — the refinement precedent). nil = off (parity).
394
+ knowledge: Coercion.deep_stringify(knowledge),
395
+ # schedules is profile DATA, deep-stringified like the other
396
+ # free-form hashes (an ARRAY of declarations); parsed into
397
+ # Insika::Schedule entries by the engine/doctor/Studio (shape-validated
398
+ # THERE, never here). nil/[] = the feature is off (parity).
399
+ schedules: normalize_schedules(schedules)
352
400
  )
353
401
  end
354
402
 
403
+ # nil/absent -> nil; a single Hash -> [Hash]; else an Array of Hashes —
404
+ # deep-stringified so JSON round-trips stay stable.
405
+ def self.normalize_schedules(list)
406
+ return nil if list.nil?
407
+
408
+ entries = list.is_a?(Hash) ? [list] : Array(list)
409
+ entries.empty? ? nil : Coercion.deep_stringify(entries)
410
+ end
411
+
355
412
  # nil -> []; strings; trim + drop empties + uniq (stable order); every name
356
413
  # must match ToolDefinition::NAME_RE (\A[a-z][a-z0-9_]*\z) or it is a
357
414
  # ValidationError at build time — the names become tool-description text,
@@ -0,0 +1,82 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "openssl"
4
+ require "time"
5
+
6
+ module Insika
7
+ # The signed-link half of the artifact serving surface. The signing key
8
+ # lives in the environment (INSIKA_ARTIFACT_SIGNING_KEY), never in a store:
9
+ # HMAC-SHA256 over (id, expiry), verified in constant time on the serve
10
+ # path. Without the key, only the authenticated Studio URL exists.
11
+ #
12
+ # The token is deterministic for a (id, expiry) pair — no nonce, on purpose:
13
+ # a rotated key invalidates every outstanding link, which is the documented
14
+ # rotation behavior (an artifact link is short-lived by TTL, not by
15
+ # unguessability of a single-use nonce).
16
+ module ArtifactSigning
17
+ module_function
18
+
19
+ # The route's path shapes — the ONE place the URL grammar lives, shared
20
+ # by the tool (which hands URLs to the model) and the route (which serves
21
+ # them).
22
+ AUTHENTICATED_PATH = "/studio/artifacts/%{id}/content"
23
+ SIGNED_PATH = "/studio/artifacts/s/%{id}?exp=%{exp}&sig=%{sig}"
24
+
25
+ # -> hex token (64 chars) | nil when the key is blank (no signed surface).
26
+ def sign(id:, expires_at:, key:)
27
+ key = key.to_s
28
+ return nil if key.empty?
29
+
30
+ OpenSSL::HMAC.hexdigest("SHA256", key, payload(id, expires_at))
31
+ end
32
+
33
+ # -> bool. Re-signs the (id, exp) pair the route extracted from the URL
34
+ # and compares in constant time; an expired link or a blank key/token is
35
+ # false (the route 404s — no oracle). Expiry is inclusive: a link at its
36
+ # exact `expires_at` still verifies.
37
+ def valid?(id:, token:, key:, exp:, now: Time.now.utc)
38
+ token = token.to_s
39
+ key = key.to_s
40
+ return false if key.empty? || token.empty?
41
+
42
+ expected = sign(id: id.to_s, expires_at: exp, key: key)
43
+ return false unless expected && secure_compare(expected, token)
44
+
45
+ exp_time = exp.is_a?(Time) ? exp.to_time.utc : Time.iso8601(exp.to_s).utc
46
+ now.to_time <= exp_time
47
+ rescue ArgumentError, TypeError
48
+ false # a malformed exp (or a time that never parses) is an invalid link
49
+ end
50
+
51
+ # -> the artifact's shareable URL. With a key + ttl: the signed link
52
+ # (shares OUTSIDE the Studio, expires). Without: the authenticated Studio
53
+ # content URL. An empty base yields the relative path — still openable in
54
+ # the Studio, useless on a channel (the doc says exactly that).
55
+ def url_for(id:, base: nil, key: nil, ttl: nil, now: Time.now.utc)
56
+ base = base.to_s.sub(%r{/\z}, "")
57
+ if key && key.to_s.length.positive? && ttl && ttl.to_i.positive?
58
+ exp = (now.to_time + ttl.to_i).utc.iso8601
59
+ sig = sign(id: id.to_s, expires_at: exp, key: key)
60
+ "#{base}#{format(SIGNED_PATH, id: id, exp: exp, sig: sig)}"
61
+ else
62
+ "#{base}#{format(AUTHENTICATED_PATH, id: id)}"
63
+ end
64
+ end
65
+
66
+ # Constant-time comparison of two hex strings. Length-independent compare
67
+ # is fine here: the token length is public (fixed by the algorithm).
68
+ def secure_compare(a, b)
69
+ return false unless a.bytesize == b.bytesize
70
+
71
+ a.bytes.zip(b.bytes).reduce(0) { |acc, (x, y)| acc | (x ^ y) }.zero?
72
+ end
73
+
74
+ # The signed payload: id + expiry — both are what the route must not let
75
+ # an attacker change. The id is the store key; the expiry bounds the link.
76
+ # Accepts a Time or an ISO8601 String (the route passes the query param).
77
+ def payload(id, expires_at)
78
+ time = expires_at.is_a?(Time) ? expires_at.utc : Time.iso8601(expires_at.to_s).utc
79
+ "#{id}:#{time.iso8601}"
80
+ end
81
+ end
82
+ end
@@ -0,0 +1,160 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "securerandom"
4
+ require "time"
5
+
6
+ module Insika
7
+ # The report destination: one record per run, no versioning, the listing
8
+ # IS the history. A store, not a CMS.
9
+ #
10
+ # The tenant is a BINDING of the row — inherited from the agent that saved
11
+ # the artifact (the tool reads it from the turn context, never from the
12
+ # model), so a purge is a tenant-prefix scan and store A's report can never
13
+ # appear in, or be linked from, store B.
14
+ #
15
+ # Record key: "<tenant>:<agent>:<id>" — per-tenant / per-agent scans are
16
+ # prefixes; the id is the LAST segment, so a find is a suffix match (the
17
+ # followup_store.rb idiom — there is no stable prefix for an id alone).
18
+ # Blank tenant -> the literal "platform" (the outcome_store.rb rule, so the
19
+ # purge prefix scans line up).
20
+ class ArtifactStore
21
+ SCOPE = "artifacts"
22
+
23
+ # The mime allowlist — a page, not an attachment: no binaries, no uploads.
24
+ MIMES = %w[text/html text/markdown image/svg+xml].freeze
25
+
26
+ # The size cap on `content` (INSIKA_ARTIFACT_MAX_BYTES; the audio-message
27
+ # precedent is 1 MB — an artifact is a page, not an attachment).
28
+ DEFAULT_MAX_BYTES = 1_000_000
29
+ TITLE_MAX = 200
30
+
31
+ Record = Data.define(:id, :tenant, :agent, :task_id, :title, :mime,
32
+ :content, :created_at)
33
+
34
+ def initialize(store:)
35
+ @store = store
36
+ end
37
+
38
+ # -> Record. Validates the mime allowlist, a non-empty title (<= 200
39
+ # chars) and content within `max_bytes` — ValidationError otherwise (the
40
+ # tool returns it to the model as `{ error: }`).
41
+ def create(tenant:, agent:, task_id:, title:, mime:, content:,
42
+ id: SecureRandom.uuid, now: Time.now.utc, max_bytes: DEFAULT_MAX_BYTES)
43
+ mime = "text/html" if mime.to_s.empty?
44
+ raise Insika::ValidationError, "mime must be one of #{MIMES.join(', ')}, got #{mime.inspect}" unless MIMES.include?(mime.to_s)
45
+
46
+ title = title.to_s
47
+ raise Insika::ValidationError, "title is required (1..#{TITLE_MAX} chars)" if title.strip.empty? || title.length > TITLE_MAX
48
+
49
+ content = content.to_s
50
+ raise Insika::ValidationError, "content is required" if content.empty?
51
+ raise Insika::ValidationError, "content exceeds #{max_bytes} bytes (#{content.bytesize})" if content.bytesize > max_bytes
52
+
53
+ record = { "id" => id.to_s, "tenant" => tenant_id(tenant), "agent" => agent.to_s,
54
+ "task_id" => task_id.to_s, "title" => title, "mime" => mime.to_s,
55
+ "content" => content, "created_at" => now.iso8601 }
56
+ @store.set(SCOPE, key(record), record)
57
+ to_record(record)
58
+ end
59
+
60
+ # -> Record | nil. Suffix match over the scope's keys (no stable prefix
61
+ # for an id alone — the followup_store.rb idiom).
62
+ def find(id)
63
+ key = @store.list(SCOPE).find { |k| k.end_with?(":#{id}") }
64
+ key && to_record(@store.get(SCOPE, key))
65
+ end
66
+
67
+ # -> [Record] — one agent's artifacts, newest first (the listing IS the
68
+ # history; the Studio tab's read).
69
+ def for_agent(tenant:, agent:)
70
+ prefix = "#{tenant_id(tenant)}:#{agent}:"
71
+ @store.list(SCOPE).filter_map do |k|
72
+ next unless k.start_with?(prefix)
73
+
74
+ to_record(@store.get(SCOPE, k))
75
+ end.sort_by { |r| [r.created_at.to_s, r.id] }.reverse
76
+ end
77
+
78
+ # -> [Record] — EVERY agent's artifacts for the tenant, newest first (same
79
+ # prefix scan as #purge, but reads instead of deletes). Needs the CALLER
80
+ # to know the right tenant string — see #all below for the Studio, which
81
+ # doesn't.
82
+ def for_tenant(tenant:)
83
+ prefix = "#{tenant_id(tenant)}:"
84
+ @store.list(SCOPE).filter_map do |k|
85
+ next unless k.start_with?(prefix)
86
+
87
+ to_record(@store.get(SCOPE, k))
88
+ end.sort_by { |r| [r.created_at.to_s, r.id] }.reverse
89
+ end
90
+
91
+ # -> [Record] — every artifact in the store, optionally narrowed to one
92
+ # agent, with NO tenant guess. For the Studio only: it is already
93
+ # operator-only (sees every tenant's agents), and unlike #for_agent/
94
+ # #for_tenant (hot, tenant-scoped runtime paths that a real multi-tenant
95
+ # caller uses because it KNOWS its own tenant) the Studio often does not
96
+ # — `save_artifact` binds tenant from the turn (`agent` when the Playground
97
+ # dispatched it, `"platform"` for a plain single-tenant API turn), so a
98
+ # page guessing one fixed string is wrong for the other half of the time.
99
+ # Filtering by the record's own `agent` field sidesteps the guess entirely.
100
+ def all(agent: nil)
101
+ @store.list(SCOPE).filter_map do |k|
102
+ record = to_record(@store.get(SCOPE, k))
103
+ next if record.nil?
104
+ next if agent && record.agent != agent.to_s
105
+
106
+ record
107
+ end.sort_by { |r| [r.created_at.to_s, r.id] }.reverse
108
+ end
109
+
110
+ # -> bool (did it exist?).
111
+ def delete(id)
112
+ key = @store.list(SCOPE).find { |k| k.end_with?(":#{id}") }
113
+ key ? @store.delete(SCOPE, key) : false
114
+ end
115
+
116
+ # -> count removed. The tenant-erasure reach — one tenant's artifacts die
117
+ # with it, never a neighbour's.
118
+ def purge(tenant:)
119
+ prefix = "#{tenant_id(tenant)}:"
120
+ keys = @store.list(SCOPE).select { |k| k.start_with?(prefix) }
121
+ keys.each { |k| @store.delete(SCOPE, k) }
122
+ keys.size
123
+ end
124
+
125
+ # -> count removed. The retention knob's reach (`artifact_ttl_days`) —
126
+ # the guarantee that PII inside a report expires even though no reader
127
+ # can see inside the opaque HTML.
128
+ def delete_older_than(time)
129
+ cutoff = time.utc.iso8601
130
+ removed = 0
131
+ @store.list(SCOPE).each do |k|
132
+ record = @store.get(SCOPE, k)
133
+ next unless record && record["created_at"].to_s < cutoff
134
+
135
+ @store.delete(SCOPE, k)
136
+ removed += 1
137
+ end
138
+ removed
139
+ end
140
+
141
+ private
142
+
143
+ def key(record)
144
+ "#{record['tenant']}:#{record['agent']}:#{record['id']}"
145
+ end
146
+
147
+ def tenant_id(tenant)
148
+ t = tenant.to_s
149
+ t.empty? ? "platform" : t
150
+ end
151
+
152
+ def to_record(rec)
153
+ return nil if rec.nil?
154
+
155
+ Record.new(id: rec["id"], tenant: rec["tenant"], agent: rec["agent"],
156
+ task_id: rec["task_id"], title: rec["title"], mime: rec["mime"],
157
+ content: rec["content"], created_at: rec["created_at"])
158
+ end
159
+ end
160
+ end
@@ -17,7 +17,7 @@ module Insika
17
17
  # party with outages, so "keep trying" is a real requirement and "keep trying
18
18
  # forever" is a real outage of ours.
19
19
  #
20
- # It is NOT a job queue: no scheduler, no priorities, no fan-out. The moment it
20
+ # It is NOT a job queue: no priorities, no fan-out. The moment it
21
21
  # grows one, the thing to do is take a real queue, not to finish building this.
22
22
  class ChannelDelivery
23
23
  MAX_ATTEMPTS = 3