insika 0.3.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (190) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +180 -0
  3. data/README.md +45 -10
  4. data/bin/insika +684 -0
  5. data/bin/insika-router +87 -0
  6. data/docs/AGENTS.md +94 -403
  7. data/docs/API.md +5 -5
  8. data/docs/ARCHITECTURE.md +3 -2
  9. data/docs/ARTIFACTS.md +95 -0
  10. data/docs/BENCHMARK.md +2 -2
  11. data/docs/CHANNELS.md +14 -14
  12. data/docs/CONTEXT.md +9 -7
  13. data/docs/DEMO.md +80 -0
  14. data/docs/DEPLOY.md +71 -3
  15. data/docs/EMBEDDING.md +1 -1
  16. data/docs/EVALS.md +128 -3
  17. data/docs/FACTS.md +3 -3
  18. data/docs/HARVEST.md +5 -6
  19. data/docs/KNOWLEDGE.md +290 -0
  20. data/docs/LOADTEST.md +2 -2
  21. data/docs/MEDIA.md +128 -0
  22. data/docs/OBSERVABILITY.md +15 -10
  23. data/docs/OUTCOMES.md +137 -0
  24. data/docs/PLUGINS.md +51 -6
  25. data/docs/POLICY.md +216 -0
  26. data/docs/REFINEMENT.md +14 -9
  27. data/docs/RELEASING.md +4 -4
  28. data/docs/ROUTER.md +213 -0
  29. data/docs/RUNNING-LOCAL.md +3 -3
  30. data/docs/SCHEDULING.md +121 -0
  31. data/docs/SECURITY.md +22 -6
  32. data/docs/SKILLS.md +11 -2
  33. data/docs/SOAK.md +2 -2
  34. data/docs/TEMPLATES.md +134 -0
  35. data/docs/TOOLS.md +152 -27
  36. data/docs/WHY.md +1 -1
  37. data/docs/WORKFLOWS.md +2 -2
  38. data/docs/_includes/head_custom.html +5 -0
  39. data/docs/_includes/title.html +13 -0
  40. data/docs/_sass/color_schemes/insika.scss +32 -0
  41. data/docs/_sass/custom/custom.scss +199 -0
  42. data/docs/_sass/custom/setup.scss +26 -0
  43. data/docs/assets/img/favicon.svg +7 -0
  44. data/docs/assets/img/insika-mark.svg +7 -0
  45. data/docs/core-concepts.md +21 -0
  46. data/docs/domain.md +4 -4
  47. data/docs/improve.md +20 -0
  48. data/docs/index.md +8 -5
  49. data/docs/integrate.md +20 -0
  50. data/docs/operate.md +13 -6
  51. data/docs/prompts/ADD-TOOL.md +118 -0
  52. data/docs/prompts/DIAGNOSE-TURN.md +65 -0
  53. data/docs/prompts/GO-LIVE.md +138 -0
  54. data/docs/prompts/RUN-EXAMPLES.md +70 -0
  55. data/docs/reference.md +19 -0
  56. data/docs/ship.md +10 -2
  57. data/docs/start-here.md +18 -0
  58. data/lib/insika/agent_profile.rb +73 -16
  59. data/lib/insika/artifact_signing.rb +82 -0
  60. data/lib/insika/artifact_store.rb +160 -0
  61. data/lib/insika/channel_delivery.rb +1 -1
  62. data/lib/insika/chat_builder.rb +22 -2
  63. data/lib/insika/commands/agent_payload.rb +2 -2
  64. data/lib/insika/commands/backfill_knowledge.rb +145 -0
  65. data/lib/insika/commands/delete_artifact.rb +35 -0
  66. data/lib/insika/commands/delete_concept.rb +34 -0
  67. data/lib/insika/commands/delete_mcp.rb +6 -2
  68. data/lib/insika/commands/delete_tenant_data.rb +15 -3
  69. data/lib/insika/commands/gate_refinement.rb +1 -1
  70. data/lib/insika/commands/refresh_mcp_tools.rb +47 -0
  71. data/lib/insika/commands/restore_concept.rb +34 -0
  72. data/lib/insika/commands/seed_demo_data.rb +31 -0
  73. data/lib/insika/commands/upsert_mcp.rb +6 -3
  74. data/lib/insika/commands/write_concept.rb +57 -0
  75. data/lib/insika/context/priority.rb +2 -0
  76. data/lib/insika/context/providers/knowledge.rb +108 -0
  77. data/lib/insika/context/providers/prompt.rb +30 -24
  78. data/lib/insika/cron.rb +189 -0
  79. data/lib/insika/demo/agent_attrs.rb +43 -0
  80. data/lib/insika/demo/golden_cases.rb +81 -0
  81. data/lib/insika/demo/seeder.rb +336 -0
  82. data/lib/insika/doctor.rb +176 -8
  83. data/lib/insika/dsl/definition.rb +3 -2
  84. data/lib/insika/dsl/runtime.rb +60 -79
  85. data/lib/insika/dsl/server_boot.rb +23 -1
  86. data/lib/insika/dsl/system.rb +10 -2
  87. data/lib/insika/dsl.rb +103 -2
  88. data/lib/insika/env_schema.rb +16 -1
  89. data/lib/insika/evals/golden.rb +41 -4
  90. data/lib/insika/evals/judge.rb +47 -2
  91. data/lib/insika/evals/pairwise.rb +11 -0
  92. data/lib/insika/evals/persona.rb +98 -0
  93. data/lib/insika/evals/runner.rb +9 -0
  94. data/lib/insika/evals/simulator.rb +225 -0
  95. data/lib/insika/evals/transport.rb +83 -1
  96. data/lib/insika/event_stream.rb +10 -0
  97. data/lib/insika/executor.rb +231 -55
  98. data/lib/insika/followup_policy.rb +2 -25
  99. data/lib/insika/golden_store.rb +16 -1
  100. data/lib/insika/grounding/matcher.rb +1 -1
  101. data/lib/insika/knowledge.rb +680 -0
  102. data/lib/insika/knowledge_store.rb +140 -0
  103. data/lib/insika/mcp_client.rb +94 -0
  104. data/lib/insika/mcp_json.rb +74 -0
  105. data/lib/insika/mcp_live_tool.rb +43 -0
  106. data/lib/insika/mcp_store.rb +98 -26
  107. data/lib/insika/mcp_tool_ingestor.rb +30 -8
  108. data/lib/insika/mcp_tool_registry.rb +100 -0
  109. data/lib/insika/media.rb +115 -31
  110. data/lib/insika/message_origin.rb +1 -1
  111. data/lib/insika/middleware.rb +9 -0
  112. data/lib/insika/onboarding.rb +17 -1
  113. data/lib/insika/outcome_store.rb +1 -1
  114. data/lib/insika/overlay_tool_registry.rb +37 -17
  115. data/lib/insika/packaging.rb +2 -2
  116. data/lib/insika/profile_source.rb +8 -1
  117. data/lib/insika/prompt_catalog.rb +10 -0
  118. data/lib/insika/retention.rb +36 -1
  119. data/lib/insika/router/app.rb +157 -0
  120. data/lib/insika/router/backend_pool.rb +98 -0
  121. data/lib/insika/router/hash_ring.rb +55 -0
  122. data/lib/insika/router/proxy_body.rb +34 -0
  123. data/lib/insika/router/session_key.rb +54 -0
  124. data/lib/insika/router.rb +18 -0
  125. data/lib/insika/schedule.rb +177 -0
  126. data/lib/insika/schedule_engine.rb +314 -0
  127. data/lib/insika/schedule_store.rb +208 -0
  128. data/lib/insika/server/app.rb +105 -15
  129. data/lib/insika/server/rack_app.rb +5 -1
  130. data/lib/insika/server/responses.rb +1 -1
  131. data/lib/insika/skill_catalog.rb +12 -0
  132. data/lib/insika/steer_injector.rb +21 -10
  133. data/lib/insika/studio/app.rb +567 -45
  134. data/lib/insika/studio/assets/dist/application.css +1 -1
  135. data/lib/insika/studio/assets/dist/application.js +21 -21
  136. data/lib/insika/studio/forms.rb +46 -5
  137. data/lib/insika/studio/nav_icons.rb +14 -1
  138. data/lib/insika/studio/views/_agent_tab_cache.erb +25 -0
  139. data/lib/insika/studio/views/_agent_tab_config.erb +514 -0
  140. data/lib/insika/studio/views/_agent_tab_history.erb +24 -0
  141. data/lib/insika/studio/views/_agent_tab_loops.erb +54 -0
  142. data/lib/insika/studio/views/_agent_tab_memory.erb +51 -0
  143. data/lib/insika/studio/views/_agent_tab_outcomes.erb +31 -0
  144. data/lib/insika/studio/views/_agent_tab_prompts.erb +108 -0
  145. data/lib/insika/studio/views/_agent_tab_skills.erb +38 -0
  146. data/lib/insika/studio/views/_agents_master.erb +44 -0
  147. data/lib/insika/studio/views/_message.erb +49 -32
  148. data/lib/insika/studio/views/agent_detail.erb +61 -820
  149. data/lib/insika/studio/views/agents.erb +70 -57
  150. data/lib/insika/studio/views/artifact.erb +23 -0
  151. data/lib/insika/studio/views/artifacts.erb +59 -0
  152. data/lib/insika/studio/views/evals.erb +2 -2
  153. data/lib/insika/studio/views/facts.erb +1 -1
  154. data/lib/insika/studio/views/funnel.erb +1 -1
  155. data/lib/insika/studio/views/home.erb +106 -67
  156. data/lib/insika/studio/views/knowledge.erb +123 -0
  157. data/lib/insika/studio/views/layout.erb +14 -11
  158. data/lib/insika/studio/views/mcp.erb +174 -80
  159. data/lib/insika/studio/views/session.erb +231 -177
  160. data/lib/insika/studio/views/settings.erb +39 -1
  161. data/lib/insika/studio/views/skills.erb +1 -1
  162. data/lib/insika/studio/views/tools.erb +24 -9
  163. data/lib/insika/templates/browser-agent/README.md +36 -0
  164. data/lib/insika/templates/browser-agent/agent.rb +49 -0
  165. data/lib/insika/templates/daily-digest/README.md +38 -0
  166. data/lib/insika/templates/daily-digest/agent.rb +77 -0
  167. data/lib/insika/templates/repo-explorer/README.md +36 -0
  168. data/lib/insika/templates/repo-explorer/agent.rb +45 -0
  169. data/lib/insika/templates/research-analyst/README.md +26 -0
  170. data/lib/insika/templates/research-analyst/agent.rb +58 -0
  171. data/lib/insika/templates/review-panel/README.md +20 -0
  172. data/lib/insika/templates/review-panel/agent.rb +50 -0
  173. data/lib/insika/templates/travel-planner/README.md +35 -0
  174. data/lib/insika/templates/travel-planner/agent.rb +87 -0
  175. data/lib/insika/templates.rb +112 -0
  176. data/lib/insika/tick.rb +24 -12
  177. data/lib/insika/timezone.rb +45 -0
  178. data/lib/insika/tools/generate_image.rb +52 -7
  179. data/lib/insika/tools/load_knowledge.rb +74 -0
  180. data/lib/insika/tools/run_persona_eval.rb +328 -0
  181. data/lib/insika/tools/save_artifact.rb +95 -0
  182. data/lib/insika/turn_output.rb +1 -1
  183. data/lib/insika/turn_state.rb +15 -4
  184. data/lib/insika/version.rb +1 -1
  185. data/lib/insika/wiring/graph.rb +184 -12
  186. data/lib/insika/wiring/graph_chat.rb +102 -0
  187. data/lib/insika.rb +57 -0
  188. metadata +105 -5
  189. data/docs/build.md +0 -14
  190. data/docs/understand.md +0 -10
data/docs/AGENTS.md CHANGED
@@ -1,6 +1,6 @@
1
1
  ---
2
2
  title: Agents
3
- parent: Build an agent
3
+ parent: Core concepts
4
4
  nav_order: 1
5
5
  permalink: /agents/
6
6
  ---
@@ -70,6 +70,23 @@ injected text as the customer repeating themselves. See
70
70
  ## The AgentProfile
71
71
 
72
72
  A profile is built through one front door — `AgentProfile.build(id:, model: nil, …)`.
73
+ `model`/`provider` are a straight pass-through to [RubyLLM](https://rubyllm.com) — Insika
74
+ keeps no allowlist of "supported" models. Whatever RubyLLM's installed version
75
+ reaches, an agent can name: today that's 13 provider adapters (Anthropic, OpenAI,
76
+ Gemini, DeepSeek, Bedrock, Vertex AI, Azure, Mistral, xAI, OpenRouter,
77
+ Perplexity, Ollama, GPUStack) and every model each one exposes — see
78
+ [rubyllm.com/available-models](https://rubyllm.com/available-models/) for the
79
+ current, live catalog. Upgrading the `ruby_llm` gem is the only thing that ever
80
+ widens this list; no Insika code changes with it.
81
+
82
+ Wire a provider's credentials once in the Studio's **Settings → LLM providers**
83
+ (or `POST /v1/settings`) — `api` is any slug RubyLLM recognizes, and
84
+ `LLMConfigurator` applies it by reflection (`<api>_api_key=`, `<api>_api_base=`),
85
+ so a provider RubyLLM doesn't expose an accessor for is skipped, never a hard
86
+ error. To *restrict* which models an agent may use — the opposite direction —
87
+ declare `model_policy: { allow: [refs] }` on its profile (exact `"provider/model"`
88
+ refs or a `"provider/*"` wildcard); absent means no fence, every configured
89
+ model is fair game.
73
90
  A prompt file is **text**. Passing a structured value where the markdown belongs —
74
91
  a `{"content": …}` wrapper in a pack, or a store entry read and written back — is
75
92
  rejected, not coerced: `to_s` on a Hash produces Ruby's `#inspect`, and a prompt made
@@ -246,7 +263,7 @@ sent the order number" without keeping a throwaway task per fragment.
246
263
 
247
264
  > **`steer` also collects at the door.** The two windows are the same policy's
248
265
  > halves, not two modes: a `steer` agent that set a `debounce_ms` also merges the
249
- > fragments that land before the turn starts (RFC-0027). The window value is what
266
+ > fragments that land before the turn starts. The window value is what
250
267
  > an operator replaces the legacy pre-batch buffer with — `steer` catches anything
251
268
  > that arrives after the turn started, the door window the fragments before it.
252
269
 
@@ -384,204 +401,9 @@ has promotions" from data is how a test suite starts lying.
384
401
 
385
402
  ## The five access layers
386
403
 
387
- What an agent may do is layered. Each layer is independent, opt-in where it
388
- matters, and editable hot.
404
+ What an agent may do is layered: which tools it may call, which calls need a human, content safety, edge limits on flood and spend, and how much it may think. See [Limits and policy](POLICY.md).
389
405
 
390
- ### Layer 1: Tools (what it can call)
391
-
392
- `tools_allow` / `tools_deny` / `tools_allow_groups` decide which tools enter the
393
- turn's tool-loop, enforced by the tool-allowlist policy. See [Tools](TOOLS.md)
394
- for how tools are defined and registered, and [`examples/data-tool/`](https://github.com/guizaols/insika/tree/main/examples/data-tool/).
395
-
396
- ### Layer 2: Policies and approvals
397
-
398
- Policies are named entries evaluated before the turn runs. Builtins cover
399
- tool-, skill-, and workflow-allowlisting, plus **`ApprovalRequired`** — which
400
- does not allow or deny but *tags* a tool as needing human approval. Set
401
- `approvals_required: [tool names]`; the gate then fires when the model tries to
402
- call that tool, suspending the turn until an operator approves it in the Studio.
403
- See [Security](SECURITY.md#human-approval).
404
-
405
- ### Layer 3: Guardrails (content safety)
406
-
407
- `guardrails` configures input/output content safety per agent — **opt-in**, so an
408
- agent that says nothing gets a conservative default (deterministic detectors on,
409
- LLM moderator off). See [Security](SECURITY.md#guardrails) and
410
- [`examples/guardrails/`](https://github.com/guizaols/insika/tree/main/examples/guardrails/).
411
-
412
- ### Layer 4: Edge limits (flood and spend control)
413
-
414
- Two independent, opt-in ceilings, enforced *before* the model is ever called —
415
- opt-in everywhere except on a public channel, where `chat_rate_limit` is
416
- [required](CHANNELS.md#a-rate-limit-is-required-not-suggested) and the
417
- [web widget](CHANNELS.md#the-web-widget) refuses to serve without one:
418
-
419
- - **`chat_rate_limit`** — turn attempts per session per `chat_rate_window`.
420
- - **`agent_token_ceiling`** — total tokens per agent per `agent_token_window`.
421
-
422
- On breach the turn halts gracefully with a configurable `limit_response` and
423
- **zero LLM calls**. Windows are set at the platform level; the ceilings can be set
424
- per agent (blank inherits the platform value, `0` explicitly disables it).
425
-
426
- > ⚠️ The token window default is **86400 (daily)**. To express "500k tokens per
427
- > **hour**", set `agent_token_window = 3600` explicitly. A per-agent key that is
428
- > *present but nil* reads as OFF for that agent — leave the key **absent** to
429
- > inherit. See [Security](SECURITY.md#edge-limits).
430
-
431
- #### Calendar budgets — the daily/monthly cost wall
432
-
433
- A third, opt-in ceiling for the *billing* shape the windows above cannot express:
434
- a spend cap over a CALENDAR day or month, per `(tenant, agent)` when
435
- multi-tenant. Data on the profile (DSL `budget` or the pack's `budget` key):
436
-
437
- ```ruby
438
- budget daily: 100_000, monthly: 2_000_000, soft: false # or soft: true
439
- ```
440
-
441
- - Tokens count the **billed** spend — `input + output + cached + cache_creation`
442
- (the cached prefix is the bulk of the bill, not an afterthought).
443
- - **Hard** (the default, `soft` absent/false): a turn that arrives with the
444
- window's spend already at/over the cap **fails** with the typed
445
- `Insika::BudgetExceeded` — the envelope reads `budget_exceeded` +
446
- `retry_after` (seconds until the window rolls). It is NOT a customer reply; it
447
- is an operator signal.
448
- - **Soft** (`soft: true`): the same turn RUNS — crossing the cap emits one
449
- `budget_warning` event per window and injects a note into the context (the
450
- model sees it, the transcript does not).
451
- - Either way, crossing `alert_at` (default `0.8` of the cap) fires the same
452
- warning **before** the wall, once per window.
453
-
454
- > ⚠️ Unlike the ceilings above, the cap that counts is per **calendar** window —
455
- > a `daily` budget rolls at UTC midnight, a `monthly` one on the 1st, whatever
456
- > the sun. `agent_token_window` is a fixed seconds window and cannot express
457
- > "the day resets at midnight".
458
-
459
- #### Reliability — retries, fallback, circuit breaker (WS3)
460
-
461
- The provider interaction is a single attempt by default (RubyLLM's own 2
462
- transport retries aside). For a store that cannot have a dead model take the
463
- chat down, the reliability policy is DATA on the profile:
464
-
465
- ```ruby
466
- reliability retries: 2, backoff: "exponential",
467
- fallback: ["openai/gpt-4o-mini"],
468
- circuit_breaker: { after: 10, within: 60, cooldown: 300 }
469
- ```
470
-
471
- - **Retries** — transient failures (`:retryable` / `:rate_limited_*` per the
472
- error classification) retry with exponential backoff, up to `retries`.
473
- A `:fatal` (auth, billing, bad request) is NEVER retried or rotated. Each
474
- attempt runs on a fresh chat — the customer-visible answer comes only from
475
- the attempt that returns.
476
- - **Fallback** — after a node's retries, the turn ROTATES to the next model in
477
- the chain: the profile's `fallback` refs first, then the platform
478
- `fallback_models`. The turn's usage is attributed to the model that actually
479
- spoke (`model_source: "fallback"`).
480
- - **Circuit breaker** — per `(tenant, provider/model)`: `after` failures within
481
- `within` seconds open the circuit; while open, the turn fail-fasts with the
482
- typed `circuit_open` + `retry_after` (remaining cooldown) and the provider is
483
- never touched. After `cooldown` a half-open trial closes the circuit on
484
- success or reopens it on failure.
485
- - **`timeout`** — per-attempt ceiling (default 30s), counted as a retryable
486
- failure.
487
-
488
- Absent `reliability` = the plain single attempt, byte-for-byte today's
489
- behavior.
490
-
491
- #### Intent routing — classify before you answer (WS4)
492
-
493
- For a store that must tell "shopping" from "order" from "human" up front,
494
- routing is data on the profile:
495
-
496
- ```ruby
497
- routes "shopping" => "the customer wants to browse products",
498
- "order" => { "description" => "asks about an existing order",
499
- "delegate" => "order-agent" },
500
- "human" => { "description" => "the customer asks for a person",
501
- "stuck" => true, "message" => "A person will help you." },
502
- "default" => "shopping",
503
- "model" => "deepseek-v4-flash" # the cheap classifier (absent = the agent's own)
504
- ```
505
-
506
- - **Classification** — when `routes:` is present, the message is classified into
507
- one route with the configured model BEFORE the agent chat is assembled, from
508
- a prompt auto-generated out of the descriptions (no per-route prompt file).
509
- The route rides the turn: `state.route`, the `:route_classified` event, and
510
- the terminal event additively.
511
- - **Deterministic default** — the model's answer must be a route name; prose,
512
- an unknown name, or an empty answer falls back to `default`, never invents.
513
- A classifier call that FAILS leaves the turn unrouted (routing is additive —
514
- it must not break the turn).
515
- - **Cost** — the classification is an extra provider call, counted in the
516
- turn's usage (the trace, the token ceiling and the budget all see it).
517
- - **Actions** — a route value may be a description string, or a Hash:
518
- `delegate: "<agent-id>"` hands the turn to that existing agent and its
519
- answer becomes the parent's; `stuck: true` ends the turn with the [stuck
520
- outcome](#the-stuck-signal--i-cannot-proceed-ws5) and the route's `message`
521
- (or description) as the lead-in — the consumer interprets it. A route with
522
- neither is just a label. A delegation counts against the same delegation
523
- depth cap as a subagent (`INSIKA_SUBAGENT_DEPTH_CAP`, default 5), so a pair
524
- of agents routing to each other stops instead of looping.
525
-
526
- Absent `routes` = no classification, no extra call, byte-identical turn.
527
-
528
- #### Operator alerts — the webhook (WS6)
529
-
530
- Three operational events — `budget_warning`, `breaker_open`, `delivery_failed` —
531
- can be answered per agent with a webhook:
532
-
533
- ```ruby
534
- alerts webhook: "https://ops.example.com/insika-alerts"
535
- ```
536
-
537
- When present, each such event is POSTed to the URL as JSON (the event's
538
- type/data/meta, plus the agent). Delivery rides the same outbox + claim +
539
- bounded-retry pipeline as channel answers — at-most-once, crashed deliveries
540
- recovered at boot. The engine transports the event and does not interpret it: a
541
- Slack/CRM adapter is the consumer's. Absent `alerts` = nothing is sent.
542
-
543
- Separately, with `INSIKA_TURN_TIMING`, the provider's **live TTFB** is carried in
544
- the streaming envelope: the first content chunk emits an `insika.ttft` frame
545
- (`ttft_ms`) on `/v1/responses`, alongside the per-turn `timing` breakdown on the
546
- final `response.completed`. Additive and opt-in — absent by default.
547
-
548
- ### Layer 5: Reasoning (thinking)
549
-
550
- Controls the model's thinking budget, resolved by precedence
551
- **Chat > Agent > Model > Global** (first non-blank wins):
552
-
553
- | Scope | Where |
554
- |-------|-------|
555
- | Chat | session var `__llm__.thinking` |
556
- | Agent | `profile.params["thinking"]` |
557
- | Model | platform `model_params[<ref>].thinking` |
558
- | Global | platform `thinking` |
559
-
560
- Values: `off | on | low | medium | high`. `off`/`on` toggle thinking; the effort
561
- levels map to the provider's thinking-effort parameter. This is a control
562
- primitive, not a latency lever — turning reasoning off does not necessarily speed
563
- up a turn, because most of a turn's latency is the provider itself, not thinking.
564
-
565
- Whether the reasoning ever reaches the **customer** is a separate switch, off by
566
- default:
567
-
568
- ```ruby
569
- edge_stream thinking: true, intermediate: false
570
- ```
571
-
572
- `thinking` is the provider's reasoning; `intermediate` is the model narrating its
573
- own tool loop ("let me look that up"). Both are always on the event stream for the
574
- Studio and the trace — this decides only whether `/v1/responses` translates them,
575
- and each opted-in channel gets its own frame type, never the answer's. See
576
- [Architecture](ARCHITECTURE.md#what-crosses-the-edge).
577
-
578
- > ⚠️ Turn it on knowing your consumer. One that concatenates every text delta into
579
- > a single message — a WhatsApp adapter — will only be affected once it learns to
580
- > read the new frames, and when it does, the deliberation is what the customer
581
- > reads. That is the operator's call, which is why it is neither a default nor a
582
- > global.
583
-
584
- ## The stuck signal — "I cannot proceed" (WS5)
406
+ ## The stuck signal "I cannot proceed"
585
407
 
586
408
  The engine doesn't decide what "I can't help you" means — the consumer does. What
587
409
  the engine provides is the deterministic signal, so that a product wanting
@@ -677,11 +499,36 @@ writes nothing to the sessions it read), filters every proposal through the
677
499
  negative list and the evidence ledger (product claims must reference IDs the
678
500
  origin sessions actually saw — an agent without `grounding.matcher.sku` does
679
501
  not mine at all), scores survivors with a double gate (the eval replay against
680
- the clone's golden set, judges mandatory in the three P18 shapes; the
681
- conversion "not worse" check against the RFC-0032 frozen baseline), and lands
502
+ the clone's golden set, judges mandatory in three shapes; the
503
+ conversion "not worse" check against the frozen funnel baseline), and lands
682
504
  a skill **only after a human approves** — snapshot-first, append-only
683
505
  promotion log, deterministic rollback. Nothing is ever applied automatically.
684
506
 
507
+ ## Knowledge from finished conversations
508
+
509
+ `knowledge` configures the post-turn learning loop documented in
510
+ [Knowledge](KNOWLEDGE.md): after a turn completes, the engine extracts durable
511
+ **concepts** — facts, procedures, policies, objections — and persists them for
512
+ the agent, separate from any one customer's memory. Pack data, `distill:`'s
513
+ shape, and absent = off for that agent:
514
+
515
+ ```ruby
516
+ knowledge extract: true,
517
+ types: %w[fact policy objection] # what the extractor may emit
518
+ # prompt: "<what counts as a concept for THIS store>" (the forge's half)
519
+ # model: "<ref — absent = the platform utility_model>"
520
+ ```
521
+
522
+ The model proposes `name`/`description`/`type`/`body`; the engine stamps
523
+ `provenance`/`confidence`/`sources`/timestamps itself (a model-authored one
524
+ of those is dropped, never trusted) and redacts the body for PII. Writing a
525
+ concept name that already exists never blindly overwrites: the engine
526
+ decides same claim (bump the evidence), related claim (merge, one extra
527
+ model call), or contradicting claim (never merged — appended under a
528
+ heading, confidence drops, a human resolves it in the Studio's Knowledge
529
+ page). Nothing is injected into a turn's prompt yet — see
530
+ [Knowledge](KNOWLEDGE.md#whats-not-here-yet) for what's not here.
531
+
685
532
  ## Delegation (subagents)
686
533
 
687
534
  An agent can delegate to **subagents**: named child agents it may invoke as a
@@ -728,89 +575,11 @@ next turn with no restart. See [Deploy](DEPLOY.md) for the durable-volume setup
728
575
  [Context](CONTEXT.md#the-volume) for why editing a committed file does *not* change
729
576
  a running agent.
730
577
 
731
- ## Media in the message contract (WS9)
732
-
733
- The engine transports media, it never means it. The message accepts additive
734
- **content parts** alongside the text — voice notes and photos travel, and any
735
- skill (a fitting room, an image QA) stays a consumer layer on top:
736
-
737
- ```bash
738
- curl -X POST /v1/messages?stream=false -H "Authorization: Bearer $TOKEN" \
739
- -d '{ "agent": "store-support", "session_id": "chat-7",
740
- "message": "", "parts": [
741
- { "type": "audio", "url": "https://cdn.example.com/voz.ogg" },
742
- { "type": "image", "url": "https://cdn.example.com/sofa.jpg" }
743
- ] }'
744
- ```
745
-
746
- - **Audio** is transcribed (RubyLLM STT; model via `INSIKA_STT_MODEL`) and the
747
- text enters the turn marked `source: "voice"` on the terminal event — the
748
- consumer's signal the person spoke. A consumer that transcribes itself can
749
- send the text with `"source": "voice"` directly.
750
- - **Images** attach to the model's ask (vision); the provider bills them and
751
- the usage flows like any ask. The first image URL is also
752
- `{{ctx.image_url}}` for data tools — photo analysis outside the prompt, the
753
- tool's own egress applying when it fetches. Media URLs (audio AND image) are fetched by
754
- the engine through the same egress guard (a private/metadata target is
755
- refused — SSRF) and the same size ceiling (1 MB audio, 5 MB image: the bytes
756
- land in this process). A refused, oversized or unreadable part fails the turn
757
- loudly at the `:media` stage, never a silent drop.
758
- - **Media alone is a turn.** A voice note with no caption is `parts` and an
759
- empty `message` — the transcription becomes the message at the `:media`
760
- stage. A media message never joins another turn (`collect`/`steer` move text
761
- only, and the parts would be left behind), and a transcription that comes
762
- back empty fails the turn instead of asking the model about nothing.
763
- - **Parts are contract at the edge** — a malformed part (unknown type, an
764
- image/audio without `url`, a text without `text`) is a 422 before dispatch
765
- on `/v1/messages` and `/v1/responses`.
766
- - `/v1/responses` accepts the OpenAI multimodal shape: `input` as an array of
767
- text/image/audio parts.
768
-
769
- ## Generated media as outputs (WS9, saída)
770
-
771
- The turn can **produce** an image or a voice clip — but only when both sides of
772
- the gate agree, because nothing leaks by default. The agent declares it may
773
- generate media (`outputs` on the profile), and the **channel** declares it can
774
- receive it (`channel.capabilities` on the request):
578
+ ## Media
775
579
 
776
- ```bash
777
- curl -X POST /v1/responses -H "Authorization: Bearer $TOKEN" -d '{
778
- "model": "openclaw:store-support", "user": "chat-7",
779
- "input": "manda a foto do sofá da promoção",
780
- "channel": { "capabilities": ["image_output", "audio_output"] }
781
- }'
782
- ```
580
+ Photos, voice notes and documents travel through the message contract as additive content parts, and the engine can hand generated images back the same way. See [Media](MEDIA.md).
783
581
 
784
- ```ruby
785
- agent = Insika.agent("store-support") do
786
- instructions "…"
787
- outputs image: { model: "gpt-image-1", size: "1024x1024" }, # the AGENT's half
788
- tts: { model: "tts-1", voice: "alloy" }
789
- end
790
- ```
791
-
792
- - **Both gates** must pass for the model to even see the `generate_image` /
793
- `tts` tools: the agent opted in (`outputs`) and the request declared the
794
- matching capability (`image_output` / `audio_output` — an unknown value is a
795
- 422, never a silent ignore). The "abstraction admits only what leaks" rule.
796
- - **The media rides the envelope, never the answer text.** The terminal event
797
- and the `/v1/responses` completed frame carry an additive `output_parts`
798
- array — `{ type: "image", mime_type:, base64:, model: }` /
799
- `{ type: "audio", mime_type:, base64:, model: }`. The model's prose stays
800
- the `:content` answer; the channel consumes the bytes next to it.
801
- - **Generation is billed and counted.** Image tokens join the turn's usage
802
- (like any ask). The speech API reports no token counts, so a TTS call adds
803
- an honest `usage.media` counter and the part carries the `model` for
804
- consumer-side pricing.
805
- - **Seams, not magic.** The generator is injectable per kind (specs stub it);
806
- the defaults are lazy: images via RubyLLM (paint), speech via a thin POST to
807
- the OpenAI-compatible `/audio/speech` endpoint using the same provider
808
- config the chat uses — RubyLLM as of 1.16.0 has no speech API. A generated
809
- part over 8 MB refuses loudly, never silently truncates.
810
- - **Not here:** what the generated image *means* — a fitting room, a product
811
- mockup — is a skill on top. The engine transports bytes and cost.
812
-
813
- ## Customer-scoped memory and the right to be forgotten (WS8)
582
+ ## Customer-scoped memory and the right to be forgotten
814
583
 
815
584
  Memory is naturally **per customer, not per tenant**. A message that carries a
816
585
  `customer` key moves the engine's memory scope to that person:
@@ -842,11 +611,12 @@ curl -X POST /v1/messages?stream=false -H "Authorization: Bearer $TOKEN" \
842
611
  purges EVERYTHING the engine holds about one tenant: its sessions and their
843
612
  whole footprint (traces, tasks, checkpoints, outbox deliveries), every memory
844
613
  cell under the tenant (its own + the customer cells — enumerated from the
845
- store, so even a cell whose session was already deleted goes) and its outcome
846
- records: `{ "tenant": "acme" }`. Its API tokens are **revoked first** (before
847
- the sweep): an offboarded tenant whose credentials still resolved kept
848
- authenticating and could open a new session over the erasure. The tenant
849
- string is the isolation boundary; a neighbour is untouched.
614
+ store, so even a cell whose session was already deleted goes), its outcome
615
+ records and its artifacts (a report is content, never kept behind an
616
+ offboarded tenant): `{ "tenant": "acme" }`. Its API tokens are **revoked
617
+ first** (before the sweep): an offboarded tenant whose credentials still
618
+ resolved kept authenticating and could open a new session over the erasure.
619
+ The tenant string is the isolation boundary; a neighbour is untouched.
850
620
  - **Retention** — the age-based counterpart, as data: the settings key
851
621
  `retention_days` (Integer days; absent/0 = OFF, the engine never sweeps by
852
622
  default). The tick's daily sweep (at most once per 24 h, behind the same
@@ -857,135 +627,56 @@ curl -X POST /v1/messages?stream=false -H "Authorization: Bearer $TOKEN" \
857
627
  still owed to somebody. One thing the same daily pass sweeps **regardless of
858
628
  `retention_days`**: the budget counter cells whose window already rolled over
859
629
  (and their once-per-window alert markers). Those are engine bookkeeping, not
860
- customer content, and nothing else ever collected them.
630
+ customer content, and nothing else ever collected them. Artifacts (reports)
631
+ likewise expire on their **own** knob, `artifact_ttl_days` (settings; absent
632
+ = OFF) — the guarantee that PII inside a report dies even in a deployment
633
+ that keeps its conversations forever. See [Artifacts](ARTIFACTS.md).
861
634
 
862
- ## Outcomes business results over real traffic (WS7)
635
+ ## Outcomes and follow-ups
863
636
 
864
- The engine measures what it is told to measure. The operator or the integration
865
- records a conversation's business outcome after the fact — `conversion`,
866
- `escalation`, `deflected`, anything, optionally with a monetary `value`:
637
+ What the agent's traffic was worth in business terms, and the tool that lets it come back later on a promise it made. See [Outcomes and follow-ups](OUTCOMES.md).
867
638
 
868
- ```bash
869
- curl -X POST /v1/outcomes -H "Authorization: Bearer $TOKEN" \
870
- -d '{ "agent": "store-support", "session_id": "chat-7",
871
- "outcome": "conversion", "value": 129.9 }'
872
- ```
873
-
874
- The endpoint is **additive and outside the response contract** — the turn never
875
- knows or cares; the engine transports the outcome and never interprets it (what
876
- "conversion" means is yours). Records are tenant-stamped (a tenant principal
877
- writes and reads only its own), and `GET /v1/outcomes?agent=` serves the last
878
- outcome per agent plus the per-day series — the last-outcome pill on the Studio
879
- agent grid, and the per-day series on the agent detail.
880
-
881
- ### The outcome funnel (RFC-0032)
882
-
883
- A store's funnel is pack data on the agent — the engine folds WS7 outcomes into
884
- the **declared** stages, and never hard-codes one itself (the stage vocabulary
885
- is the forge's):
886
-
887
- ```ruby
888
- agent = Insika.agent("store-support") do
889
- instructions "…"
890
- funnel stages: %w[greeted qualified cart paid],
891
- advance_on: { "abandoned_cart" => "cart", "pix_paid" => "paid" },
892
- primary: "paid", attribution_window: "72h"
893
- end
894
- ```
639
+ ## Schedules — recurring turns the engine fires
895
640
 
896
- The fold contract:
897
-
898
- - **Tick-driven, cumulative event counts on the declared order.** An outcome of
899
- kind K means the session *reached* `advance_on[K]`; the fold increments
900
- `stages[0..index]` for the reached stage. A per-stage-complete integration
901
- and a terminal-event integration therefore produce identical counts a
902
- session that paid also emitted the earlier events. A duplicate event
903
- double-counts (the integration's defect, not the engine's); **do not declare
904
- a stage off the linear path** (a "handoff" stage would be inflated by every
905
- later event). Counts are **event counts, not distinct sessions** — the
906
- baseline is events-based.
907
- - **Idempotent**: a per-pair `{at, ids}` cursor inside one transaction; a crash
908
- mid-fold never double counts, and a second pass folds only what is new.
909
- - **The attribution window is carried data, never computed** — `72h` is
910
- validated, rendered, and copied into the baseline snapshot; causal
911
- attribution stays human.
912
- - **The baseline freeze** (Studio > Funnel, or `:freeze_funnel_baseline` on the
913
- bus) sums the folded cells over a span of **≥ 28 days** (shorter spans are
914
- refused) into one current snapshot per `(tenant, agent)` — the number
915
- RFC-0033 (follow-up A/B) and RFC-0035 (promotion gate) compare against.
916
- - **Malformed declarations never crash the tick**: the fold skips them, the
917
- doctor names the defect, the Studio shows nothing until it is fixed.
918
- - Vocabulary note: in the gem this is the **outcome funnel** — the stage names
919
- are the forge's, and a bare install (no `funnel:` on any agent) shows no
920
- funnel and no stage names at all.
921
-
922
- ## Follow-ups — the seller who comes back (RFC-0033)
923
-
924
- The agent can book a follow-up with a customer at a future time — "te chamo
925
- amanhã se o PIX não cair" said in-conversation and meant. The engine fires the
926
- synthetic turn on its own tick, with consent and without spam. Everything is
927
- pack data on the profile:
641
+ The operator's counterpart to follow-ups: a turn **nobody sends** — the daily
642
+ report at 22:00, the eval sweep every night. Declared per agent as pack data
643
+ (`schedule "daily_report", cron: …, tz: …, message: …` or `every: N`), edited
644
+ hot in the Studio's Schedules config group, fired by the engine's own tick
645
+ one turn per claim window, never queued, no catch-up after a downtime (each
646
+ missed window is a recorded skip, visible in the Studio):
928
647
 
929
648
  ```ruby
930
- agent = Insika.agent("store-support") do
931
- instructions ""
932
- followup arm: "schedule",
933
- policy: { quiet_hours: { timezone: "America/Sao_Paulo",
934
- start: "21:30", end: "09:00" },
935
- max_frequency: "2/24h", # N outbound per window, per customer
936
- cancel_keywords: ["não quero mais contato"],
937
- silence_after_sends: 3 } # N fires without a reply -> :unavailable
649
+ agent = Insika.agent("reporter") do
650
+ schedule "daily_report", cron: "0 22 * * *", tz: "America/Sao_Paulo",
651
+ message: "Run the daily report now.",
652
+ overrides: { turn_timeout: 900, max_tool_calls: 200 }
938
653
  end
939
654
  ```
940
655
 
941
- The pieces:
942
-
943
- - **`schedule(at:, reason:)`** a built-in tool the agent calls when the
944
- customer agrees to be contacted again (a product, a cart, a pending payment).
945
- The call itself IS the consent record recorded without ever lifting
946
- `:unavailable` or resetting the silence counter (ONLY a customer message
947
- reopens, so a re-booking inside a follow-up turn cannot clear the silence
948
- protection). `cancel_followup(id:)` is the sibling. A customer who opted out
949
- can never be rescheduled.
950
- - **Contact state per customer** `granted | revoked | unavailable` in a
951
- durable cell per `(tenant, customer)`. Only `granted` may be messaged;
952
- `revoked` is immediate and permanent until the customer speaks again;
953
- `unavailable` means silence refusal the engine stops firing after
954
- `silence_after_sends` unanswered sends, and ANY customer message reopens.
955
- The policy's `cancel_keywords` are matched on every inbound message: a
956
- match revokes the contact and cancels its pending follow-ups in one
957
- transaction.
958
- - **Firing is the tick's third duty** the engine claims the due records
959
- (one per claim window, at-most-once across workers), applies the policy in
960
- force AT FIRE TIME (contact state, quiet hours, dedup per
961
- `(customer, reason)`, frequency ceiling) and either enqueues the synthetic
962
- turn or marks the record `blocked` with the failing rule — auditable, never
963
- silent. Blocking happens at fire time, never at schedule time: the schedule
964
- is a promise made in-conversation, and only the policy in force then may
965
- revoke it.
966
- - **The synthetic turn** — a first-class inbound turn stamped
967
- `origin: "scheduled"` (a refinement read can never mistake the engine's
968
- kick for the customer repeating themselves), delivered through the full
969
- pipeline on the channel the conversation came in on. It skips the edge's
970
- ENTRY rate/token checks like a resume does — a follow-up she agreed to must
971
- not receive the rate-limit reply; its usage still lands on the ledger.
972
- - **The Follow-ups page** (Studio) — per agent: the pending/fired/cancelled/
973
- blocked records (blocked rows carry the reason), the read-only policy
974
- summary and the A/B card: per arm, `sent` vs `conversions` (against the
975
- RFC-0032 baseline) vs `opt-outs`. The only mutations — cancel a pending
976
- record, force-revoke a contact — go through bus commands.
977
- - **LGPD** — the records and cells die with the customer (`forget_customer`),
978
- the tenant (`delete_tenant_data`) and age out under the same
979
- `retention_days` sweep as the rest of the footprint.
980
-
981
- Absent `followup:` = the feature is off for that agent — no tools wired, no
982
- records, byte-identical turns. The A/B against an existing cron is an
983
- operator experiment: the engine only keeps the records and the read card (the
984
- cron arm writes through the same store class with its own `arm` label).
656
+ The run is a first-class turn stamped `origin: "scheduled"`; `session_mode:
657
+ "new"` gives it a fresh session per run (the report shape), `"fixed"` a
658
+ standing one; per-run `overrides` raise the chat-time ceilings a report needs.
659
+ A hard calendar budget at its cap skips instead of burning the store's tokens.
660
+ Distinct by shape and by law from the follow-up tool. See
661
+ [Schedules](SCHEDULING.md).
662
+
663
+ ## Artifacts a report the agent can hand you a URL to
664
+
665
+ A scheduled report's natural output is not a message but a page — tables,
666
+ sections, inline charts. The `save_artifact` tool (a registry tool, in the
667
+ agent's `tools_allow`) gives the agent a durable destination: it hands in
668
+ title + content and gets a URL back, which it can include in a channel message
669
+ ("today's report: <url>"). The report is stored (one per run, the listing is
670
+ the history), served under `/studio/artifacts/…` inside a sandboxed iframe,
671
+ optionally shared outside the Studio via an expiring signed link
672
+ (`INSIKA_ARTIFACT_SIGNING_KEY`). The tenant binding is inherited from the
673
+ saving agent never a parameter the model types. Artifact content is LLM
674
+ output and is served as untrusted. See [Artifacts](ARTIFACTS.md).
985
675
 
986
676
  ## See also
987
677
 
988
678
  - [Tools](TOOLS.md) — define, register, and troubleshoot tools.
679
+ - [Artifacts](ARTIFACTS.md) — the report destination: the tool, the routes, the signed link.
989
680
  - [Skills](SKILLS.md) — progressive playbooks an agent loads on demand.
990
681
  - [Context](CONTEXT.md) — what fills a turn's prompt, and memory.
991
682
  - [Security](SECURITY.md) — guardrails, sandbox, approvals, edge limits.
data/docs/API.md CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  title: The /v1 API
3
- parent: Build an agent
4
- nav_order: 9
3
+ parent: Integrate
4
+ nav_order: 1
5
5
  permalink: /api/
6
6
  ---
7
7
 
@@ -9,7 +9,7 @@ permalink: /api/
9
9
 
10
10
  The HTTP surface is the drop-in OpenAI-Responses-compatible contract: a client
11
11
  that speaks `/v1` today speaks it tomorrow. This page is the WRITTEN promise
12
- (RFC-0036 C6) — the mechanical half is the version gate in the server, and the
12
+ — the mechanical half is the version gate in the server, and the
13
13
  two cannot drift (a spec pins the gate's version to the date below).
14
14
 
15
15
  ## The surface
@@ -38,8 +38,8 @@ field is a BREAKING change and requires, together, all three:
38
38
 
39
39
  1. a new `Insika-Version` entry in the server's known-versions gate (an
40
40
  unknown value is refused with 400 before auth — fail-closed);
41
- 2. a route branch that serves both vintages (a compatibility branch, per
42
- RFC-0016 A5), so an old client keeps working while the new shape rolls;
41
+ 2. a route branch that serves both vintages (a compatibility branch), so
42
+ an old client keeps working while the new shape rolls;
43
43
  3. a new `**Frozen as of:**` line below.
44
44
 
45
45
  Additive changes need none of the three.
data/docs/ARCHITECTURE.md CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  title: Architecture
3
- parent: Understand the idea
4
- nav_order: 2
3
+ parent: Start here
4
+ nav_order: 3
5
5
  permalink: /architecture/
6
6
  ---
7
7
 
@@ -319,6 +319,7 @@ validator as the after-task hook, so both roots enforce content safety identical
319
319
  | Tools (data/manifest/MCP) | `lib/insika/tool_definition.rb`, `tool_manifest.rb`, `mcp_tool_ingestor.rb` |
320
320
  | Plugin loading (boot) | `lib/insika/plugin.rb`, `lib/insika/plugin/loader.rb` |
321
321
  | Refinement (traffic → report) | `lib/insika/refinement/*`, `lib/insika/refinement_store.rb` |
322
+ | Post-turn learning (facts, skills, knowledge — extracted from finished conversations) | `lib/insika/distill.rb`, `lib/insika/harvest.rb`, `lib/insika/knowledge.rb`, `lib/insika/knowledge_store.rb`; the per-turn hook lives in `Executor#persist_turn`, next to `finalize_delegation` |
322
323
  | Evals (cases, judges, gate) | `lib/insika/evals/*`, `lib/insika/golden_store.rb`; `evals/run.rb` is the CLI |
323
324
  | HTTP/SSE surface | `lib/insika/server/*` |
324
325