insika 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (280) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +199 -5
  3. data/README.md +8 -2
  4. data/bin/insika +231 -13
  5. data/docs/AGENTS.md +505 -6
  6. data/docs/API.md +56 -0
  7. data/docs/CHANNELS.md +100 -10
  8. data/docs/CONTEXT.md +147 -19
  9. data/docs/DEPLOY.md +34 -11
  10. data/docs/EMBEDDING.md +11 -7
  11. data/docs/EVALS.md +20 -1
  12. data/docs/FACTS.md +135 -0
  13. data/docs/HARVEST.md +117 -0
  14. data/docs/LOADTEST.md +17 -10
  15. data/docs/OBSERVABILITY.md +65 -2
  16. data/docs/REFINEMENT.md +9 -9
  17. data/docs/RELEASING.md +34 -7
  18. data/docs/RUNNING-LOCAL.md +4 -4
  19. data/docs/SECURITY.md +85 -11
  20. data/docs/SKILLS.md +189 -3
  21. data/docs/SOAK.md +127 -0
  22. data/docs/TOOLS.md +70 -2
  23. data/docs/WHY.md +1 -1
  24. data/docs/WORKFLOWS.md +2 -2
  25. data/docs/domain.md +115 -0
  26. data/docs/index.md +2 -2
  27. data/docs/onboarding/start.md +1 -1
  28. data/lib/insika/agent_profile.rb +228 -26
  29. data/lib/insika/alert_dispatcher.rb +139 -0
  30. data/lib/insika/balloon_splitter.rb +102 -0
  31. data/lib/insika/baseline_store.rb +2 -2
  32. data/lib/insika/budget_ledger.rb +166 -0
  33. data/lib/insika/cache_series_store.rb +49 -0
  34. data/lib/insika/channel_delivery.rb +132 -24
  35. data/lib/insika/channel_registry.rb +1 -1
  36. data/lib/insika/channels/relay.rb +80 -6
  37. data/lib/insika/channels/web/widget.js +2 -2
  38. data/lib/insika/channels/web.rb +9 -9
  39. data/lib/insika/channels/webhook.rb +58 -0
  40. data/lib/insika/chat_builder.rb +145 -13
  41. data/lib/insika/checkpoint_store.rb +16 -0
  42. data/lib/insika/circuit_state.rb +114 -0
  43. data/lib/insika/coercion.rb +8 -0
  44. data/lib/insika/commands/agent_payload.rb +6 -4
  45. data/lib/insika/commands/cancel_followup.rb +49 -0
  46. data/lib/insika/commands/create_agent.rb +2 -2
  47. data/lib/insika/commands/create_session.rb +1 -1
  48. data/lib/insika/commands/delete_llm_provider.rb +1 -1
  49. data/lib/insika/commands/delete_skill.rb +43 -0
  50. data/lib/insika/commands/delete_tenant_data.rb +95 -0
  51. data/lib/insika/commands/export_customer_memory.rb +48 -0
  52. data/lib/insika/commands/forget_customer.rb +117 -0
  53. data/lib/insika/commands/freeze_funnel_baseline.rb +113 -0
  54. data/lib/insika/commands/gate_harvest.rb +138 -0
  55. data/lib/insika/commands/gate_refinement.rb +12 -12
  56. data/lib/insika/commands/import_mcp_tools.rb +1 -1
  57. data/lib/insika/commands/import_tools.rb +4 -4
  58. data/lib/insika/commands/issue_tenant_token.rb +41 -0
  59. data/lib/insika/commands/judge_shadow_pairs.rb +124 -0
  60. data/lib/insika/commands/memory_forget_fact.rb +20 -4
  61. data/lib/insika/commands/memory_put_fact.rb +23 -4
  62. data/lib/insika/commands/promote_harvest.rb +130 -0
  63. data/lib/insika/commands/record_outcome.rb +46 -0
  64. data/lib/insika/commands/record_shadow_reply.rb +68 -0
  65. data/lib/insika/commands/reject_harvest.rb +38 -0
  66. data/lib/insika/commands/resolve_proposal.rb +108 -0
  67. data/lib/insika/commands/resolve_refinement.rb +1 -1
  68. data/lib/insika/commands/revoke_contact.rb +49 -0
  69. data/lib/insika/commands/revoke_token.rb +39 -0
  70. data/lib/insika/commands/rollback_harvest.rb +86 -0
  71. data/lib/insika/commands/rotate_tenant_token.rb +43 -0
  72. data/lib/insika/commands/run_distillation.rb +186 -0
  73. data/lib/insika/commands/run_harvest.rb +393 -0
  74. data/lib/insika/commands/run_refinement.rb +5 -5
  75. data/lib/insika/commands/send_message.rb +112 -15
  76. data/lib/insika/commands/session_purge.rb +67 -0
  77. data/lib/insika/commands/set_agent_tools.rb +1 -1
  78. data/lib/insika/commands/set_skill_agents.rb +60 -19
  79. data/lib/insika/commands/trigger_workflow.rb +1 -1
  80. data/lib/insika/commands/update_agent.rb +1 -1
  81. data/lib/insika/commands/write_data_tool.rb +1 -1
  82. data/lib/insika/commands/write_golden.rb +1 -1
  83. data/lib/insika/commands/write_skill.rb +19 -9
  84. data/lib/insika/config_store.rb +8 -4
  85. data/lib/insika/contact_store.rb +183 -0
  86. data/lib/insika/context/builder.rb +23 -5
  87. data/lib/insika/context/fragment.rb +31 -3
  88. data/lib/insika/context/priority.rb +6 -2
  89. data/lib/insika/context/provider.rb +17 -3
  90. data/lib/insika/context/providers/briefing.rb +96 -0
  91. data/lib/insika/context/providers/memory.rb +16 -7
  92. data/lib/insika/context/providers/prompt.rb +30 -2
  93. data/lib/insika/context/providers/request.rb +1 -1
  94. data/lib/insika/context/providers/session.rb +17 -2
  95. data/lib/insika/context/providers/skill.rb +7 -1
  96. data/lib/insika/context/providers/skill_trigger.rb +128 -0
  97. data/lib/insika/context/providers/tool_search.rb +2 -0
  98. data/lib/insika/context_trace_store.rb +128 -0
  99. data/lib/insika/delegation_store.rb +2 -2
  100. data/lib/insika/distill.rb +224 -0
  101. data/lib/insika/distill_engine.rb +169 -0
  102. data/lib/insika/doctor.rb +962 -7
  103. data/lib/insika/dsl/runtime.rb +20 -11
  104. data/lib/insika/dsl/server_boot.rb +74 -4
  105. data/lib/insika/dsl/system.rb +1 -1
  106. data/lib/insika/dsl.rb +152 -15
  107. data/lib/insika/edge_limiter.rb +167 -8
  108. data/lib/insika/egress_guard.rb +3 -3
  109. data/lib/insika/env_schema.rb +22 -12
  110. data/lib/insika/errors.rb +72 -5
  111. data/lib/insika/evals/assertions.rb +15 -14
  112. data/lib/insika/evals/baseline.rb +3 -3
  113. data/lib/insika/evals/golden.rb +8 -8
  114. data/lib/insika/evals/judge.rb +7 -7
  115. data/lib/insika/evals/pairwise.rb +21 -9
  116. data/lib/insika/evals/report.rb +2 -2
  117. data/lib/insika/evals/runner.rb +6 -6
  118. data/lib/insika/evals/transport.rb +2 -2
  119. data/lib/insika/event_stream.rb +23 -5
  120. data/lib/insika/evidence.rb +183 -0
  121. data/lib/insika/executor.rb +1092 -160
  122. data/lib/insika/followup_engine.rb +207 -0
  123. data/lib/insika/followup_policy.rb +221 -0
  124. data/lib/insika/followup_store.rb +306 -0
  125. data/lib/insika/frontmatter.rb +1 -1
  126. data/lib/insika/funnel_declaration.rb +106 -0
  127. data/lib/insika/funnel_fold.rb +179 -0
  128. data/lib/insika/funnel_store.rb +163 -0
  129. data/lib/insika/golden_store.rb +3 -3
  130. data/lib/insika/grounding/matcher.rb +69 -0
  131. data/lib/insika/grounding.rb +44 -0
  132. data/lib/insika/harvest/conversion_gate.rb +159 -0
  133. data/lib/insika/harvest/criterion.rb +98 -0
  134. data/lib/insika/harvest/gate.rb +194 -0
  135. data/lib/insika/harvest/negative_list.rb +199 -0
  136. data/lib/insika/harvest.rb +241 -0
  137. data/lib/insika/harvest_engine.rb +193 -0
  138. data/lib/insika/harvest_store.rb +548 -0
  139. data/lib/insika/http_client.rb +3 -3
  140. data/lib/insika/inbound_log.rb +1 -1
  141. data/lib/insika/llm_configurator.rb +3 -3
  142. data/lib/insika/loop_detector.rb +143 -0
  143. data/lib/insika/mcp_http_client.rb +4 -4
  144. data/lib/insika/mcp_tool_ingestor.rb +6 -6
  145. data/lib/insika/media.rb +298 -0
  146. data/lib/insika/memory_audit_store.rb +85 -0
  147. data/lib/insika/memory_store.rb +264 -23
  148. data/lib/insika/message_origin.rb +8 -3
  149. data/lib/insika/model_resolver.rb +1 -1
  150. data/lib/insika/model_selection.rb +5 -4
  151. data/lib/insika/model_visible.rb +87 -0
  152. data/lib/insika/model_visible_trace_store.rb +66 -0
  153. data/lib/insika/onboarding.rb +8 -3
  154. data/lib/insika/outbox_store.rb +44 -6
  155. data/lib/insika/outcome_store.rb +147 -0
  156. data/lib/insika/overlay_tool_registry.rb +3 -4
  157. data/lib/insika/pack.rb +3 -3
  158. data/lib/insika/pack_importer.rb +17 -15
  159. data/lib/insika/packaging.rb +163 -0
  160. data/lib/insika/parity/criterion.rb +79 -0
  161. data/lib/insika/parity/verdict.rb +318 -0
  162. data/lib/insika/pending_action_store.rb +1 -1
  163. data/lib/insika/plugin/loader.rb +2 -2
  164. data/lib/insika/policy/policy.rb +1 -1
  165. data/lib/insika/prefix_fingerprint.rb +58 -0
  166. data/lib/insika/profile_source.rb +34 -7
  167. data/lib/insika/proposal_store.rb +271 -0
  168. data/lib/insika/provider_error_classifier.rb +160 -0
  169. data/lib/insika/queue_policy.rb +6 -3
  170. data/lib/insika/recovery.rb +47 -6
  171. data/lib/insika/refinement/candidate.rb +4 -4
  172. data/lib/insika/refinement/evidence_collector.rb +6 -6
  173. data/lib/insika/refinement/gate.rb +7 -7
  174. data/lib/insika/refinement/panel.rb +7 -7
  175. data/lib/insika/refinement/proposer.rb +10 -10
  176. data/lib/insika/refinement_store.rb +12 -12
  177. data/lib/insika/reliability.rb +211 -0
  178. data/lib/insika/retention.rb +281 -0
  179. data/lib/insika/routing.rb +101 -0
  180. data/lib/insika/safety/config.rb +46 -6
  181. data/lib/insika/safety/corpus.rb +255 -0
  182. data/lib/insika/safety/detectors.rb +34 -115
  183. data/lib/insika/safety/factory.rb +18 -5
  184. data/lib/insika/safety/grounding_enforcer.rb +59 -0
  185. data/lib/insika/safety/grounding_validator.rb +49 -0
  186. data/lib/insika/safety/input_guardrail.rb +20 -5
  187. data/lib/insika/safety/moderator.rb +19 -11
  188. data/lib/insika/safety/output_filter.rb +10 -6
  189. data/lib/insika/safety/output_validator.rb +13 -7
  190. data/lib/insika/safety/safe_responses.rb +1 -1
  191. data/lib/insika/sandbox/boundary.rb +2 -2
  192. data/lib/insika/sandbox.rb +1 -1
  193. data/lib/insika/schema_guard.rb +35 -0
  194. data/lib/insika/server/app.rb +366 -54
  195. data/lib/insika/server/boot.rb +4 -4
  196. data/lib/insika/server/rack_app.rb +31 -7
  197. data/lib/insika/server/responses.rb +58 -9
  198. data/lib/insika/server/tenant_auth.rb +61 -0
  199. data/lib/insika/session_actor.rb +11 -7
  200. data/lib/insika/session_store.rb +66 -3
  201. data/lib/insika/settings_store.rb +15 -5
  202. data/lib/insika/shadow_pair_store.rb +258 -0
  203. data/lib/insika/shutdown.rb +4 -4
  204. data/lib/insika/skill_catalog.rb +131 -20
  205. data/lib/insika/skill_store.rb +70 -22
  206. data/lib/insika/soak/envelope.rb +140 -0
  207. data/lib/insika/soak/report.rb +392 -0
  208. data/lib/insika/soak/runner.rb +554 -0
  209. data/lib/insika/steer_injector.rb +1 -1
  210. data/lib/insika/store.rb +11 -2
  211. data/lib/insika/stores/memory.rb +6 -0
  212. data/lib/insika/stores/sqlite.rb +8 -0
  213. data/lib/insika/studio/app.rb +1058 -75
  214. data/lib/insika/studio/assets/dist/application.css +1 -1
  215. data/lib/insika/studio/assets/dist/application.js +27 -26
  216. data/lib/insika/studio/assets/dist/favicon.svg +6 -0
  217. data/lib/insika/studio/forms.rb +274 -22
  218. data/lib/insika/studio/nav_icons.rb +7 -2
  219. data/lib/insika/studio/views/_message.erb +2 -2
  220. data/lib/insika/studio/views/agent_detail.erb +629 -86
  221. data/lib/insika/studio/views/agents.erb +11 -7
  222. data/lib/insika/studio/views/approvals.erb +4 -1
  223. data/lib/insika/studio/views/chats.erb +4 -1
  224. data/lib/insika/studio/views/customer.erb +94 -0
  225. data/lib/insika/studio/views/customers.erb +32 -0
  226. data/lib/insika/studio/views/evals.erb +4 -1
  227. data/lib/insika/studio/views/facts.erb +133 -0
  228. data/lib/insika/studio/views/followups.erb +125 -0
  229. data/lib/insika/studio/views/funnel.erb +106 -0
  230. data/lib/insika/studio/views/harvest.erb +234 -0
  231. data/lib/insika/studio/views/home.erb +2 -1
  232. data/lib/insika/studio/views/layout.erb +1 -0
  233. data/lib/insika/studio/views/parity.erb +147 -0
  234. data/lib/insika/studio/views/playground.erb +7 -1
  235. data/lib/insika/studio/views/refinement.erb +4 -4
  236. data/lib/insika/studio/views/session.erb +133 -3
  237. data/lib/insika/studio/views/settings.erb +9 -12
  238. data/lib/insika/studio/views/skills.erb +66 -12
  239. data/lib/insika/studio/views/system_files.erb +1 -1
  240. data/lib/insika/studio/views/task.erb +13 -0
  241. data/lib/insika/studio/views/tasks.erb +4 -1
  242. data/lib/insika/studio/views/tools.erb +0 -1
  243. data/lib/insika/subagent_graph.rb +3 -3
  244. data/lib/insika/task_actor.rb +3 -3
  245. data/lib/insika/task_store.rb +22 -2
  246. data/lib/insika/telemetry/pricing.rb +3 -3
  247. data/lib/insika/telemetry/recorder.rb +1 -1
  248. data/lib/insika/telemetry.rb +2 -2
  249. data/lib/insika/testing/store_contract.rb +54 -33
  250. data/lib/insika/tick.rb +146 -0
  251. data/lib/insika/token_store.rb +168 -0
  252. data/lib/insika/tool_assembly.rb +5 -5
  253. data/lib/insika/tool_definition.rb +25 -15
  254. data/lib/insika/tool_envelope.rb +70 -1
  255. data/lib/insika/tool_manifest.rb +11 -7
  256. data/lib/insika/tool_output_compressor.rb +100 -0
  257. data/lib/insika/tool_store.rb +1 -1
  258. data/lib/insika/tool_trace_store.rb +1 -1
  259. data/lib/insika/tools/concurrency.rb +2 -2
  260. data/lib/insika/tools/data_defined_tool.rb +14 -5
  261. data/lib/insika/tools/generate_image.rb +44 -0
  262. data/lib/insika/tools/load_skill.rb +61 -3
  263. data/lib/insika/tools/schedule_followup.rb +164 -0
  264. data/lib/insika/tools/stuck_signal.rb +44 -0
  265. data/lib/insika/tools/subagent.rb +4 -4
  266. data/lib/insika/tools/subagents.rb +1 -1
  267. data/lib/insika/tools/tts.rb +47 -0
  268. data/lib/insika/tools/update_briefing.rb +126 -0
  269. data/lib/insika/turn_output.rb +2 -2
  270. data/lib/insika/turn_state.rb +54 -13
  271. data/lib/insika/turn_timing.rb +24 -4
  272. data/lib/insika/usage_ledger.rb +1 -1
  273. data/lib/insika/version.rb +1 -1
  274. data/lib/insika/vitals.rb +84 -0
  275. data/lib/insika/wiring/graph.rb +372 -34
  276. data/lib/insika/workflow.rb +1 -1
  277. data/lib/insika/workflow_registry.rb +1 -1
  278. data/lib/insika.rb +122 -16
  279. metadata +95 -2
  280. data/lib/insika/server/admin_auth.rb +0 -29
@@ -0,0 +1,271 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "securerandom"
4
+ require "time"
5
+
6
+ module Insika
7
+ # the proposals and the two persisted mechanisms the gates
8
+ # on — the **latched dedup ledger** (D3: the rows themselves ARE the ledger —
9
+ # a dismissed/rejected tuple is never proposed again, and an unanswered
10
+ # proposal is not piled on) and the **per-session distilled marker** (D2:
11
+ # written only after a pass completes, so a crash mid-pass leaves the marker
12
+ # unwritten and the next pass re-scans). A dumb domain store — it holds no
13
+ # policy (which tuple is a fact is the distiller's job), no memory facts and
14
+ # no model. The scope string (the memory cell) is built by the callers from
15
+ # the `MemoryStore::parse_cell` shape; the store keys by
16
+ # `(tenant, customer)` explicitly.
17
+ #
18
+ # Statuses: pending -> approved | rejected | dismissed | stale.
19
+ # `stale` is the CAS-lost re-present (E3): the proposal carries the fact's
20
+ # CURRENT value (`current_value`) next to the proposed one, never a silent
21
+ # overwrite.
22
+ class ProposalStore
23
+ SCOPE = "proposals"
24
+ STATUSES = %w[pending approved rejected dismissed stale].freeze
25
+ TERMINAL = %w[approved rejected dismissed].freeze
26
+ PROPOSAL_PREFIX = "p:"
27
+ MARKER_PREFIX = "s:"
28
+
29
+ Proposal = Data.define(:id, :tenant, :customer, :scope, :session_ref, :key,
30
+ :value, :confidence, :status, :evidence,
31
+ :expected_revision, :expected_existed, :current_value,
32
+ :operator, :note, :created_at, :updated_at)
33
+
34
+ def initialize(store:)
35
+ @store = store
36
+ end
37
+
38
+ # -> Proposal (status :pending). The caller (RunDistillation) already ran
39
+ # the dedup checks; the store writes. `evidence` = message indexes; the
40
+ # revision baseline (D5) travels with the record.
41
+ #
42
+ # The tenant is stored VERBATIM — nil in a single-tenant deployment, so
43
+ # the scope is the bare `customer` cell and the approval reads/writes the
44
+ # SAME cell the Memory provider injects (memory_store.rb's
45
+ # blank-tenant + customer -> "memory:<customer>" rule). Coercing a blank
46
+ # tenant to a sentinel here would orphan every approved fact in a
47
+ # phantom "memory:platform:<customer>" cell.
48
+ def create(tenant:, customer:, session_ref:, key:, value:, confidence: nil,
49
+ evidence: [], expected_revision: nil, expected_existed: false,
50
+ id: SecureRandom.uuid, now: Time.now.utc)
51
+ tenant = tenant_key(tenant)
52
+ stamp = now.iso8601(6)
53
+ record = { "id" => id.to_s, "status" => "pending",
54
+ "tenant" => tenant, "customer" => customer.to_s,
55
+ "scope" => [tenant, customer.to_s].compact.join(":"),
56
+ "session_ref" => session_ref.to_s, "key" => key.to_s,
57
+ "value" => value.to_s, "confidence" => confidence,
58
+ "evidence" => Array(evidence).map(&:to_i),
59
+ "expected_revision" => expected_revision,
60
+ "expected_existed" => !!expected_existed,
61
+ "current_value" => nil, "operator" => nil, "note" => nil,
62
+ "created_at" => stamp, "updated_at" => stamp }
63
+ @store.set(SCOPE, PROPOSAL_PREFIX + id.to_s, record)
64
+ to_proposal(record)
65
+ end
66
+
67
+ def find(id)
68
+ record = @store.get(SCOPE, PROPOSAL_PREFIX + id.to_s)
69
+ record && to_proposal(record)
70
+ end
71
+
72
+ # The wiki's lists. `pending` = pending, oldest first (the operator works
73
+ # the oldest proposal first — evidence ages).
74
+ def pending(limit: 100)
75
+ scan.select { |p| p.status == "pending" }
76
+ .sort_by { |p| [p.created_at.to_s, p.id] }
77
+ .first(limit)
78
+ end
79
+
80
+ def stale(limit: 50)
81
+ scan.select { |p| p.status == "stale" }
82
+ .sort_by { |p| p.updated_at.to_s }
83
+ .first(limit)
84
+ end
85
+
86
+ # The wiki's Recent list: every terminal status (approved/rejected/
87
+ # dismissed), most recent first — the operator's audit trail.
88
+ def resolved(limit: 20)
89
+ scan.select { |p| TERMINAL.include?(p.status) }
90
+ .sort_by { |p| p.updated_at.to_s }
91
+ .reverse
92
+ .first(limit)
93
+ end
94
+
95
+ # ---- the latched dedup (D3) ----
96
+ # true when a dismissed/rejected row exists for the exact tuple — the
97
+ # latch. Persisted rows ARE the ledger. A *different* value for the same
98
+ # `name` is a different tuple.
99
+ def decided?(tenant:, customer:, key:, value:)
100
+ scan.any? do |p|
101
+ tenant_key(p.tenant) == tenant_key(tenant) && p.customer == customer.to_s &&
102
+ p.key == key.to_s && p.value == value.to_s &&
103
+ %w[dismissed rejected].include?(p.status)
104
+ end
105
+ end
106
+
107
+ # true when a pending row exists for (scope, key) — no piling.
108
+ def open_pending?(tenant:, customer:, key:)
109
+ scan.any? do |p|
110
+ tenant_key(p.tenant) == tenant_key(tenant) && p.customer == customer.to_s &&
111
+ p.key == key.to_s && p.status == "pending"
112
+ end
113
+ end
114
+
115
+ # ---- transitions, each read-check-write on @store.transaction ----
116
+ # pending -> terminal. ArgumentError for a wrong source state (the
117
+ # task_store.rb state-machine idiom).
118
+ def approve(id:, operator: nil, note: nil, now: Time.now.utc)
119
+ transition(id, "approved", operator: operator, note: note, now: now)
120
+ end
121
+
122
+ def reject(id:, operator: nil, note: nil, now: Time.now.utc)
123
+ transition(id, "rejected", operator: operator, note: note, now: now)
124
+ end
125
+
126
+ def dismiss(id:, operator: nil, note: nil, now: Time.now.utc)
127
+ transition(id, "dismissed", operator: operator, note: note, now: now)
128
+ end
129
+
130
+ # pending -> stale, CAS lost; `current_value` = the fact as it stands (the
131
+ # re-present's second value, E3).
132
+ def mark_stale(id:, current_value:, operator: nil, now: Time.now.utc)
133
+ transition(id, "stale", operator: operator, current_value: current_value, now: now)
134
+ end
135
+
136
+ # ---- the per-session marker (D2) ----
137
+ # Written ONLY after a pass completes (RunDistillation). -> the marker hash.
138
+ def mark_distilled(session_ref, agent:, proposals:, dropped:, deduped: 0, cost: nil, now: Time.now.utc)
139
+ marker = { "session_ref" => session_ref.to_s, "agent" => agent.to_s,
140
+ "distilled_at" => now.iso8601, "proposals" => proposals.to_i,
141
+ "dropped" => dropped, "deduped" => deduped.to_i,
142
+ "cost" => cost }
143
+ @store.set(SCOPE, MARKER_PREFIX + session_ref.to_s, marker)
144
+ marker
145
+ end
146
+
147
+ def distilled?(session_ref)
148
+ !@store.get(SCOPE, MARKER_PREFIX + session_ref.to_s).nil?
149
+ end
150
+
151
+ def distilled_sessions(agent_id = nil)
152
+ keys = agent_id ? marker_keys.select { |k| marker_agent(k) == agent_id.to_s } : marker_keys
153
+ keys.map { |k| k.delete_prefix(MARKER_PREFIX) }
154
+ end
155
+
156
+ # ---- LGPD / retention (C8) ----
157
+
158
+ # One customer's proposals, EVERY status. -> count removed.
159
+ def purge_customer(tenant:, customer:)
160
+ removed = 0
161
+ @store.transaction do
162
+ proposal_keys.each do |k|
163
+ record = @store.get(SCOPE, k)
164
+ next unless record && record["tenant"] == tenant_key(tenant)
165
+ next unless record["customer"] == customer.to_s
166
+
167
+ @store.delete(SCOPE, k)
168
+ removed += 1
169
+ end
170
+ end
171
+ removed
172
+ end
173
+
174
+ # A tenant's proposals. -> count removed.
175
+ def purge(tenant:)
176
+ removed = 0
177
+ @store.transaction do
178
+ proposal_keys.each do |k|
179
+ record = @store.get(SCOPE, k)
180
+ next unless record && record["tenant"] == tenant_key(tenant)
181
+
182
+ @store.delete(SCOPE, k)
183
+ removed += 1
184
+ end
185
+ end
186
+ removed
187
+ end
188
+
189
+ # Age-based prune (the retention sweep). TERMINAL rows age by their
190
+ # updated_at; a PENDING row is a zombie past the cutoff (its transcript is
191
+ # dead). Session MARKERS die WITH their proposals — a marker past the
192
+ # cutoff is evidence about a dead transcript (the session aged out under
193
+ # the same retention window), and keeping it would lock an unreviewed
194
+ # proposal out of re-distillation forever. -> count removed.
195
+ def delete_older_than(time)
196
+ cutoff = time.utc.iso8601
197
+ removed = 0
198
+ @store.transaction do
199
+ proposal_keys.each do |k|
200
+ record = @store.get(SCOPE, k)
201
+ next unless record
202
+
203
+ terminal = TERMINAL.include?(record["status"])
204
+ stamp = terminal ? record["updated_at"] : record["created_at"]
205
+ next unless stamp && stamp.to_s < cutoff
206
+
207
+ @store.delete(SCOPE, k)
208
+ removed += 1
209
+ end
210
+ marker_keys.each do |k|
211
+ marker = @store.get(SCOPE, k)
212
+ next unless marker && marker["distilled_at"].to_s < cutoff
213
+
214
+ @store.delete(SCOPE, k)
215
+ removed += 1
216
+ end
217
+ end
218
+ removed
219
+ end
220
+
221
+ private
222
+
223
+ def transition(id, to, operator: nil, note: nil, current_value: nil, now: Time.now.utc)
224
+ @store.transaction do
225
+ key = PROPOSAL_PREFIX + id.to_s
226
+ record = @store.get(SCOPE, key)
227
+ raise Insika::NotFoundError, "proposal not found: #{id}" if record.nil?
228
+
229
+ unless record["status"] == "pending"
230
+ raise ArgumentError,
231
+ "proposal #{id}: cannot resolve a #{record['status']} proposal — " \
232
+ "it resolves once, from pending"
233
+ end
234
+
235
+ record["status"] = to
236
+ record["operator"] = operator.to_s unless operator.nil?
237
+ record["note"] = note.to_s unless note.nil?
238
+ record["current_value"] = current_value unless current_value.nil?
239
+ record["updated_at"] = now.iso8601(6)
240
+ @store.set(SCOPE, key, record)
241
+ to_proposal(record)
242
+ end
243
+ end
244
+
245
+ def scan
246
+ proposal_keys.filter_map do |k|
247
+ record = @store.get(SCOPE, k)
248
+ record && to_proposal(record)
249
+ end
250
+ end
251
+
252
+ def proposal_keys = @store.list(SCOPE, PROPOSAL_PREFIX)
253
+ def marker_keys = @store.list(SCOPE, MARKER_PREFIX)
254
+
255
+ # nil stays nil (single-tenant); a present tenant is a String. The
256
+ # comparisons below use tenant_key on BOTH sides so nil == nil holds.
257
+ def tenant_key(tenant) = tenant.nil? ? nil : tenant.to_s
258
+
259
+ def to_proposal(rec)
260
+ Proposal.new(id: rec["id"], tenant: rec["tenant"], customer: rec["customer"],
261
+ scope: rec["scope"], session_ref: rec["session_ref"],
262
+ key: rec["key"], value: rec["value"], confidence: rec["confidence"],
263
+ status: rec["status"], evidence: rec["evidence"] || [],
264
+ expected_revision: rec["expected_revision"],
265
+ expected_existed: rec["expected_existed"] == true,
266
+ current_value: rec["current_value"], operator: rec["operator"],
267
+ note: rec["note"], created_at: rec["created_at"],
268
+ updated_at: rec["updated_at"])
269
+ end
270
+ end
271
+ end
@@ -0,0 +1,160 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Insika
4
+ # Classifies provider/transport failures by ACTION (B9). Four kinds —
5
+ # the hermes/openclaw structural rule: NON-retryable checked first, so an
6
+ # error we do not recognize defaults to :fatal (a retry would hammer a
7
+ # poisoned credential; a fatal is retried only after the operator fixes the
8
+ # cause):
9
+ #
10
+ # :fatal 401/402/403/400 (auth, billing, permanent quota,
11
+ # bad request, context too long) — retrying does not help.
12
+ # :retryable 5xx/529/socket/timeout — the same call may succeed
13
+ # moments later.
14
+ # :rate_limited_short a 429 that says "back off briefly" (RPM-scale).
15
+ # :rate_limited_long a 429 with a long retry-after — quota-scale.
16
+ #
17
+ # The classification is STRING-based (class names, no constant references):
18
+ # the core loads without ruby_llm, and the smoke-shim's fake RubyLLM is a
19
+ # drop-in. `retry_after` is read from the provider's own Retry-After header
20
+ # when the error carries a response, else a per-kind default.
21
+ class ProviderErrorClassifier
22
+ Classification = Data.define(:kind, :retryable, :retry_after) do
23
+ # the additive envelope fields — compacted so a retry_after-less fatal
24
+ # never invents one.
25
+ def to_h
26
+ { kind: kind, retryable: retryable, retry_after: retry_after }.compact
27
+ end
28
+ end
29
+
30
+ KINDS = %i[fatal retryable rate_limited_short rate_limited_long].freeze
31
+
32
+ # Above this a 429 means quota, not RPM (this is what tells the two apart —
33
+ # a short 429 wants a quick retry; a long one is a billing event).
34
+ SHORT_RETRY_LIMIT = 60 # seconds
35
+
36
+ DEFAULTS = {
37
+ retryable: 5,
38
+ rate_limited_short: 10,
39
+ rate_limited_long: 300
40
+ }.freeze
41
+
42
+ # RubyLLM's taxonomy (error.rb), matched by class name so the core stays
43
+ # ruby_llm-free at load time.
44
+ FATAL_ERROR_NAMES = %w[
45
+ RubyLLM::ContextLengthExceededError RubyLLM::BadRequestError
46
+ RubyLLM::UnauthorizedError RubyLLM::PaymentRequiredError RubyLLM::ForbiddenError
47
+ ].freeze
48
+ RATE_LIMITED_ERROR_NAME = "RubyLLM::RateLimitError"
49
+ RETRYABLE_ERROR_NAMES = %w[
50
+ RubyLLM::ServerError RubyLLM::ServiceUnavailableError RubyLLM::OverloadedError
51
+ ].freeze
52
+ RUBY_LLM_ERROR_NAMES = (
53
+ FATAL_ERROR_NAMES + [RATE_LIMITED_ERROR_NAME] +
54
+ RETRYABLE_ERROR_NAMES + ["RubyLLM::Error"]
55
+ ).freeze
56
+
57
+ # Transport failures while talking to the provider: connection
58
+ # refused/reset, DNS, TLS, timeouts (Faraday wraps its own names; the
59
+ # stdlib ones surface from raw sockets).
60
+ TRANSPORT_NAME_PATTERNS = [
61
+ /\AFaraday::/,
62
+ /\ASocketError\z/,
63
+ /\AIOError\z/,
64
+ /\AErrno::/,
65
+ /\ANet::(Read|Open)Timeout\z/,
66
+ /\ATimeout::Error\z/,
67
+ /\AOpenSSL::SSL::SSLError\z/
68
+ ].freeze
69
+
70
+ class << self
71
+ # -> Classification
72
+ def classify(error)
73
+ names = class_names(error)
74
+
75
+ # Non-retryable first (the structural rule): a known fatal is NEVER
76
+ # retried, and an unknown error defaults to fatal — never to retry.
77
+ return fatal if (names & FATAL_ERROR_NAMES).any?
78
+
79
+ return rate_limited(error) if names.include?(RATE_LIMITED_ERROR_NAME)
80
+
81
+ return retryable if (names & RETRYABLE_ERROR_NAMES).any?
82
+ return retryable if transport?(names)
83
+
84
+ # A generic RubyLLM::Error (or a raw HTTP error) still carries the
85
+ # status: 429 and 5xx are retryable regardless of the wrapping class.
86
+ case http_status(error)
87
+ when 429 then rate_limited(error)
88
+ when 500..599 then retryable
89
+ else fatal
90
+ end
91
+ end
92
+
93
+ # True when the error came from the provider call itself (RubyLLM
94
+ # family or transport) — the executor routes these to the :ruby_llm
95
+ # stage with a wrapped classification instead of :unknown.
96
+ def provider_error?(error)
97
+ names = class_names(error)
98
+ (names & RUBY_LLM_ERROR_NAMES).any? || transport?(names)
99
+ end
100
+
101
+ # The typed ProviderError the executor stores and emits.
102
+ def wrap(error)
103
+ c = classify(error)
104
+ Insika::ProviderError.new(
105
+ error.message || error.class.name,
106
+ kind: c.kind, retryable: c.retryable, retry_after: c.retry_after
107
+ )
108
+ end
109
+
110
+ private
111
+
112
+ def fatal
113
+ Classification.new(kind: :fatal, retryable: false, retry_after: nil)
114
+ end
115
+
116
+ def retryable
117
+ Classification.new(kind: :retryable, retryable: true,
118
+ retry_after: DEFAULTS[:retryable])
119
+ end
120
+
121
+ def rate_limited(error)
122
+ ra = retry_after_header(error)
123
+ if ra && ra > SHORT_RETRY_LIMIT
124
+ Classification.new(kind: :rate_limited_long, retryable: true, retry_after: ra)
125
+ else
126
+ Classification.new(kind: :rate_limited_short, retryable: true,
127
+ retry_after: ra || DEFAULTS[:rate_limited_short])
128
+ end
129
+ end
130
+
131
+ def class_names(error)
132
+ ([error.class.name] + Array(error.class.ancestors).map(&:name)).compact
133
+ end
134
+
135
+ def transport?(names)
136
+ names.any? { |n| TRANSPORT_NAME_PATTERNS.any? { |p| p.match?(n) } }
137
+ end
138
+
139
+ # The provider's own Retry-After (seconds), when the error carries a
140
+ # response. All access guarded — a bare double must not raise.
141
+ def retry_after_header(error)
142
+ headers = response_headers(error)
143
+ value = headers && (headers["retry-after"] || headers["Retry-After"])
144
+ value = value.to_s.strip
145
+ value.match?(/\A\d+\z/) ? value.to_i : nil
146
+ end
147
+
148
+ def http_status(error)
149
+ response = error.respond_to?(:response) ? error.response : nil
150
+ status = response && response.respond_to?(:status) ? response.status : nil
151
+ status&.to_i
152
+ end
153
+
154
+ def response_headers(error)
155
+ response = error.respond_to?(:response) ? error.response : nil
156
+ response && response.respond_to?(:headers) ? response.headers : nil
157
+ end
158
+ end
159
+ end
160
+ end
@@ -3,7 +3,7 @@
3
3
  require_relative "coercion"
4
4
 
5
5
  module Insika
6
- # RFC-0015 §4 — what happens to an inbound message for a session that is ALREADY
6
+ # what happens to an inbound message for a session that is ALREADY
7
7
  # busy. Today the engine has exactly one answer, "it waits in line"; this names
8
8
  # that answer `followup` and adds three others:
9
9
  #
@@ -32,7 +32,7 @@ module Insika
32
32
  # `turn_timeout` because they are bounds on the same thing — how much work one
33
33
  # turn is allowed to absorb.
34
34
  class QueuePolicy
35
- # All four of RFC-0015 are delivered, so there is no "specified but unshipped"
35
+ # All four of are delivered, so there is no "specified but unshipped"
36
36
  # tier any more — a mode outside this set is a typo, and it is refused rather than
37
37
  # approximated. Treating an unknown mode as `followup` would look exactly like a
38
38
  # mode that never fires.
@@ -85,7 +85,10 @@ module Insika
85
85
  def debounce? = @debounce_ms.positive?
86
86
 
87
87
  # Does this policy merge into a turn that has not started yet?
88
- def collect? = @mode == :collect
88
+ # True for :collect AND :steer — both absorb fragments that land before the
89
+ # turn starts. Debounce is what actually holds them; without a window a steer
90
+ # agent still steers mid-run and never waits at the door.
91
+ def collect? = @mode == :collect || @mode == :steer
89
92
 
90
93
  # Does this policy append into a turn that is already running? `steer_max_messages`
91
94
  # of 0 is the agent saying no, so it answers false rather than steering once and
@@ -17,7 +17,7 @@ module Insika
17
17
  class Recovery
18
18
  SWEEP_SCOPE = "recovery"
19
19
 
20
- # The per-boot-generation sweep claim (RFC-0016 E2). N workers share one
20
+ # The per-boot-generation sweep claim. N workers share one
21
21
  # store, and the sweep's "orphaned :running" test is per-process: a worker
22
22
  # booting while a sibling holds a live turn would see it as an orphan and
23
23
  # re-run it. So the TASK sweep runs once per boot generation — the first
@@ -52,27 +52,59 @@ module Insika
52
52
  # -> { resumed: [ids], failed: [ids] }
53
53
  # The initial sweep runs OUTSIDE the per-task rescue: a StoreError here
54
54
  # aborts the boot.
55
- def run
55
+ #
56
+ # stale_after (seconds): the periodic tick's semantics instead of
57
+ # boot's. Only :queued/:running tasks untouched for longer than that are
58
+ # candidates — :waiting/:paused are idle by nature (a human wait), so
59
+ # staleness cannot tell a live one from a dead one, and they stay boot
60
+ # recovery's. The threshold exists because a live :running turn is bounded
61
+ # by turn_timeout: anything older cannot be alive.
62
+ def run(stale_after: nil)
56
63
  resumed = []
57
64
  failed = []
58
65
  # Ordered by created_at: tasks from the SAME session are reprocessed
59
66
  # in their original order. Global time ordering is harmless for standalone tasks.
60
- # 1) interrupted (crash mid-flight) -> resume from the checkpoint.
61
- @task_store.running_or_interrupted.sort_by(&:created_at).each { |task| process(task, resumed, failed) }
67
+ # 1) interrupted (crash mid-flight) -> resume from the checkpoint. Tick mode
68
+ # narrows this to :running — the only mid-flight state staleness can judge.
69
+ running = stale_after ? @task_store.with_status(:running) : @task_store.running_or_interrupted
70
+ sweep(running, stale_after).each { |task| process(task, resumed, failed, tick: !stale_after.nil?) }
62
71
  # 2) queued but never started (turn in the SessionActor queue at crash time)
63
72
  # -> re-run from scratch (the same resume_task handles :queued). Without
64
73
  # this, a :queued turn in the volatile queue would be lost on kill -9.
65
- @task_store.queued.sort_by(&:created_at).each { |task| process(task, resumed, failed) }
74
+ sweep(@task_store.queued, stale_after).each { |task| process(task, resumed, failed, tick: !stale_after.nil?) }
66
75
  log(:info, "recovery finished: #{resumed.size} resumed, #{failed.size} failed")
67
76
  { resumed: resumed, failed: failed }
68
77
  end
69
78
 
70
79
  private
71
80
 
81
+ # Boot mode (stale_after nil) takes the candidate list as-is; tick mode keeps
82
+ # only tasks untouched past the threshold — the liveness gate of.
83
+ def sweep(tasks, stale_after)
84
+ tasks = tasks.sort_by(&:created_at)
85
+ return tasks unless stale_after
86
+
87
+ cutoff = Time.now.utc - stale_after
88
+ tasks.select { |task| stale?(task, cutoff) }
89
+ end
90
+
91
+ def stale?(task, cutoff)
92
+ Time.iso8601(task.updated_at.to_s) < cutoff
93
+ rescue ArgumentError
94
+ true # an unreadable timestamp is not proof of life — treat as stale
95
+ end
96
+
72
97
  # Failing to resume ONE task does not bring down the boot: a non-store
73
98
  # dispatch/latest error -> mark :failed and continue. StoreError -> propagate
74
99
  # (aborts the boot).
75
- def process(task, resumed, failed)
100
+ #
101
+ # tick: true flips ONE rescue: a ValidationError from the dispatch
102
+ # is ResumeTask's local liveness check ("task is running") — someone alive
103
+ # owns it, the normal case on a timer, so the task is SKIPPED for the next
104
+ # tick. At boot a ValidationError means corruption (nothing may be alive),
105
+ # so it still fails the task there. This is the trap defused in-process:
106
+ # the tick must never murder a live turn with its own recovery.
107
+ def process(task, resumed, failed, tick: false)
76
108
  # :queued never started (no checkpoint) but IS recoverable — ResumeTask
77
109
  # re-runs from the Command. An interrupted task requires a checkpoint;
78
110
  # without one, it is unrecoverable.
@@ -88,6 +120,15 @@ module Insika
88
120
  end
89
121
  rescue Insika::StoreError
90
122
  raise
123
+ rescue Insika::ValidationError => e
124
+ # Boot keeps the old contract (corruption -> :failed); the tick skips.
125
+ if tick
126
+ log(:info, "skipped (alive elsewhere): #{task.id} — #{e.message}")
127
+ else
128
+ fail_task(task.id, class_name: e.class.name, message: e.message, stage: "recovery")
129
+ failed << task.id unless failed.include?(task.id)
130
+ log(:warn, "failed to resume #{task.id}: #{e.class}: #{e.message}")
131
+ end
91
132
  rescue StandardError => e
92
133
  fail_task(task.id, class_name: e.class.name, message: e.message, stage: "recovery")
93
134
  failed << task.id unless failed.include?(task.id)
@@ -4,7 +4,7 @@ require "securerandom"
4
4
 
5
5
  module Insika
6
6
  module Refinement
7
- # ONE proposed change to an agent's instruction files (RFC-0013 §3.4). Data, not
7
+ # ONE proposed change to an agent's instruction files. Data, not
8
8
  # a diff of free text, and that is the load-bearing decision of the whole phase:
9
9
  #
10
10
  # · an ANCHORED edit is reviewable — the operator reads three lines, not a
@@ -40,7 +40,7 @@ module Insika
40
40
 
41
41
  # A dropped edit and why. Kept ON the candidate rather than logged: "the model
42
42
  # proposed four things and one was stale" is exactly what an operator reviewing
43
- # the loop's usefulness needs, and §10 asks them to judge precisely that.
43
+ # the loop's usefulness needs, and asks them to judge precisely that.
44
44
  Dropped = Data.define(:file, :op, :reason) do
45
45
  def to_h = { "file" => file, "op" => op, "reason" => reason }
46
46
  end
@@ -63,7 +63,7 @@ module Insika
63
63
  "edits" => edits.map(&:to_h), "dropped" => dropped.map(&:to_h) }
64
64
  end
65
65
 
66
- # The bounds, all config (§3.4). Defaults are deliberately small: what makes a
66
+ # The bounds, all config. Defaults are deliberately small: what makes a
67
67
  # diff reviewable is that it is short, and what keeps the gate's signal readable
68
68
  # is that a run changed few things.
69
69
  DEFAULT_LIMITS = { "max_edits" => 3, "max_bytes" => 1200, "max_total_growth" => 0.15 }.freeze
@@ -75,7 +75,7 @@ module Insika
75
75
 
76
76
  # raw: { "proposer" =>, "rationale" =>, "edits" => [ … ] } (string keys)
77
77
  # allowlist: the agent's `refinement.files`. EMPTY MEANS NOTHING IS WRITABLE —
78
- # report-only (§3.1/§3.8), so every edit drops. Not "no restriction":
78
+ # report-only, so every edit drops. Not "no restriction":
79
79
  # an unset allowlist that meant "anything" would turn a missing
80
80
  # config into the most permissive setting there is.
81
81
  # contents: name => current content, for staleness and growth.
@@ -5,7 +5,7 @@ require "time"
5
5
 
6
6
  module Insika
7
7
  module Refinement
8
- # RFC-0013 phase A. Reads a window of an agent's real traffic and emits RANKED
8
+ # Reads a window of an agent's real traffic and emits RANKED
9
9
  # FINDINGS — "here is what broke, how often, and in which conversations". No
10
10
  # model runs here and nothing is written to the agent: this is the evidence half
11
11
  # of the loop, and it is deliberately useful on its own.
@@ -17,7 +17,7 @@ module Insika
17
17
  # ToolTraceStore — per-session tool calls with ok/args/result (already masked
18
18
  # and clipped by the store itself)
19
19
  #
20
- # Two signals of RFC-0013 §3.3 are NOT computed here, and that is a finding about
20
+ # Two signals of are NOT computed here, and that is a finding about
21
21
  # the engine rather than about an agent: guardrail decisions and edge-limit hits
22
22
  # are emitted as EVENTS and never persisted, so the only durable footprint they
23
23
  # leave is the canned safe reply in the transcript — which is exactly what the
@@ -42,7 +42,7 @@ module Insika
42
42
  # result delivered as a new turn — is persisted with `role: user` like any
43
43
  # other, because it is what the model saw. Counting those as the customer
44
44
  # repeating themselves turned the first production run into 219 false positives,
45
- # every one of them the engine reading its own `<cacau_cep_obrigatorio>` back.
45
+ # every one of them the engine reading its own `<store_cep_required>` back.
46
46
  #
47
47
  # `MessageOrigin` is the structural answer and is preferred whenever a message
48
48
  # carries it. This regex stays for everything written before that field existed
@@ -188,12 +188,12 @@ module Insika
188
188
  # of an instruction the agent is not following. Heuristic on purpose (token overlap,
189
189
  # no model call); the snippet is PII-redacted.
190
190
  #
191
- # "After the agent answered" is the load-bearing half, and RFC-0015 is what forced
191
+ # "After the agent answered" is the load-bearing half, and is what forced
192
192
  # it to be said out loud. Two customer messages in a row is now ORDINARY: `collect`
193
193
  # merges the fragments a person types into one turn and `steer` appends one into a
194
194
  # run in flight, so a turn legitimately holds two of them. Someone still typing is
195
195
  # not someone repeating themselves — and a steered message cannot be told apart in
196
- # the transcript, because it correctly declares no origin (§7). The structure is the
196
+ # the transcript, because it correctly declares no origin. The structure is the
197
197
  # only honest signal: a reply has to sit between the two.
198
198
  def repetition_findings(session_ids)
199
199
  hits = session_ids.flat_map do |sid|
@@ -320,7 +320,7 @@ module Insika
320
320
 
321
321
  def words(text) = text.to_s.downcase.scan(/[[:alnum:]]+/).uniq
322
322
 
323
- # Every reply the deployment may emit INSTEAD of a real answer: the RFC-0009
323
+ # Every reply the deployment may emit INSTEAD of a real answer: the
324
324
  # defaults, the agent's own overrides, and the edge limiter's reply (per-agent
325
325
  # first, then the platform setting).
326
326
  def canned_replies(profile)