insika 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (185) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +145 -17
  3. data/README.md +7 -1
  4. data/bin/insika +211 -8
  5. data/docs/AGENTS.md +377 -2
  6. data/docs/API.md +56 -0
  7. data/docs/CHANNELS.md +99 -9
  8. data/docs/CONTEXT.md +127 -16
  9. data/docs/DEPLOY.md +5 -2
  10. data/docs/EVALS.md +19 -0
  11. data/docs/FACTS.md +135 -0
  12. data/docs/HARVEST.md +117 -0
  13. data/docs/LOADTEST.md +14 -8
  14. data/docs/OBSERVABILITY.md +54 -0
  15. data/docs/REFINEMENT.md +3 -3
  16. data/docs/RELEASING.md +27 -0
  17. data/docs/RUNNING-LOCAL.md +3 -3
  18. data/docs/SECURITY.md +61 -0
  19. data/docs/SKILLS.md +2 -2
  20. data/docs/SOAK.md +127 -0
  21. data/docs/TOOLS.md +70 -2
  22. data/docs/domain.md +115 -0
  23. data/docs/index.md +1 -1
  24. data/lib/insika/agent_profile.rb +145 -10
  25. data/lib/insika/balloon_splitter.rb +102 -0
  26. data/lib/insika/budget_ledger.rb +34 -3
  27. data/lib/insika/cache_series_store.rb +49 -0
  28. data/lib/insika/channel_delivery.rb +118 -13
  29. data/lib/insika/channels/relay.rb +77 -3
  30. data/lib/insika/channels/web.rb +2 -2
  31. data/lib/insika/chat_builder.rb +84 -1
  32. data/lib/insika/checkpoint_store.rb +16 -0
  33. data/lib/insika/commands/agent_payload.rb +3 -3
  34. data/lib/insika/commands/cancel_followup.rb +49 -0
  35. data/lib/insika/commands/delete_tenant_data.rb +95 -0
  36. data/lib/insika/commands/export_customer_memory.rb +48 -0
  37. data/lib/insika/commands/forget_customer.rb +117 -0
  38. data/lib/insika/commands/freeze_funnel_baseline.rb +113 -0
  39. data/lib/insika/commands/gate_harvest.rb +138 -0
  40. data/lib/insika/commands/gate_refinement.rb +1 -1
  41. data/lib/insika/commands/judge_shadow_pairs.rb +124 -0
  42. data/lib/insika/commands/memory_forget_fact.rb +20 -4
  43. data/lib/insika/commands/memory_put_fact.rb +23 -4
  44. data/lib/insika/commands/promote_harvest.rb +130 -0
  45. data/lib/insika/commands/record_outcome.rb +46 -0
  46. data/lib/insika/commands/record_shadow_reply.rb +68 -0
  47. data/lib/insika/commands/reject_harvest.rb +38 -0
  48. data/lib/insika/commands/resolve_proposal.rb +108 -0
  49. data/lib/insika/commands/revoke_contact.rb +49 -0
  50. data/lib/insika/commands/rollback_harvest.rb +86 -0
  51. data/lib/insika/commands/run_distillation.rb +186 -0
  52. data/lib/insika/commands/run_harvest.rb +393 -0
  53. data/lib/insika/commands/send_message.rb +103 -6
  54. data/lib/insika/commands/session_purge.rb +67 -0
  55. data/lib/insika/contact_store.rb +183 -0
  56. data/lib/insika/context/builder.rb +21 -3
  57. data/lib/insika/context/fragment.rb +7 -3
  58. data/lib/insika/context/priority.rb +3 -0
  59. data/lib/insika/context/provider.rb +17 -3
  60. data/lib/insika/context/providers/briefing.rb +96 -0
  61. data/lib/insika/context/providers/memory.rb +16 -7
  62. data/lib/insika/context/providers/prompt.rb +30 -2
  63. data/lib/insika/context/providers/skill.rb +2 -0
  64. data/lib/insika/context/providers/tool_search.rb +2 -0
  65. data/lib/insika/context_trace_store.rb +38 -2
  66. data/lib/insika/distill.rb +224 -0
  67. data/lib/insika/distill_engine.rb +169 -0
  68. data/lib/insika/doctor.rb +716 -6
  69. data/lib/insika/dsl/runtime.rb +9 -3
  70. data/lib/insika/dsl/server_boot.rb +70 -1
  71. data/lib/insika/dsl.rb +83 -3
  72. data/lib/insika/edge_limiter.rb +28 -7
  73. data/lib/insika/env_schema.rb +9 -2
  74. data/lib/insika/errors.rb +11 -0
  75. data/lib/insika/evals/assertions.rb +3 -2
  76. data/lib/insika/evals/pairwise.rb +18 -6
  77. data/lib/insika/evidence.rb +183 -0
  78. data/lib/insika/executor.rb +687 -70
  79. data/lib/insika/followup_engine.rb +207 -0
  80. data/lib/insika/followup_policy.rb +221 -0
  81. data/lib/insika/followup_store.rb +306 -0
  82. data/lib/insika/funnel_declaration.rb +106 -0
  83. data/lib/insika/funnel_fold.rb +179 -0
  84. data/lib/insika/funnel_store.rb +163 -0
  85. data/lib/insika/golden_store.rb +1 -1
  86. data/lib/insika/grounding/matcher.rb +69 -0
  87. data/lib/insika/grounding.rb +44 -0
  88. data/lib/insika/harvest/conversion_gate.rb +159 -0
  89. data/lib/insika/harvest/criterion.rb +98 -0
  90. data/lib/insika/harvest/gate.rb +194 -0
  91. data/lib/insika/harvest/negative_list.rb +199 -0
  92. data/lib/insika/harvest.rb +241 -0
  93. data/lib/insika/harvest_engine.rb +193 -0
  94. data/lib/insika/harvest_store.rb +548 -0
  95. data/lib/insika/media.rb +298 -0
  96. data/lib/insika/memory_audit_store.rb +85 -0
  97. data/lib/insika/memory_store.rb +264 -23
  98. data/lib/insika/message_origin.rb +6 -1
  99. data/lib/insika/model_visible.rb +87 -0
  100. data/lib/insika/model_visible_trace_store.rb +66 -0
  101. data/lib/insika/onboarding.rb +6 -1
  102. data/lib/insika/outbox_store.rb +42 -4
  103. data/lib/insika/outcome_store.rb +147 -0
  104. data/lib/insika/packaging.rb +163 -0
  105. data/lib/insika/parity/criterion.rb +79 -0
  106. data/lib/insika/parity/verdict.rb +318 -0
  107. data/lib/insika/prefix_fingerprint.rb +58 -0
  108. data/lib/insika/profile_source.rb +22 -1
  109. data/lib/insika/proposal_store.rb +271 -0
  110. data/lib/insika/queue_policy.rb +4 -1
  111. data/lib/insika/refinement/proposer.rb +1 -1
  112. data/lib/insika/reliability.rb +32 -6
  113. data/lib/insika/retention.rb +281 -0
  114. data/lib/insika/routing.rb +101 -0
  115. data/lib/insika/safety/config.rb +44 -4
  116. data/lib/insika/safety/corpus.rb +255 -0
  117. data/lib/insika/safety/detectors.rb +32 -113
  118. data/lib/insika/safety/factory.rb +15 -2
  119. data/lib/insika/safety/grounding_enforcer.rb +59 -0
  120. data/lib/insika/safety/grounding_validator.rb +49 -0
  121. data/lib/insika/safety/input_guardrail.rb +1 -1
  122. data/lib/insika/safety/moderator.rb +1 -1
  123. data/lib/insika/safety/output_filter.rb +9 -5
  124. data/lib/insika/safety/output_validator.rb +11 -5
  125. data/lib/insika/schema_guard.rb +35 -0
  126. data/lib/insika/server/app.rb +144 -4
  127. data/lib/insika/server/rack_app.rb +16 -0
  128. data/lib/insika/server/responses.rb +40 -1
  129. data/lib/insika/session_actor.rb +8 -4
  130. data/lib/insika/session_store.rb +65 -2
  131. data/lib/insika/settings_store.rb +10 -0
  132. data/lib/insika/shadow_pair_store.rb +258 -0
  133. data/lib/insika/skill_catalog.rb +4 -0
  134. data/lib/insika/soak/envelope.rb +140 -0
  135. data/lib/insika/soak/report.rb +392 -0
  136. data/lib/insika/soak/runner.rb +554 -0
  137. data/lib/insika/store.rb +10 -1
  138. data/lib/insika/stores/memory.rb +6 -0
  139. data/lib/insika/stores/sqlite.rb +8 -0
  140. data/lib/insika/studio/app.rb +875 -14
  141. data/lib/insika/studio/assets/dist/application.css +1 -1
  142. data/lib/insika/studio/assets/dist/application.js +22 -22
  143. data/lib/insika/studio/assets/dist/favicon.svg +6 -0
  144. data/lib/insika/studio/forms.rb +262 -5
  145. data/lib/insika/studio/nav_icons.rb +6 -1
  146. data/lib/insika/studio/views/agent_detail.erb +627 -84
  147. data/lib/insika/studio/views/agents.erb +10 -6
  148. data/lib/insika/studio/views/approvals.erb +4 -1
  149. data/lib/insika/studio/views/chats.erb +4 -1
  150. data/lib/insika/studio/views/customer.erb +94 -0
  151. data/lib/insika/studio/views/customers.erb +32 -0
  152. data/lib/insika/studio/views/evals.erb +4 -1
  153. data/lib/insika/studio/views/facts.erb +133 -0
  154. data/lib/insika/studio/views/followups.erb +125 -0
  155. data/lib/insika/studio/views/funnel.erb +106 -0
  156. data/lib/insika/studio/views/harvest.erb +234 -0
  157. data/lib/insika/studio/views/home.erb +2 -1
  158. data/lib/insika/studio/views/layout.erb +1 -0
  159. data/lib/insika/studio/views/parity.erb +147 -0
  160. data/lib/insika/studio/views/playground.erb +7 -1
  161. data/lib/insika/studio/views/session.erb +56 -1
  162. data/lib/insika/studio/views/settings.erb +2 -0
  163. data/lib/insika/studio/views/skills.erb +0 -1
  164. data/lib/insika/studio/views/system_files.erb +1 -1
  165. data/lib/insika/studio/views/task.erb +13 -0
  166. data/lib/insika/studio/views/tasks.erb +4 -1
  167. data/lib/insika/studio/views/tools.erb +0 -1
  168. data/lib/insika/task_store.rb +21 -1
  169. data/lib/insika/testing/store_contract.rb +27 -6
  170. data/lib/insika/tick.rb +25 -1
  171. data/lib/insika/tool_definition.rb +17 -7
  172. data/lib/insika/tool_envelope.rb +69 -0
  173. data/lib/insika/tool_manifest.rb +5 -1
  174. data/lib/insika/tools/data_defined_tool.rb +10 -0
  175. data/lib/insika/tools/generate_image.rb +44 -0
  176. data/lib/insika/tools/schedule_followup.rb +164 -0
  177. data/lib/insika/tools/tts.rb +47 -0
  178. data/lib/insika/tools/update_briefing.rb +126 -0
  179. data/lib/insika/turn_state.rb +38 -1
  180. data/lib/insika/turn_timing.rb +22 -2
  181. data/lib/insika/version.rb +1 -1
  182. data/lib/insika/vitals.rb +84 -0
  183. data/lib/insika/wiring/graph.rb +300 -13
  184. data/lib/insika.rb +92 -3
  185. metadata +77 -1
@@ -0,0 +1,271 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "securerandom"
4
+ require "time"
5
+
6
+ module Insika
7
+ # the proposals and the two persisted mechanisms the gates
8
+ # on — the **latched dedup ledger** (D3: the rows themselves ARE the ledger —
9
+ # a dismissed/rejected tuple is never proposed again, and an unanswered
10
+ # proposal is not piled on) and the **per-session distilled marker** (D2:
11
+ # written only after a pass completes, so a crash mid-pass leaves the marker
12
+ # unwritten and the next pass re-scans). A dumb domain store — it holds no
13
+ # policy (which tuple is a fact is the distiller's job), no memory facts and
14
+ # no model. The scope string (the memory cell) is built by the callers from
15
+ # the `MemoryStore::parse_cell` shape; the store keys by
16
+ # `(tenant, customer)` explicitly.
17
+ #
18
+ # Statuses: pending -> approved | rejected | dismissed | stale.
19
+ # `stale` is the CAS-lost re-present (E3): the proposal carries the fact's
20
+ # CURRENT value (`current_value`) next to the proposed one, never a silent
21
+ # overwrite.
22
+ class ProposalStore
23
+ SCOPE = "proposals"
24
+ STATUSES = %w[pending approved rejected dismissed stale].freeze
25
+ TERMINAL = %w[approved rejected dismissed].freeze
26
+ PROPOSAL_PREFIX = "p:"
27
+ MARKER_PREFIX = "s:"
28
+
29
+ Proposal = Data.define(:id, :tenant, :customer, :scope, :session_ref, :key,
30
+ :value, :confidence, :status, :evidence,
31
+ :expected_revision, :expected_existed, :current_value,
32
+ :operator, :note, :created_at, :updated_at)
33
+
34
+ def initialize(store:)
35
+ @store = store
36
+ end
37
+
38
+ # -> Proposal (status :pending). The caller (RunDistillation) already ran
39
+ # the dedup checks; the store writes. `evidence` = message indexes; the
40
+ # revision baseline (D5) travels with the record.
41
+ #
42
+ # The tenant is stored VERBATIM — nil in a single-tenant deployment, so
43
+ # the scope is the bare `customer` cell and the approval reads/writes the
44
+ # SAME cell the Memory provider injects (memory_store.rb's
45
+ # blank-tenant + customer -> "memory:<customer>" rule). Coercing a blank
46
+ # tenant to a sentinel here would orphan every approved fact in a
47
+ # phantom "memory:platform:<customer>" cell.
48
+ def create(tenant:, customer:, session_ref:, key:, value:, confidence: nil,
49
+ evidence: [], expected_revision: nil, expected_existed: false,
50
+ id: SecureRandom.uuid, now: Time.now.utc)
51
+ tenant = tenant_key(tenant)
52
+ stamp = now.iso8601(6)
53
+ record = { "id" => id.to_s, "status" => "pending",
54
+ "tenant" => tenant, "customer" => customer.to_s,
55
+ "scope" => [tenant, customer.to_s].compact.join(":"),
56
+ "session_ref" => session_ref.to_s, "key" => key.to_s,
57
+ "value" => value.to_s, "confidence" => confidence,
58
+ "evidence" => Array(evidence).map(&:to_i),
59
+ "expected_revision" => expected_revision,
60
+ "expected_existed" => !!expected_existed,
61
+ "current_value" => nil, "operator" => nil, "note" => nil,
62
+ "created_at" => stamp, "updated_at" => stamp }
63
+ @store.set(SCOPE, PROPOSAL_PREFIX + id.to_s, record)
64
+ to_proposal(record)
65
+ end
66
+
67
+ def find(id)
68
+ record = @store.get(SCOPE, PROPOSAL_PREFIX + id.to_s)
69
+ record && to_proposal(record)
70
+ end
71
+
72
+ # The wiki's lists. `pending` = pending, oldest first (the operator works
73
+ # the oldest proposal first — evidence ages).
74
+ def pending(limit: 100)
75
+ scan.select { |p| p.status == "pending" }
76
+ .sort_by { |p| [p.created_at.to_s, p.id] }
77
+ .first(limit)
78
+ end
79
+
80
+ def stale(limit: 50)
81
+ scan.select { |p| p.status == "stale" }
82
+ .sort_by { |p| p.updated_at.to_s }
83
+ .first(limit)
84
+ end
85
+
86
+ # The wiki's Recent list: every terminal status (approved/rejected/
87
+ # dismissed), most recent first — the operator's audit trail.
88
+ def resolved(limit: 20)
89
+ scan.select { |p| TERMINAL.include?(p.status) }
90
+ .sort_by { |p| p.updated_at.to_s }
91
+ .reverse
92
+ .first(limit)
93
+ end
94
+
95
+ # ---- the latched dedup (D3) ----
96
+ # true when a dismissed/rejected row exists for the exact tuple — the
97
+ # latch. Persisted rows ARE the ledger. A *different* value for the same
98
+ # `name` is a different tuple.
99
+ def decided?(tenant:, customer:, key:, value:)
100
+ scan.any? do |p|
101
+ tenant_key(p.tenant) == tenant_key(tenant) && p.customer == customer.to_s &&
102
+ p.key == key.to_s && p.value == value.to_s &&
103
+ %w[dismissed rejected].include?(p.status)
104
+ end
105
+ end
106
+
107
+ # true when a pending row exists for (scope, key) — no piling.
108
+ def open_pending?(tenant:, customer:, key:)
109
+ scan.any? do |p|
110
+ tenant_key(p.tenant) == tenant_key(tenant) && p.customer == customer.to_s &&
111
+ p.key == key.to_s && p.status == "pending"
112
+ end
113
+ end
114
+
115
+ # ---- transitions, each read-check-write on @store.transaction ----
116
+ # pending -> terminal. ArgumentError for a wrong source state (the
117
+ # task_store.rb state-machine idiom).
118
+ def approve(id:, operator: nil, note: nil, now: Time.now.utc)
119
+ transition(id, "approved", operator: operator, note: note, now: now)
120
+ end
121
+
122
+ def reject(id:, operator: nil, note: nil, now: Time.now.utc)
123
+ transition(id, "rejected", operator: operator, note: note, now: now)
124
+ end
125
+
126
+ def dismiss(id:, operator: nil, note: nil, now: Time.now.utc)
127
+ transition(id, "dismissed", operator: operator, note: note, now: now)
128
+ end
129
+
130
+ # pending -> stale, CAS lost; `current_value` = the fact as it stands (the
131
+ # re-present's second value, E3).
132
+ def mark_stale(id:, current_value:, operator: nil, now: Time.now.utc)
133
+ transition(id, "stale", operator: operator, current_value: current_value, now: now)
134
+ end
135
+
136
+ # ---- the per-session marker (D2) ----
137
+ # Written ONLY after a pass completes (RunDistillation). -> the marker hash.
138
+ def mark_distilled(session_ref, agent:, proposals:, dropped:, deduped: 0, cost: nil, now: Time.now.utc)
139
+ marker = { "session_ref" => session_ref.to_s, "agent" => agent.to_s,
140
+ "distilled_at" => now.iso8601, "proposals" => proposals.to_i,
141
+ "dropped" => dropped, "deduped" => deduped.to_i,
142
+ "cost" => cost }
143
+ @store.set(SCOPE, MARKER_PREFIX + session_ref.to_s, marker)
144
+ marker
145
+ end
146
+
147
+ def distilled?(session_ref)
148
+ !@store.get(SCOPE, MARKER_PREFIX + session_ref.to_s).nil?
149
+ end
150
+
151
+ def distilled_sessions(agent_id = nil)
152
+ keys = agent_id ? marker_keys.select { |k| marker_agent(k) == agent_id.to_s } : marker_keys
153
+ keys.map { |k| k.delete_prefix(MARKER_PREFIX) }
154
+ end
155
+
156
+ # ---- LGPD / retention (C8) ----
157
+
158
+ # One customer's proposals, EVERY status. -> count removed.
159
+ def purge_customer(tenant:, customer:)
160
+ removed = 0
161
+ @store.transaction do
162
+ proposal_keys.each do |k|
163
+ record = @store.get(SCOPE, k)
164
+ next unless record && record["tenant"] == tenant_key(tenant)
165
+ next unless record["customer"] == customer.to_s
166
+
167
+ @store.delete(SCOPE, k)
168
+ removed += 1
169
+ end
170
+ end
171
+ removed
172
+ end
173
+
174
+ # A tenant's proposals. -> count removed.
175
+ def purge(tenant:)
176
+ removed = 0
177
+ @store.transaction do
178
+ proposal_keys.each do |k|
179
+ record = @store.get(SCOPE, k)
180
+ next unless record && record["tenant"] == tenant_key(tenant)
181
+
182
+ @store.delete(SCOPE, k)
183
+ removed += 1
184
+ end
185
+ end
186
+ removed
187
+ end
188
+
189
+ # Age-based prune (the retention sweep). TERMINAL rows age by their
190
+ # updated_at; a PENDING row is a zombie past the cutoff (its transcript is
191
+ # dead). Session MARKERS die WITH their proposals — a marker past the
192
+ # cutoff is evidence about a dead transcript (the session aged out under
193
+ # the same retention window), and keeping it would lock an unreviewed
194
+ # proposal out of re-distillation forever. -> count removed.
195
+ def delete_older_than(time)
196
+ cutoff = time.utc.iso8601
197
+ removed = 0
198
+ @store.transaction do
199
+ proposal_keys.each do |k|
200
+ record = @store.get(SCOPE, k)
201
+ next unless record
202
+
203
+ terminal = TERMINAL.include?(record["status"])
204
+ stamp = terminal ? record["updated_at"] : record["created_at"]
205
+ next unless stamp && stamp.to_s < cutoff
206
+
207
+ @store.delete(SCOPE, k)
208
+ removed += 1
209
+ end
210
+ marker_keys.each do |k|
211
+ marker = @store.get(SCOPE, k)
212
+ next unless marker && marker["distilled_at"].to_s < cutoff
213
+
214
+ @store.delete(SCOPE, k)
215
+ removed += 1
216
+ end
217
+ end
218
+ removed
219
+ end
220
+
221
+ private
222
+
223
+ def transition(id, to, operator: nil, note: nil, current_value: nil, now: Time.now.utc)
224
+ @store.transaction do
225
+ key = PROPOSAL_PREFIX + id.to_s
226
+ record = @store.get(SCOPE, key)
227
+ raise Insika::NotFoundError, "proposal not found: #{id}" if record.nil?
228
+
229
+ unless record["status"] == "pending"
230
+ raise ArgumentError,
231
+ "proposal #{id}: cannot resolve a #{record['status']} proposal — " \
232
+ "it resolves once, from pending"
233
+ end
234
+
235
+ record["status"] = to
236
+ record["operator"] = operator.to_s unless operator.nil?
237
+ record["note"] = note.to_s unless note.nil?
238
+ record["current_value"] = current_value unless current_value.nil?
239
+ record["updated_at"] = now.iso8601(6)
240
+ @store.set(SCOPE, key, record)
241
+ to_proposal(record)
242
+ end
243
+ end
244
+
245
+ def scan
246
+ proposal_keys.filter_map do |k|
247
+ record = @store.get(SCOPE, k)
248
+ record && to_proposal(record)
249
+ end
250
+ end
251
+
252
+ def proposal_keys = @store.list(SCOPE, PROPOSAL_PREFIX)
253
+ def marker_keys = @store.list(SCOPE, MARKER_PREFIX)
254
+
255
+ # nil stays nil (single-tenant); a present tenant is a String. The
256
+ # comparisons below use tenant_key on BOTH sides so nil == nil holds.
257
+ def tenant_key(tenant) = tenant.nil? ? nil : tenant.to_s
258
+
259
+ def to_proposal(rec)
260
+ Proposal.new(id: rec["id"], tenant: rec["tenant"], customer: rec["customer"],
261
+ scope: rec["scope"], session_ref: rec["session_ref"],
262
+ key: rec["key"], value: rec["value"], confidence: rec["confidence"],
263
+ status: rec["status"], evidence: rec["evidence"] || [],
264
+ expected_revision: rec["expected_revision"],
265
+ expected_existed: rec["expected_existed"] == true,
266
+ current_value: rec["current_value"], operator: rec["operator"],
267
+ note: rec["note"], created_at: rec["created_at"],
268
+ updated_at: rec["updated_at"])
269
+ end
270
+ end
271
+ end
@@ -85,7 +85,10 @@ module Insika
85
85
  def debounce? = @debounce_ms.positive?
86
86
 
87
87
  # Does this policy merge into a turn that has not started yet?
88
- def collect? = @mode == :collect
88
+ # True for :collect AND :steer both absorb fragments that land before the
89
+ # turn starts. Debounce is what actually holds them; without a window a steer
90
+ # agent still steers mid-run and never waits at the door.
91
+ def collect? = @mode == :collect || @mode == :steer
89
92
 
90
93
  # Does this policy append into a turn that is already running? `steer_max_messages`
91
94
  # of 0 is the agent saying no, so it answers false rather than steering once and
@@ -214,7 +214,7 @@ module Insika
214
214
  end
215
215
 
216
216
  # `proposers` accepts either syntax — a bare ref ("deepseek/deepseek-v4-flash") or
217
- # the RFC's `{ "model" =>, "provider"? => }` — because the two already coexist in
217
+ # the hash form `{ "model" =>, "provider"? => }` — because the two already coexist in
218
218
  # this config (`proposer` is a bare ref, `judges` are hashes) and refusing one of
219
219
  # them would only teach operators which page they were reading.
220
220
  def refs_for(config, utility_model)
@@ -42,8 +42,12 @@ module Insika
42
42
  #
43
43
  # -> the successful response. Raises CircuitOpenError (primary open),
44
44
  # or the last retryable error when every node exhausted its retries.
45
+ # NOTHING about a run is stored on `self`: one Reliability instance serves
46
+ # every concurrent turn, and `ask` is a suspension point — an ivar written
47
+ # here would be read back after another fiber's turn overwrote it, and the
48
+ # WS6 alert would name the wrong agent (and, through it, the wrong tenant).
49
+ # The run's identity rides the stack.
45
50
  def call(policy:, tenant:, agent: nil, selection:, chain:, &attempt)
46
- @agent = agent # event attribution (WF6 alerts) for THIS run
47
51
  nodes = ([selection] + Array(chain)).map { |node| { selection: node, tries: 0 } }
48
52
  retries = [policy["retries"].to_i, 0].max
49
53
  breaker = breaker_config(policy)
@@ -61,10 +65,21 @@ module Insika
61
65
  if breaker && breaker_open?(tenant, selection, breaker)
62
66
  raise circuit_open(tenant, selection, breaker)
63
67
  end
68
+ # the last node we actually ASKED: the `from` of a rotation. A node the
69
+ # breaker skipped was never asked, so it is never the `from`.
70
+ asked = nil
64
71
  nodes.each do |node|
65
72
  selection = node[:selection]
66
73
  next if breaker && breaker_open?(tenant, selection, breaker)
67
74
 
75
+ # ROTATION is an EVENT, not an inference from the usage attribution
76
+ # (WS3): the trace names the node we left, the node we moved to and why.
77
+ # Emitted for every node past the first one asked — with or WITHOUT a
78
+ # breaker (a fallback policy with no circuit_breaker used to rotate in
79
+ # complete silence).
80
+ emit_fallback(agent, asked, selection, last_error) if asked
81
+ asked = selection
82
+
68
83
  attempts = retries + 1
69
84
  attempts.times do |index|
70
85
  node[:tries] += 1
@@ -87,7 +102,7 @@ module Insika
87
102
  raise unless retryable
88
103
  raise if kind_of(e) == :fatal && !e.is_a?(Insika::TimeoutError)
89
104
 
90
- record_failure(tenant, selection, breaker, e)
105
+ record_failure(tenant, selection, breaker, e, agent)
91
106
  # the last attempt of the last node re-raises; otherwise back off
92
107
  # and give the next attempt/node a turn.
93
108
  if index < attempts - 1 || node != nodes.last
@@ -127,17 +142,28 @@ module Insika
127
142
  )
128
143
  end
129
144
 
130
- def record_failure(tenant, selection, breaker, error)
145
+ # The failure is ALWAYS an event; only the circuit bump needs a breaker (the
146
+ # old `return unless breaker` made a retry/fallback policy without a
147
+ # circuit_breaker run with no trace at all).
148
+ def record_failure(tenant, selection, breaker, error, agent)
149
+ emit(:provider_failure,
150
+ { agent: agent, ref: ref_of(selection), error: error.class.name, kind: kind_of(error) })
131
151
  return unless breaker
132
152
 
133
153
  tripped = @circuit_store.record_failure(
134
154
  tenant: tenant, ref: ref_of(selection),
135
155
  after: breaker[:after], within: breaker[:within]
136
156
  )
137
- emit(:provider_failure,
138
- { agent: @agent, ref: ref_of(selection), error: error.class.name, kind: kind_of(error) })
139
157
  # the failure that TRIPPED the circuit is itself an alert (WS6).
140
- emit(:breaker_open, { agent: @agent, ref: ref_of(selection), tenant: tenant }) if tripped == :open
158
+ emit(:breaker_open, { agent: agent, ref: ref_of(selection), tenant: tenant }) if tripped == :open
159
+ end
160
+
161
+ # The mid-turn rotation across the fallback chain: the node we left, the one
162
+ # we moved to, and the error that spent the previous node's retries.
163
+ def emit_fallback(agent, from, to, error)
164
+ emit(:provider_fallback,
165
+ { agent: agent, from: ref_of(from), to: ref_of(to),
166
+ error: error&.class&.name, kind: error && kind_of(error) })
141
167
  end
142
168
 
143
169
  def kind_of(error) = ProviderErrorClassifier.classify(error).kind
@@ -0,0 +1,281 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "time"
4
+
5
+ module Insika
6
+ # WS8 (phase 2): retention — the engine forgets what is old enough to forget.
7
+ # Age-based purge of the CONVERSATION footprint: sessions (and their
8
+ # per-session tool/context traces), terminal tasks (and their checkpoints),
9
+ # customer/tenant memory cells and outcome records. The knob is data, not
10
+ # code: `settings.retention_days` (Integer; nil/0 = OFF — parity, nothing is
11
+ # ever swept by default).
12
+ #
13
+ # Cadence: the Tick calls `run` every pass; an internal DAILY claim (the
14
+ # tick's window idiom — one key, a timestamp, 24 h) makes the actual sweep
15
+ # once a day, so the O(n) scans never ride the 60 s loop. A sweep that
16
+ # crashes propagates to the tick, which logs and keeps ticking.
17
+ class Retention
18
+ SCOPE = "retention"
19
+ KEY = "claim"
20
+ BUDGET_KEY = "budget_claim"
21
+ # the MEMORY TTL's own daily claim. Deliberately NOT the age-based
22
+ # KEY — memory TTLs sweep on their own knob (`memory_ttl_days`), gated by
23
+ # neither retention_days nor the age-based claim (D5).
24
+ MEMORY_TTL_KEY = "memory_ttl_claim"
25
+ WINDOW = 86_400 # one sweep per day, at most
26
+
27
+ TERMINAL = %w[completed failed cancelled].freeze
28
+
29
+ def initialize(session_store:, task_store:, checkpoint_store:,
30
+ memory_store:, outcome_store:, tool_trace_store: nil,
31
+ context_trace_store: nil, outbox_store: nil, shadow_pair_store: nil,
32
+ settings_store: nil, budget_ledger: nil, funnel_store: nil,
33
+ followup_store: nil, contact_store: nil, proposal_store: nil,
34
+ model_visible_trace_store: nil,
35
+ store:, window: WINDOW, now: nil, harvest_store: nil)
36
+ @session_store = session_store
37
+ @task_store = task_store
38
+ @checkpoint_store = checkpoint_store
39
+ @memory_store = memory_store
40
+ @outcome_store = outcome_store
41
+ @tool_trace_store = tool_trace_store
42
+ @context_trace_store = context_trace_store
43
+ @model_visible_trace_store = model_visible_trace_store # ; nil = parity
44
+ @outbox_store = outbox_store
45
+ @shadow_pair_store = shadow_pair_store
46
+ @settings_store = settings_store
47
+ @budget_ledger = budget_ledger # WS2 counter GC; nil = nothing to sweep
48
+ @funnel_store = funnel_store # ; nil = nothing to sweep
49
+ @followup_store = followup_store # ; nil = nothing to sweep
50
+ @contact_store = contact_store # ; nil = nothing to sweep
51
+ @proposal_store = proposal_store # ; nil = nothing to sweep
52
+ @store = store
53
+ @window = window
54
+ @now = now # injectable for specs (a deterministic "today")
55
+ @harvest_store = harvest_store # ; nil = nothing to sweep
56
+ end
57
+
58
+ # the sweep reads the memory store's cells/records (specs seed
59
+ # facts through it).
60
+ attr_reader :memory_store
61
+
62
+ # -> { claimed: false } |
63
+ # { claimed: true, sessions:, tasks:, outcomes:, memory:, deliveries: }.
64
+ # Either shape may carry `budget_cells:` — the budget counter GC is NOT
65
+ # gated by retention_days (see #sweep_budget_cells). Either shape may carry
66
+ # `memory_ttl:` — the memory TTL sweep, gated by ITS OWN daily
67
+ # claim and knob, never by retention_days (see #sweep_memory_ttl).
68
+ def run
69
+ budget_cells = sweep_budget_cells
70
+ memory_ttl = sweep_memory_ttl
71
+ days = retention_days
72
+ unless days.to_i.positive? && claim_window
73
+ summary = { claimed: false }
74
+ summary[:budget_cells] = budget_cells if budget_cells
75
+ summary[:memory_ttl] = memory_ttl if memory_ttl
76
+ return summary
77
+ end
78
+
79
+ cutoff = now - (days.to_i * 86_400)
80
+ summary = { claimed: true, sessions: sweep_sessions(cutoff),
81
+ tasks: sweep_tasks(cutoff), outcomes: sweep_outcomes(cutoff),
82
+ memory: @memory_store.prune_older_than(cutoff),
83
+ deliveries: sweep_outbox(cutoff),
84
+ pairs: sweep_shadow_pairs(cutoff) }
85
+ summary[:funnel] = sweep_funnel(cutoff) if @funnel_store
86
+ # the follow-up footprint ages out with the rest — records
87
+ # and contact cells under the SAME retention_days gate (nil collaborator
88
+ # = nothing to sweep, base graph parity).
89
+ summary[:followups] = @followup_store.delete_older_than(cutoff) if @followup_store
90
+ summary[:contacts] = @contact_store.delete_older_than(cutoff) if @contact_store
91
+ # proposals are evidence OF a transcript — when the
92
+ # transcript dies, the proposal's excerpt is gone and the pending fact is
93
+ # stale. Pending AND terminal rows age out together; a proposal is
94
+ # re-derivable (D2), so pruning is never data loss. The session MARKERS
95
+ # are never pruned (the store's rule — the marker is the claim).
96
+ summary[:proposals] = @proposal_store.delete_older_than(cutoff) if @proposal_store
97
+ # candidates (pending AND terminal), log rows and
98
+ # snapshots are DERIVED data of transcripts (D11) — when the transcripts
99
+ # die, the candidates' excerpts are gone; they are re-derivable (D2), so
100
+ # pruning is never data loss. The session markers are never pruned.
101
+ summary[:harvest] = @harvest_store.delete_older_than(cutoff) if @harvest_store
102
+ summary[:budget_cells] = budget_cells if budget_cells
103
+ summary[:memory_ttl] = memory_ttl if memory_ttl
104
+ summary
105
+ end
106
+
107
+ private
108
+
109
+ # The BudgetLedger's expired cells (WS2). Deliberately OUTSIDE the
110
+ # retention_days gate, on its OWN daily claim: those rows are engine
111
+ # bookkeeping whose window already rolled over, not customer content, so a
112
+ # deployment that keeps its conversations forever (retention OFF — the
113
+ # default) must still not grow budget rows forever. -> count | nil (no
114
+ # ledger, or another worker holds today's claim).
115
+ def sweep_budget_cells
116
+ return nil unless @budget_ledger && claim(BUDGET_KEY)
117
+
118
+ @budget_ledger.prune(now: now)
119
+ end
120
+
121
+ def retention_days
122
+ return nil unless @settings_store
123
+
124
+ value = @settings_store.get["retention_days"]
125
+ value.to_s.empty? ? nil : Integer(value)
126
+ rescue ArgumentError, TypeError
127
+ nil # a non-numeric value reads as OFF — never a crash at sweep time
128
+ end
129
+
130
+ # the memory TTL sweep — TWO independent expiry clocks under
131
+ # ONE daily claim:
132
+ # 1. per-fact expires_at (prune_expired — past dates physically removed);
133
+ # 2. per-cell TTL by `memory_ttl_days` (age by the cell's updated_at).
134
+ # A fact with an explicit expires_at is EXCLUDED from the age-based pass
135
+ # (the explicit override owns that fact's life). Runs on its OWN claim and
136
+ # knob — a deployment with retention_days off still honors memory TTLs.
137
+ # -> Integer (removed) | nil (no knob, or another worker holds the claim).
138
+ def sweep_memory_ttl
139
+ ttl = memory_ttl_setting
140
+ return nil if ttl.nil?
141
+ return nil unless claim(MEMORY_TTL_KEY)
142
+
143
+ removed = @memory_store.prune_expired(now)
144
+ ttl_cutoffs(ttl).each do |scope, cutoff|
145
+ removed += @memory_store.prune_older_than(cutoff, scope: scope)
146
+ end
147
+ removed
148
+ end
149
+
150
+ # settings["memory_ttl_days"]: Integer | Hash{ "<tenant>" => days, "*" => days }.
151
+ # nil/empty/blank -> nil (OFF — parity). A non-numeric value -> nil (never
152
+ # a crash at sweep time, the retention_days rescue pattern).
153
+ def memory_ttl_setting
154
+ return nil unless @settings_store
155
+
156
+ raw = @settings_store.get["memory_ttl_days"]
157
+ case raw
158
+ when Integer then raw
159
+ when Hash
160
+ map = raw.each_with_object({}) do |(k, v), acc|
161
+ acc[k.to_s] = Integer(v.to_s)
162
+ rescue ArgumentError, TypeError
163
+ next
164
+ end
165
+ map.empty? ? nil : map
166
+ end
167
+ end
168
+
169
+ # -> [[scope, Time]] — one per existing cell with a resolved TTL. The
170
+ # setting is passed in (one settings-store read per sweep — the caller
171
+ # already resolved it).
172
+ def ttl_cutoffs(setting = memory_ttl_setting)
173
+ return [] unless setting
174
+
175
+ @memory_store.cells.filter_map do |cell|
176
+ days = cell_ttl(cell, setting)
177
+ next if days.nil? || days <= 0
178
+
179
+ [cell[:scope], now - days * 86_400]
180
+ end
181
+ end
182
+
183
+ # Customer cell "memory:acme:c-1" -> map["acme"]; tenant/bare cell
184
+ # "memory:acme" or "memory:c-123" -> map["acme"] / map["c-123"]; fallback
185
+ # map["*"]; an Integer setting -> every cell gets it.
186
+ def cell_ttl(cell, setting)
187
+ return setting if setting.is_a?(Integer)
188
+
189
+ key = cell[:tenant] || cell[:customer]
190
+ v = key && setting[key]
191
+ v ||= setting["*"]
192
+ v
193
+ end
194
+
195
+ # Sessions untouched past the cutoff, and their per-session traces.
196
+ def sweep_sessions(cutoff)
197
+ removed = 0
198
+ @session_store.each_id.each do |id|
199
+ session = @session_store.find(id)
200
+ next unless session && session.updated_at && session.updated_at < cutoff.iso8601
201
+
202
+ @tool_trace_store&.clear(id)
203
+ @context_trace_store&.clear(id)
204
+ @session_store.delete(id)
205
+ removed += 1
206
+ end
207
+ removed
208
+ end
209
+
210
+ # TERMINAL tasks untouched past the cutoff, and their checkpoints. A
211
+ # non-terminal task (queued/running) is never touched here — the Recovery
212
+ # sweep owns those lives. the model-visible traces are
213
+ # transcripts — they die next to their checkpoints.
214
+ def sweep_tasks(cutoff)
215
+ removed = 0
216
+ @task_store.each_id.each do |id|
217
+ task = @task_store.find(id)
218
+ next unless task && TERMINAL.include?(task.status.to_s)
219
+ next unless task.updated_at && task.updated_at < cutoff.iso8601
220
+
221
+ @checkpoint_store.purge(id)
222
+ @model_visible_trace_store&.purge(id)
223
+ @task_store.delete(id)
224
+ removed += 1
225
+ end
226
+ removed
227
+ end
228
+
229
+ def sweep_outcomes(cutoff)
230
+ @outcome_store ? @outcome_store.delete_older_than(cutoff) : 0
231
+ end
232
+
233
+ # The delivered/failed outbox records past the cutoff. Their `payload` is
234
+ # the answer the customer received — conversation content, so it ages out
235
+ # with the rest of the footprint instead of living in the store forever.
236
+ def sweep_outbox(cutoff)
237
+ @outbox_store ? @outbox_store.delete_older_than(cutoff) : 0
238
+ end
239
+
240
+ # Shadow pairs past the cutoff, TERMINAL statuses only
241
+ # (judged/incomplete) — an open/complete pair older than the window is
242
+ # still someone's unjudged evidence, exactly like the outbox's rule.
243
+ def sweep_shadow_pairs(cutoff)
244
+ @shadow_pair_store ? @shadow_pair_store.delete_older_than(cutoff) : 0
245
+ end
246
+
247
+ # the funnel DAY CELLS die with the outcomes they fold — same
248
+ # retention_days gate, same daily claim. Cursors/baselines live while their
249
+ # agent does. nil collaborator = nothing to sweep (base graph, parity).
250
+ def sweep_funnel(cutoff)
251
+ @funnel_store.delete_older_than(cutoff)
252
+ end
253
+
254
+ def claim_window = claim(KEY)
255
+
256
+ # The daily claim: one key, a timestamp, a 24 h window — the tick's
257
+ # claim_window idiom (a key per day would be a slow leak; the window is
258
+ # long enough that the store never grows). One key per SWEEP (the age-based
259
+ # one, the budget GC): they gate on different knobs, so a single shared key
260
+ # would let whichever ran first starve the other for a day.
261
+ def claim(key)
262
+ now_time = now
263
+ @store.transaction do
264
+ current = @store.get(SCOPE, key)
265
+ last = current && begin
266
+ Time.iso8601(current["claimed_at"].to_s)
267
+ rescue ArgumentError
268
+ nil
269
+ end
270
+ if last.nil? || (now_time - last) >= @window
271
+ @store.set(SCOPE, key, { "claimed_at" => now_time.iso8601 })
272
+ true
273
+ else
274
+ false
275
+ end
276
+ end
277
+ end
278
+
279
+ def now = @now || Time.now.utc
280
+ end
281
+ end