insika 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (185) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +145 -17
  3. data/README.md +7 -1
  4. data/bin/insika +211 -8
  5. data/docs/AGENTS.md +377 -2
  6. data/docs/API.md +56 -0
  7. data/docs/CHANNELS.md +99 -9
  8. data/docs/CONTEXT.md +127 -16
  9. data/docs/DEPLOY.md +5 -2
  10. data/docs/EVALS.md +19 -0
  11. data/docs/FACTS.md +135 -0
  12. data/docs/HARVEST.md +117 -0
  13. data/docs/LOADTEST.md +14 -8
  14. data/docs/OBSERVABILITY.md +54 -0
  15. data/docs/REFINEMENT.md +3 -3
  16. data/docs/RELEASING.md +27 -0
  17. data/docs/RUNNING-LOCAL.md +3 -3
  18. data/docs/SECURITY.md +61 -0
  19. data/docs/SKILLS.md +2 -2
  20. data/docs/SOAK.md +127 -0
  21. data/docs/TOOLS.md +70 -2
  22. data/docs/domain.md +115 -0
  23. data/docs/index.md +1 -1
  24. data/lib/insika/agent_profile.rb +145 -10
  25. data/lib/insika/balloon_splitter.rb +102 -0
  26. data/lib/insika/budget_ledger.rb +34 -3
  27. data/lib/insika/cache_series_store.rb +49 -0
  28. data/lib/insika/channel_delivery.rb +118 -13
  29. data/lib/insika/channels/relay.rb +77 -3
  30. data/lib/insika/channels/web.rb +2 -2
  31. data/lib/insika/chat_builder.rb +84 -1
  32. data/lib/insika/checkpoint_store.rb +16 -0
  33. data/lib/insika/commands/agent_payload.rb +3 -3
  34. data/lib/insika/commands/cancel_followup.rb +49 -0
  35. data/lib/insika/commands/delete_tenant_data.rb +95 -0
  36. data/lib/insika/commands/export_customer_memory.rb +48 -0
  37. data/lib/insika/commands/forget_customer.rb +117 -0
  38. data/lib/insika/commands/freeze_funnel_baseline.rb +113 -0
  39. data/lib/insika/commands/gate_harvest.rb +138 -0
  40. data/lib/insika/commands/gate_refinement.rb +1 -1
  41. data/lib/insika/commands/judge_shadow_pairs.rb +124 -0
  42. data/lib/insika/commands/memory_forget_fact.rb +20 -4
  43. data/lib/insika/commands/memory_put_fact.rb +23 -4
  44. data/lib/insika/commands/promote_harvest.rb +130 -0
  45. data/lib/insika/commands/record_outcome.rb +46 -0
  46. data/lib/insika/commands/record_shadow_reply.rb +68 -0
  47. data/lib/insika/commands/reject_harvest.rb +38 -0
  48. data/lib/insika/commands/resolve_proposal.rb +108 -0
  49. data/lib/insika/commands/revoke_contact.rb +49 -0
  50. data/lib/insika/commands/rollback_harvest.rb +86 -0
  51. data/lib/insika/commands/run_distillation.rb +186 -0
  52. data/lib/insika/commands/run_harvest.rb +393 -0
  53. data/lib/insika/commands/send_message.rb +103 -6
  54. data/lib/insika/commands/session_purge.rb +67 -0
  55. data/lib/insika/contact_store.rb +183 -0
  56. data/lib/insika/context/builder.rb +21 -3
  57. data/lib/insika/context/fragment.rb +7 -3
  58. data/lib/insika/context/priority.rb +3 -0
  59. data/lib/insika/context/provider.rb +17 -3
  60. data/lib/insika/context/providers/briefing.rb +96 -0
  61. data/lib/insika/context/providers/memory.rb +16 -7
  62. data/lib/insika/context/providers/prompt.rb +30 -2
  63. data/lib/insika/context/providers/skill.rb +2 -0
  64. data/lib/insika/context/providers/tool_search.rb +2 -0
  65. data/lib/insika/context_trace_store.rb +38 -2
  66. data/lib/insika/distill.rb +224 -0
  67. data/lib/insika/distill_engine.rb +169 -0
  68. data/lib/insika/doctor.rb +716 -6
  69. data/lib/insika/dsl/runtime.rb +9 -3
  70. data/lib/insika/dsl/server_boot.rb +70 -1
  71. data/lib/insika/dsl.rb +83 -3
  72. data/lib/insika/edge_limiter.rb +28 -7
  73. data/lib/insika/env_schema.rb +9 -2
  74. data/lib/insika/errors.rb +11 -0
  75. data/lib/insika/evals/assertions.rb +3 -2
  76. data/lib/insika/evals/pairwise.rb +18 -6
  77. data/lib/insika/evidence.rb +183 -0
  78. data/lib/insika/executor.rb +687 -70
  79. data/lib/insika/followup_engine.rb +207 -0
  80. data/lib/insika/followup_policy.rb +221 -0
  81. data/lib/insika/followup_store.rb +306 -0
  82. data/lib/insika/funnel_declaration.rb +106 -0
  83. data/lib/insika/funnel_fold.rb +179 -0
  84. data/lib/insika/funnel_store.rb +163 -0
  85. data/lib/insika/golden_store.rb +1 -1
  86. data/lib/insika/grounding/matcher.rb +69 -0
  87. data/lib/insika/grounding.rb +44 -0
  88. data/lib/insika/harvest/conversion_gate.rb +159 -0
  89. data/lib/insika/harvest/criterion.rb +98 -0
  90. data/lib/insika/harvest/gate.rb +194 -0
  91. data/lib/insika/harvest/negative_list.rb +199 -0
  92. data/lib/insika/harvest.rb +241 -0
  93. data/lib/insika/harvest_engine.rb +193 -0
  94. data/lib/insika/harvest_store.rb +548 -0
  95. data/lib/insika/media.rb +298 -0
  96. data/lib/insika/memory_audit_store.rb +85 -0
  97. data/lib/insika/memory_store.rb +264 -23
  98. data/lib/insika/message_origin.rb +6 -1
  99. data/lib/insika/model_visible.rb +87 -0
  100. data/lib/insika/model_visible_trace_store.rb +66 -0
  101. data/lib/insika/onboarding.rb +6 -1
  102. data/lib/insika/outbox_store.rb +42 -4
  103. data/lib/insika/outcome_store.rb +147 -0
  104. data/lib/insika/packaging.rb +163 -0
  105. data/lib/insika/parity/criterion.rb +79 -0
  106. data/lib/insika/parity/verdict.rb +318 -0
  107. data/lib/insika/prefix_fingerprint.rb +58 -0
  108. data/lib/insika/profile_source.rb +22 -1
  109. data/lib/insika/proposal_store.rb +271 -0
  110. data/lib/insika/queue_policy.rb +4 -1
  111. data/lib/insika/refinement/proposer.rb +1 -1
  112. data/lib/insika/reliability.rb +32 -6
  113. data/lib/insika/retention.rb +281 -0
  114. data/lib/insika/routing.rb +101 -0
  115. data/lib/insika/safety/config.rb +44 -4
  116. data/lib/insika/safety/corpus.rb +255 -0
  117. data/lib/insika/safety/detectors.rb +32 -113
  118. data/lib/insika/safety/factory.rb +15 -2
  119. data/lib/insika/safety/grounding_enforcer.rb +59 -0
  120. data/lib/insika/safety/grounding_validator.rb +49 -0
  121. data/lib/insika/safety/input_guardrail.rb +1 -1
  122. data/lib/insika/safety/moderator.rb +1 -1
  123. data/lib/insika/safety/output_filter.rb +9 -5
  124. data/lib/insika/safety/output_validator.rb +11 -5
  125. data/lib/insika/schema_guard.rb +35 -0
  126. data/lib/insika/server/app.rb +144 -4
  127. data/lib/insika/server/rack_app.rb +16 -0
  128. data/lib/insika/server/responses.rb +40 -1
  129. data/lib/insika/session_actor.rb +8 -4
  130. data/lib/insika/session_store.rb +65 -2
  131. data/lib/insika/settings_store.rb +10 -0
  132. data/lib/insika/shadow_pair_store.rb +258 -0
  133. data/lib/insika/skill_catalog.rb +4 -0
  134. data/lib/insika/soak/envelope.rb +140 -0
  135. data/lib/insika/soak/report.rb +392 -0
  136. data/lib/insika/soak/runner.rb +554 -0
  137. data/lib/insika/store.rb +10 -1
  138. data/lib/insika/stores/memory.rb +6 -0
  139. data/lib/insika/stores/sqlite.rb +8 -0
  140. data/lib/insika/studio/app.rb +875 -14
  141. data/lib/insika/studio/assets/dist/application.css +1 -1
  142. data/lib/insika/studio/assets/dist/application.js +22 -22
  143. data/lib/insika/studio/assets/dist/favicon.svg +6 -0
  144. data/lib/insika/studio/forms.rb +262 -5
  145. data/lib/insika/studio/nav_icons.rb +6 -1
  146. data/lib/insika/studio/views/agent_detail.erb +627 -84
  147. data/lib/insika/studio/views/agents.erb +10 -6
  148. data/lib/insika/studio/views/approvals.erb +4 -1
  149. data/lib/insika/studio/views/chats.erb +4 -1
  150. data/lib/insika/studio/views/customer.erb +94 -0
  151. data/lib/insika/studio/views/customers.erb +32 -0
  152. data/lib/insika/studio/views/evals.erb +4 -1
  153. data/lib/insika/studio/views/facts.erb +133 -0
  154. data/lib/insika/studio/views/followups.erb +125 -0
  155. data/lib/insika/studio/views/funnel.erb +106 -0
  156. data/lib/insika/studio/views/harvest.erb +234 -0
  157. data/lib/insika/studio/views/home.erb +2 -1
  158. data/lib/insika/studio/views/layout.erb +1 -0
  159. data/lib/insika/studio/views/parity.erb +147 -0
  160. data/lib/insika/studio/views/playground.erb +7 -1
  161. data/lib/insika/studio/views/session.erb +56 -1
  162. data/lib/insika/studio/views/settings.erb +2 -0
  163. data/lib/insika/studio/views/skills.erb +0 -1
  164. data/lib/insika/studio/views/system_files.erb +1 -1
  165. data/lib/insika/studio/views/task.erb +13 -0
  166. data/lib/insika/studio/views/tasks.erb +4 -1
  167. data/lib/insika/studio/views/tools.erb +0 -1
  168. data/lib/insika/task_store.rb +21 -1
  169. data/lib/insika/testing/store_contract.rb +27 -6
  170. data/lib/insika/tick.rb +25 -1
  171. data/lib/insika/tool_definition.rb +17 -7
  172. data/lib/insika/tool_envelope.rb +69 -0
  173. data/lib/insika/tool_manifest.rb +5 -1
  174. data/lib/insika/tools/data_defined_tool.rb +10 -0
  175. data/lib/insika/tools/generate_image.rb +44 -0
  176. data/lib/insika/tools/schedule_followup.rb +164 -0
  177. data/lib/insika/tools/tts.rb +47 -0
  178. data/lib/insika/tools/update_briefing.rb +126 -0
  179. data/lib/insika/turn_state.rb +38 -1
  180. data/lib/insika/turn_timing.rb +22 -2
  181. data/lib/insika/version.rb +1 -1
  182. data/lib/insika/vitals.rb +84 -0
  183. data/lib/insika/wiring/graph.rb +300 -13
  184. data/lib/insika.rb +92 -3
  185. metadata +77 -1
@@ -0,0 +1,393 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "time"
4
+
5
+ module Insika
6
+ module Commands
7
+ # the ONLY path that writes candidates. Mines ONE window end
8
+ # to end: resolve the sessions, read the transcripts + evidence, ask the
9
+ # miner, schema-drop, apply the negative list and the grounding filter,
10
+ # dedup against the ledger, write the run + candidates, stamp the markers.
11
+ #
12
+ # Synchronous (it runs on the engine's worker fiber, C12, or the CLI, C15)
13
+ # — it creates no task and no turn. It writes NOTHING to sessions or
14
+ # skills: D2's fork discipline is this command's contract — a customer
15
+ # turn's prefix is untouched by construction, and the only harvest-side
16
+ # spend is the run's cost (E1).
17
+ #
18
+ # Payload: { agent:, last_sessions?, since?, full?, session_ids?,
19
+ # max_proposals?, exclude_sessions? }
20
+ class RunHarvest
21
+ DEFAULT_LAST_SESSIONS = 200 # the EvidenceCollector's window default
22
+ DEFAULT_MIN_MESSAGES = 3
23
+ DEFAULT_MAX_PROPOSALS = 10
24
+ DEFAULT_IDLE_HOURS = 24
25
+
26
+ def initialize(profiles:, harvest_store:, session_store:, task_store:,
27
+ skill_store: nil, tool_trace_store: nil, settings_store: nil,
28
+ negative_list: nil, miner_factory: nil, event_stream:)
29
+ @profiles = ProfileSource.coerce(profiles)
30
+ @harvest_store = harvest_store
31
+ @session_store = session_store
32
+ @task_store = task_store
33
+ @skill_store = skill_store
34
+ @tool_trace_store = tool_trace_store
35
+ @settings_store = settings_store
36
+ @negative_list = negative_list
37
+ @miner_factory = miner_factory ||
38
+ ->(config) { Harvest::MinerFactory.build(config, utility_model: utility_model) }
39
+ @event_stream = event_stream
40
+ end
41
+
42
+ # -> { mined: true, run_id:, candidates: N,
43
+ # rejected: { "<rule-id>" => N, "ungrounded" => N, "dedup" => N,
44
+ # "schema" => N, ... }, cost: {...} | nil }
45
+ # | { mined: false, skipped: "disabled|no_model|no_grounding_matcher" }
46
+ def call(command)
47
+ p = AgentPayload.symbolize(command.payload)
48
+ agent = AgentPayload.presence(p[:agent])
49
+ raise Insika::ValidationError, "agent is required" if agent.nil?
50
+
51
+ profile = @profiles[agent] ||
52
+ (raise Insika::NotFoundError, "agent '#{agent}' not configured")
53
+ config = Coercion.deep_stringify(profile.harvest)
54
+ return skip("disabled") if config.nil? || !Coercion.truthy?(config["enabled"])
55
+
56
+ # D4: the OPERATIVE negative list lives on the profile (hot-editable,
57
+ # seeded by `insika harvest:negative import`); the injected list is
58
+ # the deployment's fallback. The engine applies data, never authors it.
59
+ @list = negative_list_for(config)
60
+
61
+ # D3: product claims cannot be verified without a matcher, so NOTHING
62
+ # mines — refused, not warned (D12's "by refusal, not by prompt").
63
+ grounding = Grounding.parse(profile.grounding)
64
+ return skip("no_grounding_matcher") if grounding.nil? || !grounding.matcher.sku?
65
+
66
+ miner = @miner_factory.call(config)
67
+ return skip("no_model") if miner.nil?
68
+
69
+ run = @harvest_store.create_run(agent_id: agent, window: window_record(p),
70
+ budget: budget_cap(config))
71
+ sessions = resolve_sessions(agent, p, config)
72
+
73
+ begin
74
+ # No eligible sessions: a run that says "we looked and it was clean"
75
+ # without paying a bill; markers untouched (re-scan, D10).
76
+ if sessions.empty?
77
+ @harvest_store.complete_run(run.id, candidates: 0)
78
+ return { mined: true, run_id: run.id, candidates: 0,
79
+ rejected: empty_rejected, cost: nil }
80
+ end
81
+
82
+ prompt = build_prompt(config, sessions, agent)
83
+ result = miner.mine(prompt: prompt,
84
+ message_counts: sessions.map { |s| s[:messages].size },
85
+ max_proposals: max_proposals(p, config))
86
+
87
+ # The mining budget is a REAL cap (the review fix): a pass that
88
+ # spent more than the pack declared is failed with the numbers and
89
+ # proposes NOTHING — the docs' "the budget cap bounds it" is
90
+ # enforced here, post-hoc for the one model call, pre-hoc for every
91
+ # downstream write and gate.
92
+ if budget_exceeded?(budget_cap(config), result[:cost])
93
+ @harvest_store.fail_run(run.id, error: "mining budget exceeded: " \
94
+ "spent #{result[:cost]['spent']} > " \
95
+ "#{budget_cap(config)['tokens']}")
96
+ emit(:harvest_mined, agent: agent, run_id: run.id, candidates: 0,
97
+ rejected: empty_rejected, cost: result[:cost])
98
+ return { mined: true, run_id: run.id, candidates: 0,
99
+ rejected: empty_rejected, cost: result[:cost] }
100
+ end
101
+
102
+ survivors, rejected = filter_skills(result[:skills], sessions, agent, grounding.matcher)
103
+
104
+ survivors.each do |skill|
105
+ @harvest_store.create_candidate(
106
+ run_id: run.id, agent: agent, name: skill["name"],
107
+ description: skill["description"], body: skill_md(skill),
108
+ triggers: skill["triggers"] || [], rationale: skill["rationale"].to_s,
109
+ origin: sessions.map { |s| s[:id] }, evidence_turns: skill["evidence_turns"] || [],
110
+ proposer: miner.model
111
+ )
112
+ end
113
+
114
+ final_rejected = merge_rejected(result[:dropped], rejected)
115
+ @harvest_store.complete_run(run.id, candidates: survivors.size,
116
+ cost: result[:cost],
117
+ rejected: final_rejected)
118
+ # Markers AFTER the pass completes (D10's crash-safe re-scan).
119
+ sessions.each { |s| @harvest_store.mark_mined(s[:id], candidates: survivors.size) }
120
+
121
+ emit(:harvest_mined, agent: agent, run_id: run.id, candidates: survivors.size,
122
+ rejected: final_rejected, cost: result[:cost])
123
+ { mined: true, run_id: run.id, candidates: survivors.size,
124
+ rejected: final_rejected, cost: result[:cost] }
125
+ rescue StandardError => e
126
+ # The run is failed; the markers are NOT written (re-scan, D10);
127
+ # the exception propagates to the caller's fiber or the CLI.
128
+ begin
129
+ @harvest_store.fail_run(run.id, error: e.message)
130
+ rescue StandardError
131
+ nil
132
+ end
133
+ raise
134
+ end
135
+ end
136
+
137
+ private
138
+
139
+ def skip(reason)
140
+ { mined: false, skipped: reason }
141
+ end
142
+
143
+ # The pack's harvest.miner.budget as { "tokens" => N } — nil when
144
+ # absent/not positive (no cap, the refinement discipline).
145
+ def budget_cap(config)
146
+ raw = config && config.dig("miner", "budget")
147
+ raw.is_a?(Hash) && raw["tokens"].to_i.positive? ? { "tokens" => raw["tokens"].to_i } : nil
148
+ end
149
+
150
+ def budget_exceeded?(cap, cost)
151
+ cap && cost && cost["spent"].to_i > cap["tokens"].to_i
152
+ end
153
+
154
+ # The candidate's body is a full SKILL.md by construction: the model
155
+ # writes the procedure, the engine wraps the frontmatter (name /
156
+ # description / triggers — the miner's own fields), so a candidate is
157
+ # servable the moment a human promotes it (WriteSkill validates the
158
+ # frontmatter, and the gate's clone serves it the same way).
159
+ def skill_md(skill)
160
+ fm = { "name" => skill["name"], "description" => skill["description"] }
161
+ fm["triggers"] = Array(skill["triggers"]).join(", ") if Array(skill["triggers"]).any?
162
+ "---\n#{fm.map { |k, v| "#{k}: #{v}" }.join("\n")}\n---\n#{skill['body']}"
163
+ end
164
+
165
+ # The profile's own rules win over the injected seed (D4). A malformed
166
+ # profile list parses to nil — the injected list stays the fallback,
167
+ # never a silent empty.
168
+ def negative_list_for(config)
169
+ return @negative_list unless config && config["negative_list"]
170
+
171
+ Harvest::NegativeList.parse(config["negative_list"]) || @negative_list
172
+ end
173
+
174
+ def empty_rejected = { "schema" => 0, "unknown_key" => 0, "oversized" => 0,
175
+ "bad_turns" => 0, "duplicate" => 0, "capped" => 0 }
176
+
177
+ def window_record(p)
178
+ return { "session_ids" => Array(p[:session_ids]).map(&:to_s) } if p[:session_ids]
179
+ return { "since" => p[:since].to_s } if AgentPayload.presence(p[:since])
180
+ return { "last_sessions" => Integer(p[:last_sessions]) } if p[:last_sessions]
181
+
182
+ {}
183
+ end
184
+
185
+ # The EvidenceCollector window discipline, applied to session ids: the
186
+ # agent's turns (by the task's command payload), distinct session ids,
187
+ # newest first, capped at the "last N conversations". Marker suppression
188
+ # unless `full` (a re-mine is explicit). A session's evidence ledger
189
+ # is the grounding filter's input.
190
+ def resolve_sessions(agent, p, config)
191
+ ids = window_session_ids(agent, p, config)
192
+ ids = ids.reject { |sid| @harvest_store.mined?(sid) } unless EnvSchema.truthy?(p[:full])
193
+ ids = ids.first(Harvest::Miner::MAX_SESSIONS)
194
+
195
+ min_messages = (config["min_messages"] || DEFAULT_MIN_MESSAGES).to_i
196
+ idle_hours = (config["idle_hours"] || DEFAULT_IDLE_HOURS).to_i
197
+ ids.filter_map do |sid|
198
+ session = @session_store.find(sid)
199
+ next unless session
200
+
201
+ messages = session.messages.to_a
202
+ next if messages.size < min_messages # a 2-message session mines noise
203
+ # D10: the per-agent maturity bound is re-checked here — the engine
204
+ # scan's default is only the LOWER bound (a pack that wants 12 h is
205
+ # never mined at 6, and a manual CLI run over fresh traffic mines
206
+ # nothing).
207
+ next unless idle?(session.updated_at, idle_hours)
208
+
209
+ evidence = (session.evidence || {})["ids"] || []
210
+ { id: sid.to_s, messages: messages, evidence: Array(evidence).map(&:to_s) }
211
+ end
212
+ end
213
+
214
+ def window_session_ids(agent, p, config)
215
+ explicit = Array(p[:session_ids]).map(&:to_s)
216
+ return explicit unless explicit.empty?
217
+
218
+ since = AgentPayload.presence(p[:since])
219
+ tasks = @task_store.each_id
220
+ .filter_map { |id| @task_store.find(id) }
221
+ .select { |t| task_agent(t) == agent }
222
+ .sort_by { |t| [t.created_at.to_s, t.id] }.reverse
223
+
224
+ tasks = tasks.select { |t| t.created_at.to_s >= since.to_s } if since
225
+
226
+ distinct = []
227
+ seen = {}
228
+ tasks.each do |t|
229
+ sid = presence(t.session_id)
230
+ next if sid.nil? || seen[sid]
231
+
232
+ seen[sid] = true
233
+ distinct << sid
234
+ end
235
+ return distinct if since
236
+
237
+ last = AgentPayload.presence(p[:last_sessions])
238
+ count = last ? Integer(last) : config_window(config)
239
+ distinct.first(count)
240
+ end
241
+
242
+ # The config's miner.window last_sessions, or the collector default.
243
+ def config_window(config)
244
+ configured = config.dig("miner", "window", "last_sessions")
245
+ configured ? Integer(configured) : DEFAULT_LAST_SESSIONS
246
+ end
247
+
248
+ def task_agent(task)
249
+ task.command.is_a?(Hash) ? task.command.dig("payload", "agent").to_s : ""
250
+ end
251
+
252
+ def idle?(updated_at, idle_hours)
253
+ return false if Coercion.blank?(updated_at)
254
+
255
+ Time.iso8601(updated_at.to_s) <= Time.now.utc - idle_hours * 3600
256
+ rescue ArgumentError
257
+ false
258
+ end
259
+
260
+ def presence(value)
261
+ Insika::Coercion.presence(value)
262
+ end
263
+
264
+ def max_proposals(p, config)
265
+ raw = p[:max_proposals] || config.dig("miner", "max_proposals")
266
+ raw ? Integer(raw) : DEFAULT_MAX_PROPOSALS
267
+ end
268
+
269
+ # The prompt: the transcript slices (masked through the output
270
+ # filter — the redaction rule), the evidence ids per
271
+ # session, the agent's CURRENT skill names (the model should not
272
+ # re-propose them), and the answer rules. The pack prompt replaces
273
+ # DEFAULT_PROMPT whole (the forge's half).
274
+ def build_prompt(config, sessions, agent)
275
+ base = Coercion.presence(config["prompt"]) || Harvest::DEFAULT_PROMPT
276
+ current_skills = current_skill_names(agent)
277
+ blocks = sessions.map do |s|
278
+ evidence = s[:evidence].join(", ")
279
+ lines = ["## Session #{s[:id]} (#{s[:messages].size} messages; " \
280
+ "evidence ids: #{evidence.empty? ? '(none)' : evidence})"]
281
+ lines << render_transcript(s[:messages])
282
+ lines.join("\n")
283
+ end
284
+ <<~PROMPT
285
+ #{base.rstrip}
286
+
287
+ ## Conversations to mine
288
+
289
+ #{blocks.join("\n")}
290
+
291
+ ## Skills this agent already has (do not re-propose them)
292
+
293
+ #{current_skills.empty? ? "(none)" : current_skills.join(", ")}
294
+ PROMPT
295
+ end
296
+
297
+ def current_skill_names(agent)
298
+ return [] unless @skill_store
299
+
300
+ (@skill_store.names | @skill_store.names(agent: agent)).sort
301
+ end
302
+
303
+ def render_transcript(messages)
304
+ redacted, = Insika::Safety::Detectors.redact(
305
+ messages.each_with_index.map { |m, i| "[#{i}] #{m['role']}: #{m['content']}" }.join("\n")
306
+ )
307
+ redacted
308
+ end
309
+
310
+ # The safety order: negative list (D4) -> grounding (D3) -> dedup. A
311
+ # drop is COUNTED and NEVER written. `rejected` keys: the matched RULE
312
+ # ids for negative hits (each named — E2), "ungrounded"/"dedup" for the
313
+ # other two filters.
314
+ # -> [[raw skill], { key => count }]
315
+ def filter_skills(skills, sessions, agent, matcher)
316
+ rejected = Hash.new(0)
317
+ survivors = skills.select do |skill|
318
+ hits = negative_hits(skill)
319
+ if hits.any?
320
+ hits.each { |r| rejected[r.rule] += 1 }
321
+ false
322
+ elsif !grounded?(skill, sessions, matcher)
323
+ rejected["ungrounded"] += 1
324
+ false
325
+ elsif deduped?(skill, agent)
326
+ rejected["dedup"] += 1
327
+ false
328
+ else
329
+ true
330
+ end
331
+ end
332
+ [survivors, rejected.to_h]
333
+ end
334
+
335
+ # D4: every rejected-by-list candidate is logged with the matching rule
336
+ # id. The BODY is part of the match (the review fix): it is exactly what
337
+ # enters the model's context when the skill loads, so a banned phrase
338
+ # hidden there is the Hermes failure the design cites — the same text
339
+ # the grounding filter already reads (name+description+body). The name
340
+ # uses the stricter substring reading, the prose the word-boundary one.
341
+ def negative_hits(skill)
342
+ return [] unless @list
343
+
344
+ text = [skill["name"], skill["description"], skill["body"]].join(" ")
345
+ hits = @list.matches_name(skill["name"].to_s)
346
+ hits | @list.matches(text)
347
+ end
348
+
349
+ # D3: every reference must be in the union of the origin sessions'
350
+ # persisted evidence ids. The empty-ledger conservative reading (the
351
+ # enforcer's rule): a candidate whose origin sessions hold NO evidence
352
+ # ids and that carries a reference is dropped too. A skill with no
353
+ # references is not a grounding casualty. -> bool
354
+ def grounded?(skill, sessions, matcher)
355
+ text = [skill["name"], skill["description"], skill["body"]].join(" ")
356
+ refs = matcher.references(text)
357
+ return true if refs.empty?
358
+
359
+ union = sessions.flat_map { |s| s[:evidence] }.uniq
360
+ return false if union.empty?
361
+
362
+ missing = refs.reject { |r| union.include?(r) }
363
+ missing.empty?
364
+ end
365
+
366
+ # The dedup ledger: the same name already in the store's SkillStore
367
+ # (shared or agent scope) or an open (agent, name) tuple.
368
+ def deduped?(skill, agent)
369
+ name = skill["name"].to_s
370
+ store_has = @skill_store && (@skill_store.get(name) || @skill_store.get(name, agent: agent))
371
+ store_has || @harvest_store.open_pending?(agent: agent, name: name)
372
+ end
373
+
374
+ # The miner's schema drops + the filters' — one record; the keys never
375
+ # collide (the filters own rule ids and their own names).
376
+ def merge_rejected(dropped, filters)
377
+ dropped.merge(filters) { |_k, left, right| left + right }
378
+ end
379
+
380
+ def utility_model
381
+ return nil unless @settings_store
382
+
383
+ @settings_store.get["utility_model"]
384
+ end
385
+
386
+ def emit(type, **data)
387
+ @event_stream.emit(Insika::Event.new(
388
+ type: type, data: data, meta: { at: Time.now.utc.iso8601 }
389
+ ))
390
+ end
391
+ end
392
+ end
393
+ end
@@ -20,12 +20,22 @@ module Insika
20
20
  # nil = no dedup (every surface that does not send an `event_id`, which is all
21
21
  # of them today), and a caller that cannot supply a stable id gets
22
22
  # at-least-once turns rather than a content hash pretending to be dedup.
23
- def initialize(profiles:, session_store:, task_store:, executor:, inbound_log: nil)
23
+ #
24
+ # `contact_store`/`followup_store` : the contact-state hook —
25
+ # a customer message reopens the contact cell and a cancellation keyword
26
+ # revokes it + falls the pending records. nil = the hook is off (parity).
27
+ # `store` is the SHARED backend the two stores ride — the keyword revoke
28
+ # commits in ONE transaction (D2). nil = best-effort separate writes.
29
+ def initialize(profiles:, session_store:, task_store:, executor:, inbound_log: nil,
30
+ contact_store: nil, followup_store: nil, store: nil)
24
31
  @profiles = ProfileSource.coerce(profiles)
25
32
  @session_store = session_store
26
33
  @task_store = task_store
27
34
  @executor = executor
28
35
  @inbound_log = inbound_log
36
+ @contact_store = contact_store
37
+ @followup_store = followup_store
38
+ @store = store
29
39
  end
30
40
 
31
41
  def call(command)
@@ -37,8 +47,15 @@ module Insika
37
47
  profile = @profiles[agent] ||
38
48
  (raise Insika::NotFoundError, "agent '#{agent}' not configured")
39
49
 
50
+ # A turn is text OR media. The media half (WS9) is what a voice note
51
+ # with no caption looks like on the wire — `{ parts: [{type: "audio",
52
+ # url: …}] }` and nothing else — and demanding a message here made the
53
+ # anchor use case unreachable end to end: the audio becomes the message
54
+ # at the :media stage, one step later.
40
55
  message = p[:message]
41
- raise Insika::ValidationError, "message is required and non-empty" if message.to_s.strip.empty?
56
+ if message.to_s.strip.empty? && !media?(p[:parts])
57
+ raise Insika::ValidationError, "message is required and non-empty (or a media part)"
58
+ end
42
59
 
43
60
  # session_id XOR history (both -> error; neither -> one-shot).
44
61
  if p[:session_id] && p[:history]
@@ -50,7 +67,14 @@ module Insika
50
67
  # is what a turn has always meant. Refused here rather than downstream: a
51
68
  # typo'd origin would read as absent, and a marker that silently means
52
69
  # "unmarked" is worse than none — it looks like the filtering is on.
53
- Insika::MessageOrigin.parse!(p[:origin])
70
+ origin = Insika::MessageOrigin.parse!(p[:origin])
71
+ # `scheduled` is ENGINE-RESERVED — the FollowupEngine's
72
+ # synthetic turn stamps it, and the edge must not let a consumer
73
+ # impersonate the engine's kick (a spoofed follow-up is the spam bug).
74
+ if origin == Insika::MessageOrigin::SCHEDULED
75
+ raise Insika::ValidationError,
76
+ "origin 'scheduled' is engine-reserved: it is stamped by the follow-up engine only"
77
+ end
54
78
  if p[:session_id]
55
79
  @session_store.find(p[:session_id]) ||
56
80
  (raise Insika::NotFoundError, "session '#{p[:session_id]}' not found")
@@ -66,6 +90,14 @@ module Insika
66
90
  return { task_id: prior, duplicate: true }
67
91
  end
68
92
 
93
+ # the contact-state hook — the ONLY path that sees every
94
+ # customer message. A real customer message reopens the contact cell; a
95
+ # cancellation keyword revokes it and falls the pending records in ONE
96
+ # transaction. Runs AFTER validation and dedup: a refused or duplicated
97
+ # message must not touch contact state. Nil-safe and policy-gated: no
98
+ # profile declaration = the hook is off (parity).
99
+ touch_contact(p, profile, command, origin)
100
+
69
101
  result = start_turn(command, p, profile)
70
102
  @inbound_log.record(key, result[:task_id]) if key
71
103
  result
@@ -73,6 +105,41 @@ module Insika
73
105
 
74
106
  private
75
107
 
108
+ # the contact bookkeeping of a customer message. The
109
+ # keyword cast IS a revocation: the customer just said the shut-off
110
+ # words — the pending records must fall with the state (D2, ONE
111
+ # transaction). The reset-on-origin list: only an origin that is
112
+ # nil/customer acts — an engine/operator-composed line (even one QUOTING
113
+ # the customer's words) never touches contact state. Nil collaborators
114
+ # or no profile declaration = the hook is a no-op (byte-parity).
115
+ def touch_contact(p, profile, command, origin)
116
+ return unless @contact_store
117
+ return unless origin.nil? || origin == Insika::MessageOrigin::CUSTOMER
118
+
119
+ policy = Insika::FollowupPolicy.parse(profile.followup)
120
+ return unless policy
121
+
122
+ customer = p[:customer]
123
+ return if customer.to_s.empty?
124
+
125
+ keyword = policy.match_keyword(p[:message].to_s)
126
+ if keyword
127
+ # one transaction on the shared backend: the revoke and the pending
128
+ # fall commit together — a half-cancelled opt-out is the spam bug.
129
+ if @store
130
+ @store.transaction do
131
+ @contact_store.set_revoked(tenant: command.meta[:tenant], customer: customer)
132
+ @followup_store&.cancel_pending_for(tenant: command.meta[:tenant], customer: customer)
133
+ end
134
+ else
135
+ @contact_store.set_revoked(tenant: command.meta[:tenant], customer: customer)
136
+ @followup_store&.cancel_pending_for(tenant: command.meta[:tenant], customer: customer)
137
+ end
138
+ else
139
+ @contact_store.set_granted(tenant: command.meta[:tenant], customer: customer)
140
+ end
141
+ end
142
+
76
143
  # The turn (or the verdict that this message joined someone else's).
77
144
  def start_turn(command, p, profile)
78
145
  message = p[:message]
@@ -83,7 +150,10 @@ module Insika
83
150
  # boot). Only offered on a surface that can report the verdict back —
84
151
  # coalescing a caller that cannot hear `merged` makes it deliver the
85
152
  # same answer twice.
86
- if coalescable?(command)
153
+ # A message carrying MEDIA never joins another turn: `collect`/`steer`
154
+ # move TEXT into a task that is already at the door, and its parts would
155
+ # be left behind — the customer's photo would silently not exist.
156
+ if coalescable?(command) && !media?(p[:parts])
87
157
  if (joined = @executor.collect_into_pending(p[:session_id], message, profile: profile))
88
158
  return { task_id: joined, merged: true }
89
159
  end
@@ -101,16 +171,41 @@ module Insika
101
171
  # command.to_h persists the entire Command in the Task;
102
172
  # ResumeTask re-reads payload.message from there.
103
173
  task = @task_store.create(command: command.to_h, session_id: p[:session_id])
174
+ # the channel clock starts HERE — the 202-owning request
175
+ # is accepted, before the SessionActor FIFO and the debounce window.
176
+ # `first_balloon_ms` is the wait the customer feels, so t0 is not the
177
+ # moment the turn finally runs; the same object travels with the turn
178
+ # and `:first_balloon` closes the window at the outbox flush.
104
179
  # `interrupt` mode: the turn in flight is now answering the wrong
105
180
  # question, so it is abandoned at its next boundary. This message keeps its OWN
106
181
  # task and its own reply (that is why it needs no verdict and no surface gate), and
107
182
  # the cancel is posted after `create` so the event can name what replaced what.
108
183
  # No-op in every other mode.
109
184
  @executor.interrupt_running(p[:session_id], profile: profile, replaced_by: task.id)
110
- @executor.spawn_in_session(task, profile: profile)
185
+ @executor.spawn_in_session(task, profile: profile,
186
+ timing: channel_inbound_timing(command))
111
187
  { task_id: task.id }
112
188
  end
113
189
 
190
+ # allocate the channel clock at 202 acceptance and stamp
191
+ # `:inbound` — the window's start. `breakdown: false` when INSIKA_TURN_TIMING
192
+ # is off, so a channel turn measures ONLY first_balloon_ms (H-latência never
193
+ # depends on the flag). nil for every non-channel transport: no clock to start.
194
+ def channel_inbound_timing(command)
195
+ return nil unless command.meta[:transport].to_s.start_with?("channel:")
196
+
197
+ timing = Insika::TurnTiming.new(breakdown: Insika::TurnTiming.enabled?)
198
+ timing.mark(:inbound)
199
+ timing
200
+ end
201
+
202
+ # Does the payload carry a part the engine will turn into the turn's
203
+ # substance — audio (transcribed into the message) or an image (attached
204
+ # to the ask)? A text part is not media: it is the message, spelled long.
205
+ def media?(parts)
206
+ Insika::Media.parts(parts).any? { |p| p.audio? || p.image? }
207
+ end
208
+
114
209
  def coalescable?(command)
115
210
  transport = command.meta[:transport]
116
211
  COALESCABLE_TRANSPORTS.include?(transport) || transport.to_s.start_with?("channel:")
@@ -133,7 +228,9 @@ module Insika
133
228
  session_id: payload[:session_id] || payload["session_id"],
134
229
  history: payload[:history] || payload["history"],
135
230
  origin: payload[:origin] || payload["origin"],
136
- event_id: payload[:event_id] || payload["event_id"]
231
+ event_id: payload[:event_id] || payload["event_id"],
232
+ parts: payload[:parts] || payload["parts"],
233
+ customer: payload[:customer] || payload["customer"]
137
234
  }
138
235
  end
139
236
 
@@ -0,0 +1,67 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Insika
4
+ module Commands
5
+ # The CONVERSATION footprint of a session, purged (WS8/LGPD). Shared by
6
+ # `forget_customer` and `delete_tenant_data` because "erase this person"
7
+ # and "erase this tenant" differ only in WHICH sessions they name — what a
8
+ # session leaves behind is the same list, and a second copy of that list is
9
+ # a second thing to forget to update.
10
+ #
11
+ # Deleting the session record alone is NOT erasure: the customer's own text
12
+ # lives in the task's persisted command payload, the whole transcript lives
13
+ # in the turn's checkpoints, and the answer as it was handed to the channel
14
+ # lives in the outbox record's payload. All four go together or none of them
15
+ # counts.
16
+ #
17
+ # A task is deleted whatever its status — this is a deletion order, not the
18
+ # retention sweep (which spares live tasks on purpose). The stores it needs
19
+ # beyond the session are optional (deployment components): a graph without
20
+ # them purges what it has and reports zero for the rest.
21
+ module SessionPurge
22
+ # ids: the session ids to erase. -> { tasks:, checkpoints:, deliveries:, pairs: }
23
+ def purge_sessions(ids)
24
+ ids = Array(ids).map(&:to_s)
25
+ return { tasks: 0, checkpoints: 0, deliveries: 0, pairs: 0 } if ids.empty?
26
+
27
+ tasks, checkpoints, model_visible = purge_tasks_of(ids)
28
+ deliveries = @outbox_store ? @outbox_store.purge_sessions(ids) : 0
29
+ pairs = @shadow_pairs ? @shadow_pairs.purge_sessions(ids) : 0
30
+ ids.each do |id|
31
+ @tool_trace_store&.clear(id)
32
+ @context_trace_store&.clear(id)
33
+ @session_store.delete(id)
34
+ end
35
+ { tasks: tasks, checkpoints: checkpoints, model_visible: model_visible,
36
+ deliveries: deliveries, pairs: pairs }
37
+ end
38
+
39
+ private
40
+
41
+ # -> [tasks removed, checkpoint records removed, model-visible records removed].
42
+ # the model-visible traces are transcripts — they die next to
43
+ # their checkpoints (the same single list, so "erase this person" cannot
44
+ # forget one half). The id list is SNAPSHOTTED (`to_a`) before the deletes:
45
+ # `each_id` enumerates the backend's keys lazily, and deleting under it
46
+ # would skip records.
47
+ def purge_tasks_of(ids)
48
+ return [0, 0, 0] unless @task_store
49
+
50
+ wanted = ids.each_with_object({}) { |id, acc| acc[id] = true }
51
+ tasks = 0
52
+ checkpoints = 0
53
+ model_visible = 0
54
+ @task_store.each_id.to_a.each do |task_id|
55
+ task = @task_store.find(task_id)
56
+ next unless task && wanted[task.session_id.to_s]
57
+
58
+ checkpoints += @checkpoint_store ? @checkpoint_store.purge(task_id) : 0
59
+ model_visible += @model_visible_trace_store ? @model_visible_trace_store.purge(task_id) : 0
60
+ @task_store.delete(task_id)
61
+ tasks += 1
62
+ end
63
+ [tasks, checkpoints, model_visible]
64
+ end
65
+ end
66
+ end
67
+ end