insika 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (280) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +199 -5
  3. data/README.md +8 -2
  4. data/bin/insika +231 -13
  5. data/docs/AGENTS.md +505 -6
  6. data/docs/API.md +56 -0
  7. data/docs/CHANNELS.md +100 -10
  8. data/docs/CONTEXT.md +147 -19
  9. data/docs/DEPLOY.md +34 -11
  10. data/docs/EMBEDDING.md +11 -7
  11. data/docs/EVALS.md +20 -1
  12. data/docs/FACTS.md +135 -0
  13. data/docs/HARVEST.md +117 -0
  14. data/docs/LOADTEST.md +17 -10
  15. data/docs/OBSERVABILITY.md +65 -2
  16. data/docs/REFINEMENT.md +9 -9
  17. data/docs/RELEASING.md +34 -7
  18. data/docs/RUNNING-LOCAL.md +4 -4
  19. data/docs/SECURITY.md +85 -11
  20. data/docs/SKILLS.md +189 -3
  21. data/docs/SOAK.md +127 -0
  22. data/docs/TOOLS.md +70 -2
  23. data/docs/WHY.md +1 -1
  24. data/docs/WORKFLOWS.md +2 -2
  25. data/docs/domain.md +115 -0
  26. data/docs/index.md +2 -2
  27. data/docs/onboarding/start.md +1 -1
  28. data/lib/insika/agent_profile.rb +228 -26
  29. data/lib/insika/alert_dispatcher.rb +139 -0
  30. data/lib/insika/balloon_splitter.rb +102 -0
  31. data/lib/insika/baseline_store.rb +2 -2
  32. data/lib/insika/budget_ledger.rb +166 -0
  33. data/lib/insika/cache_series_store.rb +49 -0
  34. data/lib/insika/channel_delivery.rb +132 -24
  35. data/lib/insika/channel_registry.rb +1 -1
  36. data/lib/insika/channels/relay.rb +80 -6
  37. data/lib/insika/channels/web/widget.js +2 -2
  38. data/lib/insika/channels/web.rb +9 -9
  39. data/lib/insika/channels/webhook.rb +58 -0
  40. data/lib/insika/chat_builder.rb +145 -13
  41. data/lib/insika/checkpoint_store.rb +16 -0
  42. data/lib/insika/circuit_state.rb +114 -0
  43. data/lib/insika/coercion.rb +8 -0
  44. data/lib/insika/commands/agent_payload.rb +6 -4
  45. data/lib/insika/commands/cancel_followup.rb +49 -0
  46. data/lib/insika/commands/create_agent.rb +2 -2
  47. data/lib/insika/commands/create_session.rb +1 -1
  48. data/lib/insika/commands/delete_llm_provider.rb +1 -1
  49. data/lib/insika/commands/delete_skill.rb +43 -0
  50. data/lib/insika/commands/delete_tenant_data.rb +95 -0
  51. data/lib/insika/commands/export_customer_memory.rb +48 -0
  52. data/lib/insika/commands/forget_customer.rb +117 -0
  53. data/lib/insika/commands/freeze_funnel_baseline.rb +113 -0
  54. data/lib/insika/commands/gate_harvest.rb +138 -0
  55. data/lib/insika/commands/gate_refinement.rb +12 -12
  56. data/lib/insika/commands/import_mcp_tools.rb +1 -1
  57. data/lib/insika/commands/import_tools.rb +4 -4
  58. data/lib/insika/commands/issue_tenant_token.rb +41 -0
  59. data/lib/insika/commands/judge_shadow_pairs.rb +124 -0
  60. data/lib/insika/commands/memory_forget_fact.rb +20 -4
  61. data/lib/insika/commands/memory_put_fact.rb +23 -4
  62. data/lib/insika/commands/promote_harvest.rb +130 -0
  63. data/lib/insika/commands/record_outcome.rb +46 -0
  64. data/lib/insika/commands/record_shadow_reply.rb +68 -0
  65. data/lib/insika/commands/reject_harvest.rb +38 -0
  66. data/lib/insika/commands/resolve_proposal.rb +108 -0
  67. data/lib/insika/commands/resolve_refinement.rb +1 -1
  68. data/lib/insika/commands/revoke_contact.rb +49 -0
  69. data/lib/insika/commands/revoke_token.rb +39 -0
  70. data/lib/insika/commands/rollback_harvest.rb +86 -0
  71. data/lib/insika/commands/rotate_tenant_token.rb +43 -0
  72. data/lib/insika/commands/run_distillation.rb +186 -0
  73. data/lib/insika/commands/run_harvest.rb +393 -0
  74. data/lib/insika/commands/run_refinement.rb +5 -5
  75. data/lib/insika/commands/send_message.rb +112 -15
  76. data/lib/insika/commands/session_purge.rb +67 -0
  77. data/lib/insika/commands/set_agent_tools.rb +1 -1
  78. data/lib/insika/commands/set_skill_agents.rb +60 -19
  79. data/lib/insika/commands/trigger_workflow.rb +1 -1
  80. data/lib/insika/commands/update_agent.rb +1 -1
  81. data/lib/insika/commands/write_data_tool.rb +1 -1
  82. data/lib/insika/commands/write_golden.rb +1 -1
  83. data/lib/insika/commands/write_skill.rb +19 -9
  84. data/lib/insika/config_store.rb +8 -4
  85. data/lib/insika/contact_store.rb +183 -0
  86. data/lib/insika/context/builder.rb +23 -5
  87. data/lib/insika/context/fragment.rb +31 -3
  88. data/lib/insika/context/priority.rb +6 -2
  89. data/lib/insika/context/provider.rb +17 -3
  90. data/lib/insika/context/providers/briefing.rb +96 -0
  91. data/lib/insika/context/providers/memory.rb +16 -7
  92. data/lib/insika/context/providers/prompt.rb +30 -2
  93. data/lib/insika/context/providers/request.rb +1 -1
  94. data/lib/insika/context/providers/session.rb +17 -2
  95. data/lib/insika/context/providers/skill.rb +7 -1
  96. data/lib/insika/context/providers/skill_trigger.rb +128 -0
  97. data/lib/insika/context/providers/tool_search.rb +2 -0
  98. data/lib/insika/context_trace_store.rb +128 -0
  99. data/lib/insika/delegation_store.rb +2 -2
  100. data/lib/insika/distill.rb +224 -0
  101. data/lib/insika/distill_engine.rb +169 -0
  102. data/lib/insika/doctor.rb +962 -7
  103. data/lib/insika/dsl/runtime.rb +20 -11
  104. data/lib/insika/dsl/server_boot.rb +74 -4
  105. data/lib/insika/dsl/system.rb +1 -1
  106. data/lib/insika/dsl.rb +152 -15
  107. data/lib/insika/edge_limiter.rb +167 -8
  108. data/lib/insika/egress_guard.rb +3 -3
  109. data/lib/insika/env_schema.rb +22 -12
  110. data/lib/insika/errors.rb +72 -5
  111. data/lib/insika/evals/assertions.rb +15 -14
  112. data/lib/insika/evals/baseline.rb +3 -3
  113. data/lib/insika/evals/golden.rb +8 -8
  114. data/lib/insika/evals/judge.rb +7 -7
  115. data/lib/insika/evals/pairwise.rb +21 -9
  116. data/lib/insika/evals/report.rb +2 -2
  117. data/lib/insika/evals/runner.rb +6 -6
  118. data/lib/insika/evals/transport.rb +2 -2
  119. data/lib/insika/event_stream.rb +23 -5
  120. data/lib/insika/evidence.rb +183 -0
  121. data/lib/insika/executor.rb +1092 -160
  122. data/lib/insika/followup_engine.rb +207 -0
  123. data/lib/insika/followup_policy.rb +221 -0
  124. data/lib/insika/followup_store.rb +306 -0
  125. data/lib/insika/frontmatter.rb +1 -1
  126. data/lib/insika/funnel_declaration.rb +106 -0
  127. data/lib/insika/funnel_fold.rb +179 -0
  128. data/lib/insika/funnel_store.rb +163 -0
  129. data/lib/insika/golden_store.rb +3 -3
  130. data/lib/insika/grounding/matcher.rb +69 -0
  131. data/lib/insika/grounding.rb +44 -0
  132. data/lib/insika/harvest/conversion_gate.rb +159 -0
  133. data/lib/insika/harvest/criterion.rb +98 -0
  134. data/lib/insika/harvest/gate.rb +194 -0
  135. data/lib/insika/harvest/negative_list.rb +199 -0
  136. data/lib/insika/harvest.rb +241 -0
  137. data/lib/insika/harvest_engine.rb +193 -0
  138. data/lib/insika/harvest_store.rb +548 -0
  139. data/lib/insika/http_client.rb +3 -3
  140. data/lib/insika/inbound_log.rb +1 -1
  141. data/lib/insika/llm_configurator.rb +3 -3
  142. data/lib/insika/loop_detector.rb +143 -0
  143. data/lib/insika/mcp_http_client.rb +4 -4
  144. data/lib/insika/mcp_tool_ingestor.rb +6 -6
  145. data/lib/insika/media.rb +298 -0
  146. data/lib/insika/memory_audit_store.rb +85 -0
  147. data/lib/insika/memory_store.rb +264 -23
  148. data/lib/insika/message_origin.rb +8 -3
  149. data/lib/insika/model_resolver.rb +1 -1
  150. data/lib/insika/model_selection.rb +5 -4
  151. data/lib/insika/model_visible.rb +87 -0
  152. data/lib/insika/model_visible_trace_store.rb +66 -0
  153. data/lib/insika/onboarding.rb +8 -3
  154. data/lib/insika/outbox_store.rb +44 -6
  155. data/lib/insika/outcome_store.rb +147 -0
  156. data/lib/insika/overlay_tool_registry.rb +3 -4
  157. data/lib/insika/pack.rb +3 -3
  158. data/lib/insika/pack_importer.rb +17 -15
  159. data/lib/insika/packaging.rb +163 -0
  160. data/lib/insika/parity/criterion.rb +79 -0
  161. data/lib/insika/parity/verdict.rb +318 -0
  162. data/lib/insika/pending_action_store.rb +1 -1
  163. data/lib/insika/plugin/loader.rb +2 -2
  164. data/lib/insika/policy/policy.rb +1 -1
  165. data/lib/insika/prefix_fingerprint.rb +58 -0
  166. data/lib/insika/profile_source.rb +34 -7
  167. data/lib/insika/proposal_store.rb +271 -0
  168. data/lib/insika/provider_error_classifier.rb +160 -0
  169. data/lib/insika/queue_policy.rb +6 -3
  170. data/lib/insika/recovery.rb +47 -6
  171. data/lib/insika/refinement/candidate.rb +4 -4
  172. data/lib/insika/refinement/evidence_collector.rb +6 -6
  173. data/lib/insika/refinement/gate.rb +7 -7
  174. data/lib/insika/refinement/panel.rb +7 -7
  175. data/lib/insika/refinement/proposer.rb +10 -10
  176. data/lib/insika/refinement_store.rb +12 -12
  177. data/lib/insika/reliability.rb +211 -0
  178. data/lib/insika/retention.rb +281 -0
  179. data/lib/insika/routing.rb +101 -0
  180. data/lib/insika/safety/config.rb +46 -6
  181. data/lib/insika/safety/corpus.rb +255 -0
  182. data/lib/insika/safety/detectors.rb +34 -115
  183. data/lib/insika/safety/factory.rb +18 -5
  184. data/lib/insika/safety/grounding_enforcer.rb +59 -0
  185. data/lib/insika/safety/grounding_validator.rb +49 -0
  186. data/lib/insika/safety/input_guardrail.rb +20 -5
  187. data/lib/insika/safety/moderator.rb +19 -11
  188. data/lib/insika/safety/output_filter.rb +10 -6
  189. data/lib/insika/safety/output_validator.rb +13 -7
  190. data/lib/insika/safety/safe_responses.rb +1 -1
  191. data/lib/insika/sandbox/boundary.rb +2 -2
  192. data/lib/insika/sandbox.rb +1 -1
  193. data/lib/insika/schema_guard.rb +35 -0
  194. data/lib/insika/server/app.rb +366 -54
  195. data/lib/insika/server/boot.rb +4 -4
  196. data/lib/insika/server/rack_app.rb +31 -7
  197. data/lib/insika/server/responses.rb +58 -9
  198. data/lib/insika/server/tenant_auth.rb +61 -0
  199. data/lib/insika/session_actor.rb +11 -7
  200. data/lib/insika/session_store.rb +66 -3
  201. data/lib/insika/settings_store.rb +15 -5
  202. data/lib/insika/shadow_pair_store.rb +258 -0
  203. data/lib/insika/shutdown.rb +4 -4
  204. data/lib/insika/skill_catalog.rb +131 -20
  205. data/lib/insika/skill_store.rb +70 -22
  206. data/lib/insika/soak/envelope.rb +140 -0
  207. data/lib/insika/soak/report.rb +392 -0
  208. data/lib/insika/soak/runner.rb +554 -0
  209. data/lib/insika/steer_injector.rb +1 -1
  210. data/lib/insika/store.rb +11 -2
  211. data/lib/insika/stores/memory.rb +6 -0
  212. data/lib/insika/stores/sqlite.rb +8 -0
  213. data/lib/insika/studio/app.rb +1058 -75
  214. data/lib/insika/studio/assets/dist/application.css +1 -1
  215. data/lib/insika/studio/assets/dist/application.js +27 -26
  216. data/lib/insika/studio/assets/dist/favicon.svg +6 -0
  217. data/lib/insika/studio/forms.rb +274 -22
  218. data/lib/insika/studio/nav_icons.rb +7 -2
  219. data/lib/insika/studio/views/_message.erb +2 -2
  220. data/lib/insika/studio/views/agent_detail.erb +629 -86
  221. data/lib/insika/studio/views/agents.erb +11 -7
  222. data/lib/insika/studio/views/approvals.erb +4 -1
  223. data/lib/insika/studio/views/chats.erb +4 -1
  224. data/lib/insika/studio/views/customer.erb +94 -0
  225. data/lib/insika/studio/views/customers.erb +32 -0
  226. data/lib/insika/studio/views/evals.erb +4 -1
  227. data/lib/insika/studio/views/facts.erb +133 -0
  228. data/lib/insika/studio/views/followups.erb +125 -0
  229. data/lib/insika/studio/views/funnel.erb +106 -0
  230. data/lib/insika/studio/views/harvest.erb +234 -0
  231. data/lib/insika/studio/views/home.erb +2 -1
  232. data/lib/insika/studio/views/layout.erb +1 -0
  233. data/lib/insika/studio/views/parity.erb +147 -0
  234. data/lib/insika/studio/views/playground.erb +7 -1
  235. data/lib/insika/studio/views/refinement.erb +4 -4
  236. data/lib/insika/studio/views/session.erb +133 -3
  237. data/lib/insika/studio/views/settings.erb +9 -12
  238. data/lib/insika/studio/views/skills.erb +66 -12
  239. data/lib/insika/studio/views/system_files.erb +1 -1
  240. data/lib/insika/studio/views/task.erb +13 -0
  241. data/lib/insika/studio/views/tasks.erb +4 -1
  242. data/lib/insika/studio/views/tools.erb +0 -1
  243. data/lib/insika/subagent_graph.rb +3 -3
  244. data/lib/insika/task_actor.rb +3 -3
  245. data/lib/insika/task_store.rb +22 -2
  246. data/lib/insika/telemetry/pricing.rb +3 -3
  247. data/lib/insika/telemetry/recorder.rb +1 -1
  248. data/lib/insika/telemetry.rb +2 -2
  249. data/lib/insika/testing/store_contract.rb +54 -33
  250. data/lib/insika/tick.rb +146 -0
  251. data/lib/insika/token_store.rb +168 -0
  252. data/lib/insika/tool_assembly.rb +5 -5
  253. data/lib/insika/tool_definition.rb +25 -15
  254. data/lib/insika/tool_envelope.rb +70 -1
  255. data/lib/insika/tool_manifest.rb +11 -7
  256. data/lib/insika/tool_output_compressor.rb +100 -0
  257. data/lib/insika/tool_store.rb +1 -1
  258. data/lib/insika/tool_trace_store.rb +1 -1
  259. data/lib/insika/tools/concurrency.rb +2 -2
  260. data/lib/insika/tools/data_defined_tool.rb +14 -5
  261. data/lib/insika/tools/generate_image.rb +44 -0
  262. data/lib/insika/tools/load_skill.rb +61 -3
  263. data/lib/insika/tools/schedule_followup.rb +164 -0
  264. data/lib/insika/tools/stuck_signal.rb +44 -0
  265. data/lib/insika/tools/subagent.rb +4 -4
  266. data/lib/insika/tools/subagents.rb +1 -1
  267. data/lib/insika/tools/tts.rb +47 -0
  268. data/lib/insika/tools/update_briefing.rb +126 -0
  269. data/lib/insika/turn_output.rb +2 -2
  270. data/lib/insika/turn_state.rb +54 -13
  271. data/lib/insika/turn_timing.rb +24 -4
  272. data/lib/insika/usage_ledger.rb +1 -1
  273. data/lib/insika/version.rb +1 -1
  274. data/lib/insika/vitals.rb +84 -0
  275. data/lib/insika/wiring/graph.rb +372 -34
  276. data/lib/insika/workflow.rb +1 -1
  277. data/lib/insika/workflow_registry.rb +1 -1
  278. data/lib/insika.rb +122 -16
  279. metadata +95 -2
  280. data/lib/insika/server/admin_auth.rb +0 -29
@@ -0,0 +1,393 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "time"
4
+
5
+ module Insika
6
+ module Commands
7
+ # the ONLY path that writes candidates. Mines ONE window end
8
+ # to end: resolve the sessions, read the transcripts + evidence, ask the
9
+ # miner, schema-drop, apply the negative list and the grounding filter,
10
+ # dedup against the ledger, write the run + candidates, stamp the markers.
11
+ #
12
+ # Synchronous (it runs on the engine's worker fiber, C12, or the CLI, C15)
13
+ # — it creates no task and no turn. It writes NOTHING to sessions or
14
+ # skills: D2's fork discipline is this command's contract — a customer
15
+ # turn's prefix is untouched by construction, and the only harvest-side
16
+ # spend is the run's cost (E1).
17
+ #
18
+ # Payload: { agent:, last_sessions?, since?, full?, session_ids?,
19
+ # max_proposals?, exclude_sessions? }
20
+ class RunHarvest
21
+ DEFAULT_LAST_SESSIONS = 200 # the EvidenceCollector's window default
22
+ DEFAULT_MIN_MESSAGES = 3
23
+ DEFAULT_MAX_PROPOSALS = 10
24
+ DEFAULT_IDLE_HOURS = 24
25
+
26
+ def initialize(profiles:, harvest_store:, session_store:, task_store:,
27
+ skill_store: nil, tool_trace_store: nil, settings_store: nil,
28
+ negative_list: nil, miner_factory: nil, event_stream:)
29
+ @profiles = ProfileSource.coerce(profiles)
30
+ @harvest_store = harvest_store
31
+ @session_store = session_store
32
+ @task_store = task_store
33
+ @skill_store = skill_store
34
+ @tool_trace_store = tool_trace_store
35
+ @settings_store = settings_store
36
+ @negative_list = negative_list
37
+ @miner_factory = miner_factory ||
38
+ ->(config) { Harvest::MinerFactory.build(config, utility_model: utility_model) }
39
+ @event_stream = event_stream
40
+ end
41
+
42
+ # -> { mined: true, run_id:, candidates: N,
43
+ # rejected: { "<rule-id>" => N, "ungrounded" => N, "dedup" => N,
44
+ # "schema" => N, ... }, cost: {...} | nil }
45
+ # | { mined: false, skipped: "disabled|no_model|no_grounding_matcher" }
46
+ def call(command)
47
+ p = AgentPayload.symbolize(command.payload)
48
+ agent = AgentPayload.presence(p[:agent])
49
+ raise Insika::ValidationError, "agent is required" if agent.nil?
50
+
51
+ profile = @profiles[agent] ||
52
+ (raise Insika::NotFoundError, "agent '#{agent}' not configured")
53
+ config = Coercion.deep_stringify(profile.harvest)
54
+ return skip("disabled") if config.nil? || !Coercion.truthy?(config["enabled"])
55
+
56
+ # D4: the OPERATIVE negative list lives on the profile (hot-editable,
57
+ # seeded by `insika harvest:negative import`); the injected list is
58
+ # the deployment's fallback. The engine applies data, never authors it.
59
+ @list = negative_list_for(config)
60
+
61
+ # D3: product claims cannot be verified without a matcher, so NOTHING
62
+ # mines — refused, not warned (D12's "by refusal, not by prompt").
63
+ grounding = Grounding.parse(profile.grounding)
64
+ return skip("no_grounding_matcher") if grounding.nil? || !grounding.matcher.sku?
65
+
66
+ miner = @miner_factory.call(config)
67
+ return skip("no_model") if miner.nil?
68
+
69
+ run = @harvest_store.create_run(agent_id: agent, window: window_record(p),
70
+ budget: budget_cap(config))
71
+ sessions = resolve_sessions(agent, p, config)
72
+
73
+ begin
74
+ # No eligible sessions: a run that says "we looked and it was clean"
75
+ # without paying a bill; markers untouched (re-scan, D10).
76
+ if sessions.empty?
77
+ @harvest_store.complete_run(run.id, candidates: 0)
78
+ return { mined: true, run_id: run.id, candidates: 0,
79
+ rejected: empty_rejected, cost: nil }
80
+ end
81
+
82
+ prompt = build_prompt(config, sessions, agent)
83
+ result = miner.mine(prompt: prompt,
84
+ message_counts: sessions.map { |s| s[:messages].size },
85
+ max_proposals: max_proposals(p, config))
86
+
87
+ # The mining budget is a REAL cap (the review fix): a pass that
88
+ # spent more than the pack declared is failed with the numbers and
89
+ # proposes NOTHING — the docs' "the budget cap bounds it" is
90
+ # enforced here, post-hoc for the one model call, pre-hoc for every
91
+ # downstream write and gate.
92
+ if budget_exceeded?(budget_cap(config), result[:cost])
93
+ @harvest_store.fail_run(run.id, error: "mining budget exceeded: " \
94
+ "spent #{result[:cost]['spent']} > " \
95
+ "#{budget_cap(config)['tokens']}")
96
+ emit(:harvest_mined, agent: agent, run_id: run.id, candidates: 0,
97
+ rejected: empty_rejected, cost: result[:cost])
98
+ return { mined: true, run_id: run.id, candidates: 0,
99
+ rejected: empty_rejected, cost: result[:cost] }
100
+ end
101
+
102
+ survivors, rejected = filter_skills(result[:skills], sessions, agent, grounding.matcher)
103
+
104
+ survivors.each do |skill|
105
+ @harvest_store.create_candidate(
106
+ run_id: run.id, agent: agent, name: skill["name"],
107
+ description: skill["description"], body: skill_md(skill),
108
+ triggers: skill["triggers"] || [], rationale: skill["rationale"].to_s,
109
+ origin: sessions.map { |s| s[:id] }, evidence_turns: skill["evidence_turns"] || [],
110
+ proposer: miner.model
111
+ )
112
+ end
113
+
114
+ final_rejected = merge_rejected(result[:dropped], rejected)
115
+ @harvest_store.complete_run(run.id, candidates: survivors.size,
116
+ cost: result[:cost],
117
+ rejected: final_rejected)
118
+ # Markers AFTER the pass completes (D10's crash-safe re-scan).
119
+ sessions.each { |s| @harvest_store.mark_mined(s[:id], candidates: survivors.size) }
120
+
121
+ emit(:harvest_mined, agent: agent, run_id: run.id, candidates: survivors.size,
122
+ rejected: final_rejected, cost: result[:cost])
123
+ { mined: true, run_id: run.id, candidates: survivors.size,
124
+ rejected: final_rejected, cost: result[:cost] }
125
+ rescue StandardError => e
126
+ # The run is failed; the markers are NOT written (re-scan, D10);
127
+ # the exception propagates to the caller's fiber or the CLI.
128
+ begin
129
+ @harvest_store.fail_run(run.id, error: e.message)
130
+ rescue StandardError
131
+ nil
132
+ end
133
+ raise
134
+ end
135
+ end
136
+
137
+ private
138
+
139
+ def skip(reason)
140
+ { mined: false, skipped: reason }
141
+ end
142
+
143
+ # The pack's harvest.miner.budget as { "tokens" => N } — nil when
144
+ # absent/not positive (no cap, the refinement discipline).
145
+ def budget_cap(config)
146
+ raw = config && config.dig("miner", "budget")
147
+ raw.is_a?(Hash) && raw["tokens"].to_i.positive? ? { "tokens" => raw["tokens"].to_i } : nil
148
+ end
149
+
150
+ def budget_exceeded?(cap, cost)
151
+ cap && cost && cost["spent"].to_i > cap["tokens"].to_i
152
+ end
153
+
154
+ # The candidate's body is a full SKILL.md by construction: the model
155
+ # writes the procedure, the engine wraps the frontmatter (name /
156
+ # description / triggers — the miner's own fields), so a candidate is
157
+ # servable the moment a human promotes it (WriteSkill validates the
158
+ # frontmatter, and the gate's clone serves it the same way).
159
+ def skill_md(skill)
160
+ fm = { "name" => skill["name"], "description" => skill["description"] }
161
+ fm["triggers"] = Array(skill["triggers"]).join(", ") if Array(skill["triggers"]).any?
162
+ "---\n#{fm.map { |k, v| "#{k}: #{v}" }.join("\n")}\n---\n#{skill['body']}"
163
+ end
164
+
165
+ # The profile's own rules win over the injected seed (D4). A malformed
166
+ # profile list parses to nil — the injected list stays the fallback,
167
+ # never a silent empty.
168
+ def negative_list_for(config)
169
+ return @negative_list unless config && config["negative_list"]
170
+
171
+ Harvest::NegativeList.parse(config["negative_list"]) || @negative_list
172
+ end
173
+
174
+ def empty_rejected = { "schema" => 0, "unknown_key" => 0, "oversized" => 0,
175
+ "bad_turns" => 0, "duplicate" => 0, "capped" => 0 }
176
+
177
+ def window_record(p)
178
+ return { "session_ids" => Array(p[:session_ids]).map(&:to_s) } if p[:session_ids]
179
+ return { "since" => p[:since].to_s } if AgentPayload.presence(p[:since])
180
+ return { "last_sessions" => Integer(p[:last_sessions]) } if p[:last_sessions]
181
+
182
+ {}
183
+ end
184
+
185
+ # The EvidenceCollector window discipline, applied to session ids: the
186
+ # agent's turns (by the task's command payload), distinct session ids,
187
+ # newest first, capped at the "last N conversations". Marker suppression
188
+ # unless `full` (a re-mine is explicit). A session's evidence ledger
189
+ # is the grounding filter's input.
190
+ def resolve_sessions(agent, p, config)
191
+ ids = window_session_ids(agent, p, config)
192
+ ids = ids.reject { |sid| @harvest_store.mined?(sid) } unless EnvSchema.truthy?(p[:full])
193
+ ids = ids.first(Harvest::Miner::MAX_SESSIONS)
194
+
195
+ min_messages = (config["min_messages"] || DEFAULT_MIN_MESSAGES).to_i
196
+ idle_hours = (config["idle_hours"] || DEFAULT_IDLE_HOURS).to_i
197
+ ids.filter_map do |sid|
198
+ session = @session_store.find(sid)
199
+ next unless session
200
+
201
+ messages = session.messages.to_a
202
+ next if messages.size < min_messages # a 2-message session mines noise
203
+ # D10: the per-agent maturity bound is re-checked here — the engine
204
+ # scan's default is only the LOWER bound (a pack that wants 12 h is
205
+ # never mined at 6, and a manual CLI run over fresh traffic mines
206
+ # nothing).
207
+ next unless idle?(session.updated_at, idle_hours)
208
+
209
+ evidence = (session.evidence || {})["ids"] || []
210
+ { id: sid.to_s, messages: messages, evidence: Array(evidence).map(&:to_s) }
211
+ end
212
+ end
213
+
214
+ def window_session_ids(agent, p, config)
215
+ explicit = Array(p[:session_ids]).map(&:to_s)
216
+ return explicit unless explicit.empty?
217
+
218
+ since = AgentPayload.presence(p[:since])
219
+ tasks = @task_store.each_id
220
+ .filter_map { |id| @task_store.find(id) }
221
+ .select { |t| task_agent(t) == agent }
222
+ .sort_by { |t| [t.created_at.to_s, t.id] }.reverse
223
+
224
+ tasks = tasks.select { |t| t.created_at.to_s >= since.to_s } if since
225
+
226
+ distinct = []
227
+ seen = {}
228
+ tasks.each do |t|
229
+ sid = presence(t.session_id)
230
+ next if sid.nil? || seen[sid]
231
+
232
+ seen[sid] = true
233
+ distinct << sid
234
+ end
235
+ return distinct if since
236
+
237
+ last = AgentPayload.presence(p[:last_sessions])
238
+ count = last ? Integer(last) : config_window(config)
239
+ distinct.first(count)
240
+ end
241
+
242
+ # The config's miner.window last_sessions, or the collector default.
243
+ def config_window(config)
244
+ configured = config.dig("miner", "window", "last_sessions")
245
+ configured ? Integer(configured) : DEFAULT_LAST_SESSIONS
246
+ end
247
+
248
+ def task_agent(task)
249
+ task.command.is_a?(Hash) ? task.command.dig("payload", "agent").to_s : ""
250
+ end
251
+
252
+ def idle?(updated_at, idle_hours)
253
+ return false if Coercion.blank?(updated_at)
254
+
255
+ Time.iso8601(updated_at.to_s) <= Time.now.utc - idle_hours * 3600
256
+ rescue ArgumentError
257
+ false
258
+ end
259
+
260
+ def presence(value)
261
+ Insika::Coercion.presence(value)
262
+ end
263
+
264
+ def max_proposals(p, config)
265
+ raw = p[:max_proposals] || config.dig("miner", "max_proposals")
266
+ raw ? Integer(raw) : DEFAULT_MAX_PROPOSALS
267
+ end
268
+
269
+ # The prompt: the transcript slices (masked through the output
270
+ # filter — the redaction rule), the evidence ids per
271
+ # session, the agent's CURRENT skill names (the model should not
272
+ # re-propose them), and the answer rules. The pack prompt replaces
273
+ # DEFAULT_PROMPT whole (the forge's half).
274
+ def build_prompt(config, sessions, agent)
275
+ base = Coercion.presence(config["prompt"]) || Harvest::DEFAULT_PROMPT
276
+ current_skills = current_skill_names(agent)
277
+ blocks = sessions.map do |s|
278
+ evidence = s[:evidence].join(", ")
279
+ lines = ["## Session #{s[:id]} (#{s[:messages].size} messages; " \
280
+ "evidence ids: #{evidence.empty? ? '(none)' : evidence})"]
281
+ lines << render_transcript(s[:messages])
282
+ lines.join("\n")
283
+ end
284
+ <<~PROMPT
285
+ #{base.rstrip}
286
+
287
+ ## Conversations to mine
288
+
289
+ #{blocks.join("\n")}
290
+
291
+ ## Skills this agent already has (do not re-propose them)
292
+
293
+ #{current_skills.empty? ? "(none)" : current_skills.join(", ")}
294
+ PROMPT
295
+ end
296
+
297
+ def current_skill_names(agent)
298
+ return [] unless @skill_store
299
+
300
+ (@skill_store.names | @skill_store.names(agent: agent)).sort
301
+ end
302
+
303
+ def render_transcript(messages)
304
+ redacted, = Insika::Safety::Detectors.redact(
305
+ messages.each_with_index.map { |m, i| "[#{i}] #{m['role']}: #{m['content']}" }.join("\n")
306
+ )
307
+ redacted
308
+ end
309
+
310
+ # The safety order: negative list (D4) -> grounding (D3) -> dedup. A
311
+ # drop is COUNTED and NEVER written. `rejected` keys: the matched RULE
312
+ # ids for negative hits (each named — E2), "ungrounded"/"dedup" for the
313
+ # other two filters.
314
+ # -> [[raw skill], { key => count }]
315
+ def filter_skills(skills, sessions, agent, matcher)
316
+ rejected = Hash.new(0)
317
+ survivors = skills.select do |skill|
318
+ hits = negative_hits(skill)
319
+ if hits.any?
320
+ hits.each { |r| rejected[r.rule] += 1 }
321
+ false
322
+ elsif !grounded?(skill, sessions, matcher)
323
+ rejected["ungrounded"] += 1
324
+ false
325
+ elsif deduped?(skill, agent)
326
+ rejected["dedup"] += 1
327
+ false
328
+ else
329
+ true
330
+ end
331
+ end
332
+ [survivors, rejected.to_h]
333
+ end
334
+
335
+ # D4: every rejected-by-list candidate is logged with the matching rule
336
+ # id. The BODY is part of the match (the review fix): it is exactly what
337
+ # enters the model's context when the skill loads, so a banned phrase
338
+ # hidden there is the Hermes failure the design cites — the same text
339
+ # the grounding filter already reads (name+description+body). The name
340
+ # uses the stricter substring reading, the prose the word-boundary one.
341
+ def negative_hits(skill)
342
+ return [] unless @list
343
+
344
+ text = [skill["name"], skill["description"], skill["body"]].join(" ")
345
+ hits = @list.matches_name(skill["name"].to_s)
346
+ hits | @list.matches(text)
347
+ end
348
+
349
+ # D3: every reference must be in the union of the origin sessions'
350
+ # persisted evidence ids. The empty-ledger conservative reading (the
351
+ # enforcer's rule): a candidate whose origin sessions hold NO evidence
352
+ # ids and that carries a reference is dropped too. A skill with no
353
+ # references is not a grounding casualty. -> bool
354
+ def grounded?(skill, sessions, matcher)
355
+ text = [skill["name"], skill["description"], skill["body"]].join(" ")
356
+ refs = matcher.references(text)
357
+ return true if refs.empty?
358
+
359
+ union = sessions.flat_map { |s| s[:evidence] }.uniq
360
+ return false if union.empty?
361
+
362
+ missing = refs.reject { |r| union.include?(r) }
363
+ missing.empty?
364
+ end
365
+
366
+ # The dedup ledger: the same name already in the store's SkillStore
367
+ # (shared or agent scope) or an open (agent, name) tuple.
368
+ def deduped?(skill, agent)
369
+ name = skill["name"].to_s
370
+ store_has = @skill_store && (@skill_store.get(name) || @skill_store.get(name, agent: agent))
371
+ store_has || @harvest_store.open_pending?(agent: agent, name: name)
372
+ end
373
+
374
+ # The miner's schema drops + the filters' — one record; the keys never
375
+ # collide (the filters own rule ids and their own names).
376
+ def merge_rejected(dropped, filters)
377
+ dropped.merge(filters) { |_k, left, right| left + right }
378
+ end
379
+
380
+ def utility_model
381
+ return nil unless @settings_store
382
+
383
+ @settings_store.get["utility_model"]
384
+ end
385
+
386
+ def emit(type, **data)
387
+ @event_stream.emit(Insika::Event.new(
388
+ type: type, data: data, meta: { at: Time.now.utc.iso8601 }
389
+ ))
390
+ end
391
+ end
392
+ end
393
+ end
@@ -4,11 +4,11 @@ require "time"
4
4
 
5
5
  module Insika
6
6
  module Commands
7
- # Control command (RFC-0013 phase A): reads a window of the agent's real traffic
7
+ # Control command: reads a window of the agent's real traffic
8
8
  # and records a ranked failure REPORT. Synchronous — it only scans durable stores
9
9
  # (no provider call, no fiber), so it answers with the Run and does not create a
10
10
  # Task. It is the ONLY way a refinement run starts: the CLI, the Studio button
11
- # and any external cron all dispatch this one command (§3.10 — there is no
11
+ # and any external cron all dispatch this one command (there is no
12
12
  # scheduler in the engine).
13
13
  #
14
14
  # Payload:
@@ -23,8 +23,8 @@ module Insika
23
23
  # the previous run for this agent, unless `full`) -> the agent's configured
24
24
  # `refinement.window` -> the collector's default.
25
25
  #
26
- # Phase A writes NOTHING to the agent, so it needs no opt-in: an absent
27
- # `refinement` config reads as report-only. Only `propose`/`auto_apply` (phase C)
26
+ # writes NOTHING to the agent, so it needs no opt-in: an absent
27
+ # `refinement` config reads as report-only. Only `propose`/`auto_apply`
28
28
  # require the operator to enable them explicitly.
29
29
  class RunRefinement
30
30
  READ_ONLY_MODES = %w[report propose auto_apply].freeze
@@ -77,7 +77,7 @@ module Insika
77
77
 
78
78
  # An unknown mode is a config typo, and a typo that silently degrades to
79
79
  # "report" would be the kind of quiet wrong the strict-config rule exists to
80
- # prevent (item 23).
80
+ # prevent.
81
81
  def validate_mode!(config)
82
82
  mode = AgentPayload.presence(config["mode"]) || "report"
83
83
  return if READ_ONLY_MODES.include?(mode)
@@ -7,25 +7,35 @@ module Insika
7
7
  # Stream. Validations that fail do NOT create a Task
8
8
  # (ValidationError/NotFoundError -> direct HTTP response).
9
9
  class SendMessage
10
- # RFC-0015 §5.5 — surfaces whose response can carry the "you do not own the reply"
10
+ # surfaces whose response can carry the "you do not own the reply"
11
11
  # verdict (`merged` for `collect`, `steered` for `steer`), and therefore the only
12
12
  # ones where a message may join another turn. `/v1/responses` is NOT here: its body
13
13
  # is OpenAI-shaped SSE with nowhere to put the field, and it is frozen because a
14
14
  # live consumer speaks it. Joining a caller that cannot hear the verdict makes it
15
15
  # deliver the same answer once per message, which is worse than not joining at all.
16
- # Channels declare themselves by `channel:<id>` once RFC-0011 §6 lands.
16
+ # Channels declare themselves by `channel:<id>` once lands.
17
17
  COALESCABLE_TRANSPORTS = %i[http:json].freeze
18
18
 
19
- # RFC-0011 §6.4 — `inbound_log` is the retry window for channel event ids.
19
+ # `inbound_log` is the retry window for channel event ids.
20
20
  # nil = no dedup (every surface that does not send an `event_id`, which is all
21
21
  # of them today), and a caller that cannot supply a stable id gets
22
22
  # at-least-once turns rather than a content hash pretending to be dedup.
23
- def initialize(profiles:, session_store:, task_store:, executor:, inbound_log: nil)
23
+ #
24
+ # `contact_store`/`followup_store` : the contact-state hook —
25
+ # a customer message reopens the contact cell and a cancellation keyword
26
+ # revokes it + falls the pending records. nil = the hook is off (parity).
27
+ # `store` is the SHARED backend the two stores ride — the keyword revoke
28
+ # commits in ONE transaction (D2). nil = best-effort separate writes.
29
+ def initialize(profiles:, session_store:, task_store:, executor:, inbound_log: nil,
30
+ contact_store: nil, followup_store: nil, store: nil)
24
31
  @profiles = ProfileSource.coerce(profiles)
25
32
  @session_store = session_store
26
33
  @task_store = task_store
27
34
  @executor = executor
28
35
  @inbound_log = inbound_log
36
+ @contact_store = contact_store
37
+ @followup_store = followup_store
38
+ @store = store
29
39
  end
30
40
 
31
41
  def call(command)
@@ -37,12 +47,19 @@ module Insika
37
47
  profile = @profiles[agent] ||
38
48
  (raise Insika::NotFoundError, "agent '#{agent}' not configured")
39
49
 
50
+ # A turn is text OR media. The media half (WS9) is what a voice note
51
+ # with no caption looks like on the wire — `{ parts: [{type: "audio",
52
+ # url: …}] }` and nothing else — and demanding a message here made the
53
+ # anchor use case unreachable end to end: the audio becomes the message
54
+ # at the :media stage, one step later.
40
55
  message = p[:message]
41
- raise Insika::ValidationError, "message is required and non-empty" if message.to_s.strip.empty?
56
+ if message.to_s.strip.empty? && !media?(p[:parts])
57
+ raise Insika::ValidationError, "message is required and non-empty (or a media part)"
58
+ end
42
59
 
43
60
  # session_id XOR history (both -> error; neither -> one-shot).
44
61
  if p[:session_id] && p[:history]
45
- raise Insika::ValidationError, "session_id and history are mutually exclusive (D2)"
62
+ raise Insika::ValidationError, "session_id and history are mutually exclusive"
46
63
  end
47
64
 
48
65
  validate_history!(p[:history]) if p[:history]
@@ -50,13 +67,20 @@ module Insika
50
67
  # is what a turn has always meant. Refused here rather than downstream: a
51
68
  # typo'd origin would read as absent, and a marker that silently means
52
69
  # "unmarked" is worse than none — it looks like the filtering is on.
53
- Insika::MessageOrigin.parse!(p[:origin])
70
+ origin = Insika::MessageOrigin.parse!(p[:origin])
71
+ # `scheduled` is ENGINE-RESERVED — the FollowupEngine's
72
+ # synthetic turn stamps it, and the edge must not let a consumer
73
+ # impersonate the engine's kick (a spoofed follow-up is the spam bug).
74
+ if origin == Insika::MessageOrigin::SCHEDULED
75
+ raise Insika::ValidationError,
76
+ "origin 'scheduled' is engine-reserved: it is stamped by the follow-up engine only"
77
+ end
54
78
  if p[:session_id]
55
79
  @session_store.find(p[:session_id]) ||
56
80
  (raise Insika::NotFoundError, "session '#{p[:session_id]}' not found")
57
81
  end
58
82
 
59
- # RFC-0011 §6.4 — the platform retried a webhook it already delivered. Answer
83
+ # the platform retried a webhook it already delivered. Answer
60
84
  # with the turn it ALREADY produced and run nothing: without this, one flaky
61
85
  # ack costs a second LLM turn and sends the customer the same answer twice.
62
86
  # Checked before the queue doors on purpose — a duplicate is not a fragment to
@@ -66,6 +90,14 @@ module Insika
66
90
  return { task_id: prior, duplicate: true }
67
91
  end
68
92
 
93
+ # the contact-state hook — the ONLY path that sees every
94
+ # customer message. A real customer message reopens the contact cell; a
95
+ # cancellation keyword revokes it and falls the pending records in ONE
96
+ # transaction. Runs AFTER validation and dedup: a refused or duplicated
97
+ # message must not touch contact state. Nil-safe and policy-gated: no
98
+ # profile declaration = the hook is off (parity).
99
+ touch_contact(p, profile, command, origin)
100
+
69
101
  result = start_turn(command, p, profile)
70
102
  @inbound_log.record(key, result[:task_id]) if key
71
103
  result
@@ -73,22 +105,60 @@ module Insika
73
105
 
74
106
  private
75
107
 
108
+ # the contact bookkeeping of a customer message. The
109
+ # keyword cast IS a revocation: the customer just said the shut-off
110
+ # words — the pending records must fall with the state (D2, ONE
111
+ # transaction). The reset-on-origin list: only an origin that is
112
+ # nil/customer acts — an engine/operator-composed line (even one QUOTING
113
+ # the customer's words) never touches contact state. Nil collaborators
114
+ # or no profile declaration = the hook is a no-op (byte-parity).
115
+ def touch_contact(p, profile, command, origin)
116
+ return unless @contact_store
117
+ return unless origin.nil? || origin == Insika::MessageOrigin::CUSTOMER
118
+
119
+ policy = Insika::FollowupPolicy.parse(profile.followup)
120
+ return unless policy
121
+
122
+ customer = p[:customer]
123
+ return if customer.to_s.empty?
124
+
125
+ keyword = policy.match_keyword(p[:message].to_s)
126
+ if keyword
127
+ # one transaction on the shared backend: the revoke and the pending
128
+ # fall commit together — a half-cancelled opt-out is the spam bug.
129
+ if @store
130
+ @store.transaction do
131
+ @contact_store.set_revoked(tenant: command.meta[:tenant], customer: customer)
132
+ @followup_store&.cancel_pending_for(tenant: command.meta[:tenant], customer: customer)
133
+ end
134
+ else
135
+ @contact_store.set_revoked(tenant: command.meta[:tenant], customer: customer)
136
+ @followup_store&.cancel_pending_for(tenant: command.meta[:tenant], customer: customer)
137
+ end
138
+ else
139
+ @contact_store.set_granted(tenant: command.meta[:tenant], customer: customer)
140
+ end
141
+ end
142
+
76
143
  # The turn (or the verdict that this message joined someone else's).
77
144
  def start_turn(command, p, profile)
78
145
  message = p[:message]
79
146
 
80
- # RFC-0015 §5.3 — a fragment for a session whose turn is still at the door
147
+ # a fragment for a session whose turn is still at the door
81
148
  # joins it instead of becoming a turn of its own. Asked BEFORE `create` so a
82
149
  # merge leaves no orphan :queued task behind (Recovery replays :queued at
83
150
  # boot). Only offered on a surface that can report the verdict back —
84
- # §5.5: coalescing a caller that cannot hear `merged` makes it deliver the
151
+ # coalescing a caller that cannot hear `merged` makes it deliver the
85
152
  # same answer twice.
86
- if coalescable?(command)
153
+ # A message carrying MEDIA never joins another turn: `collect`/`steer`
154
+ # move TEXT into a task that is already at the door, and its parts would
155
+ # be left behind — the customer's photo would silently not exist.
156
+ if coalescable?(command) && !media?(p[:parts])
87
157
  if (joined = @executor.collect_into_pending(p[:session_id], message, profile: profile))
88
158
  return { task_id: joined, merged: true }
89
159
  end
90
160
 
91
- # RFC-0015 §5.1 — the turn is already RUNNING: the message is appended to it at
161
+ # the turn is already RUNNING: the message is appended to it at
92
162
  # the next tool-batch boundary. Same verdict as a merge, different word: the
93
163
  # answer comes out of `task_id`, which is not this call's to deliver. Asked
94
164
  # after `collect` because the two cannot both apply — a turn is either still at
@@ -101,16 +171,41 @@ module Insika
101
171
  # command.to_h persists the entire Command in the Task;
102
172
  # ResumeTask re-reads payload.message from there.
103
173
  task = @task_store.create(command: command.to_h, session_id: p[:session_id])
104
- # RFC-0015 §6.4 — `interrupt` mode: the turn in flight is now answering the wrong
174
+ # the channel clock starts HERE — the 202-owning request
175
+ # is accepted, before the SessionActor FIFO and the debounce window.
176
+ # `first_balloon_ms` is the wait the customer feels, so t0 is not the
177
+ # moment the turn finally runs; the same object travels with the turn
178
+ # and `:first_balloon` closes the window at the outbox flush.
179
+ # `interrupt` mode: the turn in flight is now answering the wrong
105
180
  # question, so it is abandoned at its next boundary. This message keeps its OWN
106
181
  # task and its own reply (that is why it needs no verdict and no surface gate), and
107
182
  # the cancel is posted after `create` so the event can name what replaced what.
108
183
  # No-op in every other mode.
109
184
  @executor.interrupt_running(p[:session_id], profile: profile, replaced_by: task.id)
110
- @executor.spawn_in_session(task, profile: profile)
185
+ @executor.spawn_in_session(task, profile: profile,
186
+ timing: channel_inbound_timing(command))
111
187
  { task_id: task.id }
112
188
  end
113
189
 
190
+ # allocate the channel clock at 202 acceptance and stamp
191
+ # `:inbound` — the window's start. `breakdown: false` when INSIKA_TURN_TIMING
192
+ # is off, so a channel turn measures ONLY first_balloon_ms (H-latência never
193
+ # depends on the flag). nil for every non-channel transport: no clock to start.
194
+ def channel_inbound_timing(command)
195
+ return nil unless command.meta[:transport].to_s.start_with?("channel:")
196
+
197
+ timing = Insika::TurnTiming.new(breakdown: Insika::TurnTiming.enabled?)
198
+ timing.mark(:inbound)
199
+ timing
200
+ end
201
+
202
+ # Does the payload carry a part the engine will turn into the turn's
203
+ # substance — audio (transcribed into the message) or an image (attached
204
+ # to the ask)? A text part is not media: it is the message, spelled long.
205
+ def media?(parts)
206
+ Insika::Media.parts(parts).any? { |p| p.audio? || p.image? }
207
+ end
208
+
114
209
  def coalescable?(command)
115
210
  transport = command.meta[:transport]
116
211
  COALESCABLE_TRANSPORTS.include?(transport) || transport.to_s.start_with?("channel:")
@@ -133,7 +228,9 @@ module Insika
133
228
  session_id: payload[:session_id] || payload["session_id"],
134
229
  history: payload[:history] || payload["history"],
135
230
  origin: payload[:origin] || payload["origin"],
136
- event_id: payload[:event_id] || payload["event_id"]
231
+ event_id: payload[:event_id] || payload["event_id"],
232
+ parts: payload[:parts] || payload["parts"],
233
+ customer: payload[:customer] || payload["customer"]
137
234
  }
138
235
  end
139
236