insika 0.2.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (300) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +325 -17
  3. data/README.md +52 -11
  4. data/bin/insika +895 -8
  5. data/bin/insika-router +87 -0
  6. data/docs/AGENTS.md +229 -163
  7. data/docs/API.md +56 -0
  8. data/docs/ARCHITECTURE.md +3 -2
  9. data/docs/ARTIFACTS.md +95 -0
  10. data/docs/BENCHMARK.md +2 -2
  11. data/docs/CHANNELS.md +111 -21
  12. data/docs/CONTEXT.md +132 -19
  13. data/docs/DEMO.md +80 -0
  14. data/docs/DEPLOY.md +75 -4
  15. data/docs/EMBEDDING.md +1 -1
  16. data/docs/EVALS.md +146 -2
  17. data/docs/FACTS.md +135 -0
  18. data/docs/HARVEST.md +116 -0
  19. data/docs/KNOWLEDGE.md +290 -0
  20. data/docs/LOADTEST.md +16 -10
  21. data/docs/MEDIA.md +128 -0
  22. data/docs/OBSERVABILITY.md +65 -6
  23. data/docs/OUTCOMES.md +137 -0
  24. data/docs/PLUGINS.md +51 -6
  25. data/docs/POLICY.md +216 -0
  26. data/docs/REFINEMENT.md +17 -12
  27. data/docs/RELEASING.md +27 -0
  28. data/docs/ROUTER.md +213 -0
  29. data/docs/RUNNING-LOCAL.md +6 -6
  30. data/docs/SCHEDULING.md +121 -0
  31. data/docs/SECURITY.md +79 -2
  32. data/docs/SKILLS.md +13 -4
  33. data/docs/SOAK.md +127 -0
  34. data/docs/TEMPLATES.md +134 -0
  35. data/docs/TOOLS.md +220 -27
  36. data/docs/WHY.md +1 -1
  37. data/docs/WORKFLOWS.md +2 -2
  38. data/docs/_includes/head_custom.html +5 -0
  39. data/docs/_includes/title.html +13 -0
  40. data/docs/_sass/color_schemes/insika.scss +32 -0
  41. data/docs/_sass/custom/custom.scss +199 -0
  42. data/docs/_sass/custom/setup.scss +26 -0
  43. data/docs/assets/img/favicon.svg +7 -0
  44. data/docs/assets/img/insika-mark.svg +7 -0
  45. data/docs/core-concepts.md +21 -0
  46. data/docs/domain.md +115 -0
  47. data/docs/improve.md +20 -0
  48. data/docs/index.md +9 -6
  49. data/docs/integrate.md +20 -0
  50. data/docs/operate.md +13 -6
  51. data/docs/prompts/ADD-TOOL.md +118 -0
  52. data/docs/prompts/DIAGNOSE-TURN.md +65 -0
  53. data/docs/prompts/GO-LIVE.md +138 -0
  54. data/docs/prompts/RUN-EXAMPLES.md +70 -0
  55. data/docs/reference.md +19 -0
  56. data/docs/ship.md +10 -2
  57. data/docs/start-here.md +18 -0
  58. data/lib/insika/agent_profile.rb +202 -10
  59. data/lib/insika/artifact_signing.rb +82 -0
  60. data/lib/insika/artifact_store.rb +160 -0
  61. data/lib/insika/balloon_splitter.rb +102 -0
  62. data/lib/insika/budget_ledger.rb +34 -3
  63. data/lib/insika/cache_series_store.rb +49 -0
  64. data/lib/insika/channel_delivery.rb +119 -14
  65. data/lib/insika/channels/relay.rb +77 -3
  66. data/lib/insika/channels/web.rb +2 -2
  67. data/lib/insika/chat_builder.rb +105 -2
  68. data/lib/insika/checkpoint_store.rb +16 -0
  69. data/lib/insika/commands/agent_payload.rb +3 -3
  70. data/lib/insika/commands/backfill_knowledge.rb +145 -0
  71. data/lib/insika/commands/cancel_followup.rb +49 -0
  72. data/lib/insika/commands/delete_artifact.rb +35 -0
  73. data/lib/insika/commands/delete_concept.rb +34 -0
  74. data/lib/insika/commands/delete_mcp.rb +6 -2
  75. data/lib/insika/commands/delete_tenant_data.rb +107 -0
  76. data/lib/insika/commands/export_customer_memory.rb +48 -0
  77. data/lib/insika/commands/forget_customer.rb +117 -0
  78. data/lib/insika/commands/freeze_funnel_baseline.rb +113 -0
  79. data/lib/insika/commands/gate_harvest.rb +138 -0
  80. data/lib/insika/commands/gate_refinement.rb +1 -1
  81. data/lib/insika/commands/judge_shadow_pairs.rb +124 -0
  82. data/lib/insika/commands/memory_forget_fact.rb +20 -4
  83. data/lib/insika/commands/memory_put_fact.rb +23 -4
  84. data/lib/insika/commands/promote_harvest.rb +130 -0
  85. data/lib/insika/commands/record_outcome.rb +46 -0
  86. data/lib/insika/commands/record_shadow_reply.rb +68 -0
  87. data/lib/insika/commands/refresh_mcp_tools.rb +47 -0
  88. data/lib/insika/commands/reject_harvest.rb +38 -0
  89. data/lib/insika/commands/resolve_proposal.rb +108 -0
  90. data/lib/insika/commands/restore_concept.rb +34 -0
  91. data/lib/insika/commands/revoke_contact.rb +49 -0
  92. data/lib/insika/commands/rollback_harvest.rb +86 -0
  93. data/lib/insika/commands/run_distillation.rb +186 -0
  94. data/lib/insika/commands/run_harvest.rb +393 -0
  95. data/lib/insika/commands/seed_demo_data.rb +31 -0
  96. data/lib/insika/commands/send_message.rb +103 -6
  97. data/lib/insika/commands/session_purge.rb +67 -0
  98. data/lib/insika/commands/upsert_mcp.rb +6 -3
  99. data/lib/insika/commands/write_concept.rb +57 -0
  100. data/lib/insika/contact_store.rb +183 -0
  101. data/lib/insika/context/builder.rb +21 -3
  102. data/lib/insika/context/fragment.rb +7 -3
  103. data/lib/insika/context/priority.rb +5 -0
  104. data/lib/insika/context/provider.rb +17 -3
  105. data/lib/insika/context/providers/briefing.rb +96 -0
  106. data/lib/insika/context/providers/knowledge.rb +108 -0
  107. data/lib/insika/context/providers/memory.rb +16 -7
  108. data/lib/insika/context/providers/prompt.rb +57 -23
  109. data/lib/insika/context/providers/skill.rb +2 -0
  110. data/lib/insika/context/providers/tool_search.rb +2 -0
  111. data/lib/insika/context_trace_store.rb +38 -2
  112. data/lib/insika/cron.rb +189 -0
  113. data/lib/insika/demo/agent_attrs.rb +43 -0
  114. data/lib/insika/demo/golden_cases.rb +81 -0
  115. data/lib/insika/demo/seeder.rb +336 -0
  116. data/lib/insika/distill.rb +224 -0
  117. data/lib/insika/distill_engine.rb +169 -0
  118. data/lib/insika/doctor.rb +889 -11
  119. data/lib/insika/dsl/definition.rb +3 -2
  120. data/lib/insika/dsl/runtime.rb +69 -82
  121. data/lib/insika/dsl/server_boot.rb +92 -1
  122. data/lib/insika/dsl/system.rb +10 -2
  123. data/lib/insika/dsl.rb +186 -5
  124. data/lib/insika/edge_limiter.rb +28 -7
  125. data/lib/insika/env_schema.rb +25 -3
  126. data/lib/insika/errors.rb +11 -0
  127. data/lib/insika/evals/assertions.rb +3 -2
  128. data/lib/insika/evals/golden.rb +41 -4
  129. data/lib/insika/evals/judge.rb +47 -2
  130. data/lib/insika/evals/pairwise.rb +29 -6
  131. data/lib/insika/evals/persona.rb +98 -0
  132. data/lib/insika/evals/runner.rb +9 -0
  133. data/lib/insika/evals/simulator.rb +225 -0
  134. data/lib/insika/evals/transport.rb +83 -1
  135. data/lib/insika/event_stream.rb +10 -0
  136. data/lib/insika/evidence.rb +183 -0
  137. data/lib/insika/executor.rb +869 -76
  138. data/lib/insika/followup_engine.rb +207 -0
  139. data/lib/insika/followup_policy.rb +198 -0
  140. data/lib/insika/followup_store.rb +306 -0
  141. data/lib/insika/funnel_declaration.rb +106 -0
  142. data/lib/insika/funnel_fold.rb +179 -0
  143. data/lib/insika/funnel_store.rb +163 -0
  144. data/lib/insika/golden_store.rb +17 -2
  145. data/lib/insika/grounding/matcher.rb +69 -0
  146. data/lib/insika/grounding.rb +44 -0
  147. data/lib/insika/harvest/conversion_gate.rb +159 -0
  148. data/lib/insika/harvest/criterion.rb +98 -0
  149. data/lib/insika/harvest/gate.rb +194 -0
  150. data/lib/insika/harvest/negative_list.rb +199 -0
  151. data/lib/insika/harvest.rb +241 -0
  152. data/lib/insika/harvest_engine.rb +193 -0
  153. data/lib/insika/harvest_store.rb +548 -0
  154. data/lib/insika/knowledge.rb +680 -0
  155. data/lib/insika/knowledge_store.rb +140 -0
  156. data/lib/insika/mcp_client.rb +94 -0
  157. data/lib/insika/mcp_json.rb +74 -0
  158. data/lib/insika/mcp_live_tool.rb +43 -0
  159. data/lib/insika/mcp_store.rb +98 -26
  160. data/lib/insika/mcp_tool_ingestor.rb +30 -8
  161. data/lib/insika/mcp_tool_registry.rb +100 -0
  162. data/lib/insika/media.rb +382 -0
  163. data/lib/insika/memory_audit_store.rb +85 -0
  164. data/lib/insika/memory_store.rb +264 -23
  165. data/lib/insika/message_origin.rb +7 -2
  166. data/lib/insika/middleware.rb +9 -0
  167. data/lib/insika/model_visible.rb +87 -0
  168. data/lib/insika/model_visible_trace_store.rb +66 -0
  169. data/lib/insika/onboarding.rb +22 -1
  170. data/lib/insika/outbox_store.rb +42 -4
  171. data/lib/insika/outcome_store.rb +147 -0
  172. data/lib/insika/overlay_tool_registry.rb +37 -17
  173. data/lib/insika/packaging.rb +163 -0
  174. data/lib/insika/parity/criterion.rb +79 -0
  175. data/lib/insika/parity/verdict.rb +318 -0
  176. data/lib/insika/prefix_fingerprint.rb +58 -0
  177. data/lib/insika/profile_source.rb +29 -1
  178. data/lib/insika/prompt_catalog.rb +10 -0
  179. data/lib/insika/proposal_store.rb +271 -0
  180. data/lib/insika/queue_policy.rb +4 -1
  181. data/lib/insika/refinement/proposer.rb +1 -1
  182. data/lib/insika/reliability.rb +32 -6
  183. data/lib/insika/retention.rb +316 -0
  184. data/lib/insika/router/app.rb +157 -0
  185. data/lib/insika/router/backend_pool.rb +98 -0
  186. data/lib/insika/router/hash_ring.rb +55 -0
  187. data/lib/insika/router/proxy_body.rb +34 -0
  188. data/lib/insika/router/session_key.rb +54 -0
  189. data/lib/insika/router.rb +18 -0
  190. data/lib/insika/routing.rb +101 -0
  191. data/lib/insika/safety/config.rb +44 -4
  192. data/lib/insika/safety/corpus.rb +255 -0
  193. data/lib/insika/safety/detectors.rb +32 -113
  194. data/lib/insika/safety/factory.rb +15 -2
  195. data/lib/insika/safety/grounding_enforcer.rb +59 -0
  196. data/lib/insika/safety/grounding_validator.rb +49 -0
  197. data/lib/insika/safety/input_guardrail.rb +1 -1
  198. data/lib/insika/safety/moderator.rb +1 -1
  199. data/lib/insika/safety/output_filter.rb +9 -5
  200. data/lib/insika/safety/output_validator.rb +11 -5
  201. data/lib/insika/schedule.rb +177 -0
  202. data/lib/insika/schedule_engine.rb +314 -0
  203. data/lib/insika/schedule_store.rb +208 -0
  204. data/lib/insika/schema_guard.rb +35 -0
  205. data/lib/insika/server/app.rb +247 -17
  206. data/lib/insika/server/rack_app.rb +21 -1
  207. data/lib/insika/server/responses.rb +40 -1
  208. data/lib/insika/session_actor.rb +8 -4
  209. data/lib/insika/session_store.rb +65 -2
  210. data/lib/insika/settings_store.rb +10 -0
  211. data/lib/insika/shadow_pair_store.rb +258 -0
  212. data/lib/insika/skill_catalog.rb +16 -0
  213. data/lib/insika/soak/envelope.rb +140 -0
  214. data/lib/insika/soak/report.rb +392 -0
  215. data/lib/insika/soak/runner.rb +554 -0
  216. data/lib/insika/steer_injector.rb +21 -10
  217. data/lib/insika/store.rb +10 -1
  218. data/lib/insika/stores/memory.rb +6 -0
  219. data/lib/insika/stores/sqlite.rb +8 -0
  220. data/lib/insika/studio/app.rb +1437 -54
  221. data/lib/insika/studio/assets/dist/application.css +1 -1
  222. data/lib/insika/studio/assets/dist/application.js +23 -23
  223. data/lib/insika/studio/assets/dist/favicon.svg +6 -0
  224. data/lib/insika/studio/forms.rb +308 -10
  225. data/lib/insika/studio/nav_icons.rb +19 -1
  226. data/lib/insika/studio/views/_agent_tab_cache.erb +25 -0
  227. data/lib/insika/studio/views/_agent_tab_config.erb +514 -0
  228. data/lib/insika/studio/views/_agent_tab_history.erb +24 -0
  229. data/lib/insika/studio/views/_agent_tab_loops.erb +54 -0
  230. data/lib/insika/studio/views/_agent_tab_memory.erb +51 -0
  231. data/lib/insika/studio/views/_agent_tab_outcomes.erb +31 -0
  232. data/lib/insika/studio/views/_agent_tab_prompts.erb +108 -0
  233. data/lib/insika/studio/views/_agent_tab_skills.erb +38 -0
  234. data/lib/insika/studio/views/_agents_master.erb +44 -0
  235. data/lib/insika/studio/views/_message.erb +49 -32
  236. data/lib/insika/studio/views/agent_detail.erb +62 -278
  237. data/lib/insika/studio/views/agents.erb +71 -54
  238. data/lib/insika/studio/views/approvals.erb +4 -1
  239. data/lib/insika/studio/views/artifact.erb +23 -0
  240. data/lib/insika/studio/views/artifacts.erb +59 -0
  241. data/lib/insika/studio/views/chats.erb +4 -1
  242. data/lib/insika/studio/views/customer.erb +94 -0
  243. data/lib/insika/studio/views/customers.erb +32 -0
  244. data/lib/insika/studio/views/evals.erb +6 -3
  245. data/lib/insika/studio/views/facts.erb +133 -0
  246. data/lib/insika/studio/views/followups.erb +125 -0
  247. data/lib/insika/studio/views/funnel.erb +106 -0
  248. data/lib/insika/studio/views/harvest.erb +234 -0
  249. data/lib/insika/studio/views/home.erb +106 -66
  250. data/lib/insika/studio/views/knowledge.erb +123 -0
  251. data/lib/insika/studio/views/layout.erb +15 -11
  252. data/lib/insika/studio/views/mcp.erb +174 -80
  253. data/lib/insika/studio/views/parity.erb +147 -0
  254. data/lib/insika/studio/views/playground.erb +7 -1
  255. data/lib/insika/studio/views/session.erb +233 -124
  256. data/lib/insika/studio/views/settings.erb +41 -1
  257. data/lib/insika/studio/views/skills.erb +1 -2
  258. data/lib/insika/studio/views/system_files.erb +1 -1
  259. data/lib/insika/studio/views/task.erb +13 -0
  260. data/lib/insika/studio/views/tasks.erb +4 -1
  261. data/lib/insika/studio/views/tools.erb +24 -10
  262. data/lib/insika/task_store.rb +21 -1
  263. data/lib/insika/templates/browser-agent/README.md +36 -0
  264. data/lib/insika/templates/browser-agent/agent.rb +49 -0
  265. data/lib/insika/templates/daily-digest/README.md +38 -0
  266. data/lib/insika/templates/daily-digest/agent.rb +77 -0
  267. data/lib/insika/templates/repo-explorer/README.md +36 -0
  268. data/lib/insika/templates/repo-explorer/agent.rb +45 -0
  269. data/lib/insika/templates/research-analyst/README.md +26 -0
  270. data/lib/insika/templates/research-analyst/agent.rb +58 -0
  271. data/lib/insika/templates/review-panel/README.md +20 -0
  272. data/lib/insika/templates/review-panel/agent.rb +50 -0
  273. data/lib/insika/templates/travel-planner/README.md +35 -0
  274. data/lib/insika/templates/travel-planner/agent.rb +87 -0
  275. data/lib/insika/templates.rb +112 -0
  276. data/lib/insika/testing/store_contract.rb +27 -6
  277. data/lib/insika/tick.rb +47 -11
  278. data/lib/insika/timezone.rb +45 -0
  279. data/lib/insika/tool_definition.rb +17 -7
  280. data/lib/insika/tool_envelope.rb +69 -0
  281. data/lib/insika/tool_manifest.rb +5 -1
  282. data/lib/insika/tools/data_defined_tool.rb +10 -0
  283. data/lib/insika/tools/generate_image.rb +89 -0
  284. data/lib/insika/tools/load_knowledge.rb +74 -0
  285. data/lib/insika/tools/run_persona_eval.rb +328 -0
  286. data/lib/insika/tools/save_artifact.rb +95 -0
  287. data/lib/insika/tools/schedule_followup.rb +164 -0
  288. data/lib/insika/tools/tts.rb +47 -0
  289. data/lib/insika/tools/update_briefing.rb +126 -0
  290. data/lib/insika/turn_output.rb +1 -1
  291. data/lib/insika/turn_state.rb +49 -1
  292. data/lib/insika/turn_timing.rb +22 -2
  293. data/lib/insika/version.rb +1 -1
  294. data/lib/insika/vitals.rb +84 -0
  295. data/lib/insika/wiring/graph.rb +472 -13
  296. data/lib/insika/wiring/graph_chat.rb +102 -0
  297. data/lib/insika.rb +149 -3
  298. metadata +181 -5
  299. data/docs/build.md +0 -14
  300. data/docs/understand.md +0 -10
@@ -23,7 +23,7 @@ module Insika
23
23
  # a third party that refused N times is an operator problem, not
24
24
  # something to replay forever.
25
25
  #
26
- # `failed` and `delivering` are a deliberate widening of the RFC's two-status
26
+ # `failed` and `delivering` are a deliberate widening of the two-status
27
27
  # sketch (`pending -> delivered`): without them a crashed claim and an
28
28
  # exhausted retry are indistinguishable from a fresh record, and the sweep
29
29
  # would redeliver both.
@@ -38,7 +38,7 @@ module Insika
38
38
 
39
39
  Delivery = Data.define(
40
40
  :id, :channel, :to, :task_id, :session_id, :payload,
41
- :status, :attempts, :last_error, :created_at, :updated_at
41
+ :status, :attempts, :last_error, :index, :created_at, :updated_at
42
42
  )
43
43
 
44
44
  def initialize(store:)
@@ -47,7 +47,9 @@ module Insika
47
47
 
48
48
  # -> Delivery (:pending). `payload` is the body the channel will send; it is
49
49
  # DATA (string keys, JSON types) and the store never interprets it.
50
- def create(channel:, to:, task_id:, session_id:, payload:, id: SecureRandom.uuid)
50
+ # `index` is the balloon's position inside its turn — 0 for a
51
+ # plain `:at_end` delivery, written by a progressive flush.
52
+ def create(channel:, to:, task_id:, session_id:, payload:, index: 0, id: SecureRandom.uuid)
51
53
  record = {
52
54
  "id" => id.to_s,
53
55
  "channel" => channel.to_s,
@@ -58,6 +60,7 @@ module Insika
58
60
  "status" => "pending",
59
61
  "attempts" => 0,
60
62
  "last_error" => nil,
63
+ "index" => index.to_i,
61
64
  "created_at" => timestamp,
62
65
  "updated_at" => timestamp
63
66
  }
@@ -75,8 +78,12 @@ module Insika
75
78
  # `delivering`: that one was claimed by a process that then died, and whether
76
79
  # its POST landed is unknowable — replaying it is the duplicate the claim
77
80
  # exists to prevent.
81
+ #
82
+ # Ordered by [task_id, index] : a crashed progressive turn re-drives
83
+ # balloon 1 only after balloon 0, never the reverse.
78
84
  def pending
79
85
  scan.select { |d| d.status == :pending }
86
+ .sort_by { |d| [d.task_id.to_s, d.index] }
80
87
  end
81
88
 
82
89
  # pending -> delivering, ATOMICALLY — across processes, not just fibers: the
@@ -127,8 +134,38 @@ module Insika
127
134
  touch(id, record)
128
135
  end
129
136
 
137
+ # WS8 (LGPD): drops every delivery of these sessions, whatever its status.
138
+ # `payload` is the ANSWER as it was handed to the channel, so a purge that
139
+ # stops at the session record leaves the conversation readable here forever.
140
+ # -> count removed.
141
+ def purge_sessions(session_ids)
142
+ wanted = Array(session_ids).map(&:to_s)
143
+ return 0 if wanted.empty?
144
+
145
+ delete_where { |d| wanted.include?(d.session_id.to_s) }
146
+ end
147
+
148
+ # WS8 retention: deliveries created before the cutoff. TERMINAL ones only —
149
+ # a `pending`/`delivering` record older than the window is still somebody's
150
+ # undelivered answer, and the sweep is not the place to decide it is lost.
151
+ # -> count removed.
152
+ def delete_older_than(time)
153
+ cutoff = time.utc.iso8601
154
+ delete_where do |d|
155
+ %i[delivered failed].include?(d.status) && d.created_at.to_s < cutoff
156
+ end
157
+ end
158
+
130
159
  private
131
160
 
161
+ # The id list is SNAPSHOTTED before the deletes: `scan` enumerates the
162
+ # backend's keys lazily and deleting under it would skip records.
163
+ def delete_where(&match)
164
+ doomed = scan.select(&match)
165
+ doomed.each { |d| @store.delete(SCOPE, key_for(d.id)) }
166
+ doomed.size
167
+ end
168
+
132
169
  def scan
133
170
  return enum_for(:scan) unless block_given?
134
171
 
@@ -157,7 +194,8 @@ module Insika
157
194
  task_id: record["task_id"], session_id: record["session_id"],
158
195
  payload: record["payload"], status: record["status"].to_sym,
159
196
  attempts: record["attempts"].to_i, last_error: record["last_error"],
160
- created_at: record["created_at"], updated_at: record["updated_at"]
197
+ index: record["index"].to_i, created_at: record["created_at"],
198
+ updated_at: record["updated_at"]
161
199
  )
162
200
  end
163
201
 
@@ -0,0 +1,147 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "securerandom"
4
+ require "time"
5
+
6
+ module Insika
7
+ # WS7: business outcomes over real traffic, recorded per conversation by the
8
+ # OPERATOR or the integration (`POST /v1/outcomes`) — `conversion`,
9
+ # `escalation`, `deflected`, … optionally with a monetary `value`. The engine
10
+ # TRANSPORTS the outcome and never interprets it (the same rule as alerts):
11
+ # what "conversion" means is the operator's. Durable in the KV backend,
12
+ # tenant-scoped (WS1) — a tenant can only ever read/write its own cells (the
13
+ # key carries the tenant, and the read methods narrow on it).
14
+ #
15
+ # The Studio's scorecard is the consumer: the LAST outcome per agent as a
16
+ # pill on the grid, and the per-day series on the agent detail — both fed
17
+ # from the same store, read on demand.
18
+ class OutcomeStore
19
+ SCOPE = "outcomes"
20
+
21
+ Record = Data.define(:id, :tenant, :agent, :session_id, :outcome, :value, :at)
22
+
23
+ def initialize(store:)
24
+ @store = store
25
+ end
26
+
27
+ # -> Record. `at` defaults to now (UTC); `value` defaults to 0.0 (an
28
+ # outcome without a value is a unit outcome).
29
+ def create(tenant:, agent:, session_id: nil, outcome:, value: nil, id: SecureRandom.uuid, at: Time.now.utc)
30
+ time = at.utc
31
+ record = { "id" => id, "tenant" => tenant.to_s, "agent" => agent.to_s,
32
+ "session_id" => session_id.to_s, "outcome" => outcome.to_s,
33
+ "value" => value.to_f, "at" => time.iso8601 }
34
+ @store.set(SCOPE, key(tenant, agent, time, id), record)
35
+ to_record(record)
36
+ end
37
+
38
+ # All records, NEWEST first. `tenant:` narrows to one tenant's (WS1 — the
39
+ # read path a tenant query uses); `agent:` narrows further. An outcome is a
40
+ # fact, never a secret: the fields are ids, an outcome name and a number.
41
+ # NOTE: `tenant: nil` means NO filter — the record's tenant field is
42
+ # `tenant.to_s` ("" for a single-tenant write), so a caller that wants only
43
+ # the no-tenant records must pass `tenant: ""` explicitly (the FOLD uses
44
+ # `for_pair`, never this nil-means-everything trap).
45
+ def all(tenant: nil, agent: nil)
46
+ records = @store.list(SCOPE).filter_map { |k| to_record(@store.get(SCOPE, k)) }
47
+ records.select! { |r| r.tenant == tenant } if tenant
48
+ records.select! { |r| r.agent == agent } if agent
49
+ records.sort_by(&:at).reverse
50
+ end
51
+
52
+ # the fold's per-pair read — one KEY-prefix scan (the key IS
53
+ # tenant:agent:YYYY-MM-DD:id, WS1), optionally skipping the keys older than
54
+ # an ISO date WITHOUT reading them (the fold's cursor day — everything
55
+ # before it is already folded). `tenant:` takes either spelling (nil/""
56
+ # and "platform" all mean the no-tenant key segment). -> [Record].
57
+ def for_pair(tenant:, agent:, since_date: nil)
58
+ prefix = "#{key_tenant(tenant)}:#{agent}:"
59
+ keys = @store.list(SCOPE).select { |k| k.start_with?(prefix) }
60
+ keys.select! { |k| date_segment(k) >= since_date } if since_date
61
+ keys.filter_map { |k| to_record(@store.get(SCOPE, k)) }
62
+ end
63
+
64
+ # the distinct (tenant, agent) pairs present in the store —
65
+ # one key scan, no record reads. `tenant` is nil for the no-tenant segment
66
+ # (the FunnelStore#pairs spelling; every key-based API normalizes it back).
67
+ # -> [{ tenant: String | nil, agent: String }]
68
+ def pairs
69
+ @store.list(SCOPE).map { |k| pair_of(k) }.uniq
70
+ end
71
+
72
+ # -> { agent => { outcome:, value:, at:, session_id: } }: the LAST outcome
73
+ # per agent — the Studio's scorecard state card. A map so the agent grid
74
+ # looks up in O(1) per card.
75
+ def latest_per_agent(tenant: nil)
76
+ all(tenant: tenant).each_with_object({}) do |r, acc|
77
+ acc[r.agent] ||= { outcome: r.outcome, value: r.value, at: r.at,
78
+ session_id: r.session_id }
79
+ end
80
+ end
81
+
82
+ # -> { "YYYY-MM-DD" => { outcome => { count:, value: } } } — per-period
83
+ # series for the Studio. `period: :month` buckets by "YYYY-MM".
84
+ def series(tenant: nil, agent: nil, period: :day)
85
+ all(tenant: tenant, agent: agent).each_with_object({}) do |r, acc|
86
+ bucket = period == :month ? r.at[0, 7] : r.at[0, 10]
87
+ cell = ((acc[bucket] ||= {})[r.outcome] ||= { count: 0, value: 0.0 })
88
+ cell[:count] += 1
89
+ cell[:value] += r.value
90
+ end
91
+ end
92
+
93
+ # Purges a tenant's records (WS8 phase 2 — delete_tenant_data). The tenant
94
+ # is the FIRST key segment, so the purge is a prefix scan — the key IS the
95
+ # isolation (WS1). -> count removed.
96
+ def purge(tenant:)
97
+ prefix = "#{tenant}:"
98
+ keys = @store.list(SCOPE).select { |k| k.start_with?(prefix) }
99
+ keys.each { |k| @store.delete(SCOPE, k) }
100
+ keys.size
101
+ end
102
+
103
+ # Purges records older than the cutoff (WS8 retention — the tick's sweep).
104
+ # `at` is ISO8601 UTC, so the comparison is lexicographic. -> count removed.
105
+ def delete_older_than(time)
106
+ cutoff = time.utc.iso8601
107
+ removed = 0
108
+ @store.list(SCOPE).each do |k|
109
+ rec = @store.get(SCOPE, k)
110
+ next unless rec && rec["at"].to_s < cutoff
111
+
112
+ @store.delete(SCOPE, k)
113
+ removed += 1
114
+ end
115
+ removed
116
+ end
117
+
118
+ private
119
+
120
+ def key_tenant(tenant) = tenant.to_s.empty? ? "platform" : tenant.to_s
121
+
122
+ # The date segment of the key (the 3rd segment, 0-indexed: 2).
123
+ def date_segment(k) = k.split(":").fetch(2, "")
124
+
125
+ # Parses the key's tenant segment back: "platform" -> nil (the
126
+ # FunnelStore#pairs spelling); anything else stays.
127
+ def pair_of(k)
128
+ tenant, agent, = k.split(":")
129
+ { tenant: tenant == "platform" ? nil : tenant, agent: agent }
130
+ end
131
+
132
+ def key(tenant, agent, time, id)
133
+ # tenant + agent + UTC date prefix: per-period / per-agent listing is a
134
+ # prefix scan, and the tenant IS the first segment — WS1 isolation is the
135
+ # key itself, like the session namespace.
136
+ "#{(tenant || 'platform')}:#{agent}:#{time.strftime('%Y-%m-%d')}:#{id}"
137
+ end
138
+
139
+ def to_record(rec)
140
+ return nil if rec.nil?
141
+
142
+ Record.new(id: rec["id"], tenant: rec["tenant"], agent: rec["agent"],
143
+ session_id: rec["session_id"], outcome: rec["outcome"],
144
+ value: rec["value"].to_f, at: rec["at"])
145
+ end
146
+ end
147
+ end
@@ -2,47 +2,57 @@
2
2
 
3
3
  module Insika
4
4
  # DYNAMIC tool registry: composes the CODE registry (base, built at boot,
5
- # immutable) with the DATA-DEFINED tools from the ToolStore. Drop-in for ToolRegistry
6
- # the Executor/ToolCatalog/ToolEnvelope only use entries/resolve/side_effect?.,
5
+ # immutable), the DATA-DEFINED tools from the ToolStore, and optionally
6
+ # the LIVE MCP tools from Insika::McpToolRegistry. Drop-in
7
+ # for ToolRegistry — the Executor/ToolCatalog/ToolEnvelope only use
8
+ # entries/resolve/side_effect?.
7
9
  #
8
10
  # Rules:
9
- # - COLLISION: the base (code) ALWAYS wins — a data-tool cannot hijack
10
- # the name of a code tool (security, R3). The authoring Command also
11
+ # - COLLISION: the base (code) ALWAYS wins — a data-tool or an MCP tool
12
+ # cannot hijack the name of a code tool (security, R3); a data-tool
13
+ # also wins over an MCP tool of the same name (an operator-authored
14
+ # definition over a server's own naming). The authoring Command also
11
15
  # refuses to create with a colliding name (code_tool?), but the defense stays here.
12
16
  # - HOT: `reload` re-reads the store and swaps the dynamic index atomically — a
13
17
  # new/edited data-tool takes effect on the next turn without a restart, mirroring
14
- # SkillCatalog.reload. An in-flight turn has already captured the index.
15
- # PARITY: empty ToolStore entries/resolve/side_effect? identical to the
16
- # pure base. The base (config/wiring.rb) does not even use the overlay — zero regression.
18
+ # SkillCatalog.reload. An in-flight turn has already captured the index. MCP
19
+ # entries need no such reload they read McpStore#tools_cache fresh every call
20
+ # (no I/O, so there is nothing to memoize-then-invalidate).
21
+ # PARITY: empty ToolStore + no mcp_registry ⇒ entries/resolve/side_effect? identical
22
+ # to the pure base. The base (config/wiring.rb) does not even use the overlay — zero regression.
17
23
  #
18
- # The data-tools enter as NORMAL Registry::Entry (optional: false) — they obey
19
- # the same per-agent allow/deny as code tools; exposure is the operator's
20
- # (the /tools matrix), not automatic just because they are "data-defined".
24
+ # The data-tools and MCP tools enter as NORMAL Registry::Entry (optional:
25
+ # false) — they obey the same per-agent allow/deny as code tools; exposure
26
+ # is the operator's (the /tools matrix), not automatic just because they
27
+ # are "data-defined" or MCP-discovered.
21
28
  class OverlayToolRegistry
22
- def initialize(base:, tool_store:, http:, egress: Insika::EgressGuard, egress_options: {}, event_stream: nil)
29
+ def initialize(base:, tool_store:, http:, egress: Insika::EgressGuard, egress_options: {}, event_stream: nil,
30
+ mcp_registry: nil)
23
31
  @base = base
24
32
  @tool_store = tool_store
25
33
  @http = http
26
34
  @egress = egress
27
35
  @egress_options = egress_options
28
36
  @event_stream = event_stream
37
+ @mcp_registry = mcp_registry
29
38
  end
30
39
 
31
- # Base + dynamic, except dynamic ones that collide with the base (base wins).
40
+ # Base + dynamic + mcp, except any that collide with something higher in
41
+ # the precedence (base > data-tools > mcp).
32
42
  def entries
33
- @base.entries + dynamic.reject { |e| code_tool?(e.name) }
43
+ @base.entries + dynamic.reject { |e| code_tool?(e.name) } + mcp_entries
34
44
  end
35
45
 
36
46
  def names
37
- (@base.names + dynamic.map(&:name)).uniq
47
+ (@base.names + dynamic.map(&:name) + mcp_entries.map(&:name)).uniq
38
48
  end
39
49
 
40
- # -> instance (base wins) | raise NotFoundError.
50
+ # -> instance (base wins, then data-tools) | raise NotFoundError.
41
51
  def resolve(name)
42
52
  key = name.to_s
43
53
  return @base.resolve(key) if code_tool?(key)
44
54
 
45
- entry = dynamic.find { |e| e.name == key }
55
+ entry = dynamic.find { |e| e.name == key } || mcp_entries.find { |e| e.name == key }
46
56
  raise Insika::NotFoundError, "'#{name}' not registered in #{self.class}" unless entry
47
57
 
48
58
  entry.factory.call
@@ -53,7 +63,7 @@ module Insika
53
63
  key = name.to_s
54
64
  return @base.side_effect?(key) if code_tool?(key)
55
65
 
56
- entry = dynamic.find { |e| e.name == key }
66
+ entry = dynamic.find { |e| e.name == key } || mcp_entries.find { |e| e.name == key }
57
67
  entry ? !!entry.metadata[:side_effect] : false
58
68
  end
59
69
 
@@ -72,6 +82,16 @@ module Insika
72
82
  @dynamic ||= build_dynamic
73
83
  end
74
84
 
85
+ # No I/O (McpToolRegistry#entries only reads McpStore#tools_cache) ->
86
+ # recomputed fresh every call, unlike `dynamic` (nothing to reload).
87
+ # Drops anything already claimed by the base or a data-tool.
88
+ def mcp_entries
89
+ return [] unless @mcp_registry
90
+
91
+ claimed = @base.names + dynamic.map(&:name)
92
+ @mcp_registry.entries.reject { |e| claimed.include?(e.name) }
93
+ end
94
+
75
95
  def build_dynamic
76
96
  @tool_store.all_raw.filter_map { |raw| entry_for(raw) }
77
97
  end
@@ -0,0 +1,163 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Insika
4
+ # The single source of the gem payload selection. The
5
+ # gemspec and the domain-boundary audit spec call THE SAME function, so
6
+ # "what ships in the gem" is a fact the suite asserts on, never a prose
7
+ # promise: `git ls-files` (fallback: a glob where there is no .git) filtered
8
+ # to lib/ + docs/ + the four root files, minus the never-ship set.
9
+ #
10
+ # Pure and dependency-free (no git gem, no FileUtils beyond reads) so both
11
+ # the gemspec (built outside the app) and the spec can load it. It does NOT
12
+ # ship anything itself — it answers a list.
13
+ #
14
+ # `domain_content?` is the audit's yardstick: best-effort detection of pt-BR
15
+ # domain content in a payload file. Ruby files are scanned for their string
16
+ # literals, regex literals and heredoc bodies (comments stripped);
17
+ # everything else is scanned as raw text. The token set is the pt-BR
18
+ # vocabulary + the demo persona name (PT_BR_TOKENS). The corpus data files
19
+ # and the audit's own token table live in the spec's named allowlist.
20
+ module Packaging
21
+ # The pt-BR domain vocabulary the audit scans for. `loja`/`shop`/`store`
22
+ # are generic-retail stopwords — the ONE justification the inventory
23
+ # allows (D3); `bia` is the demo persona name (D8).
24
+ #
25
+ # The scan is FOLD-aware: both the scanned text and these words go through
26
+ # `fold` (accents stripped, regex bracket classes like `voc[êe]` collapsed
27
+ # to their canonical chars) before matching. That is what makes the
28
+ # shipped corpus data — whose patterns encode pt-BR as `voc[êe]`,
29
+ # `instru[çc][õo]es` — detectable by the same token table (D1: the
30
+ # boundary holds for the content the inventory names).
31
+ ACCENT_MAP = {
32
+ "á" => "a", "à" => "a", "â" => "a", "ã" => "a", "ä" => "a",
33
+ "é" => "e", "è" => "e", "ê" => "e", "ë" => "e",
34
+ "í" => "i", "ì" => "i", "î" => "i", "ï" => "i",
35
+ "ó" => "o", "ò" => "o", "ô" => "o", "õ" => "o", "ö" => "o",
36
+ "ú" => "u", "ù" => "u", "û" => "u", "ü" => "u",
37
+ "ç" => "c"
38
+ }.freeze
39
+ BRACKET_CLASS = /\[([^\]]*)\]/
40
+
41
+ # -> String: the canonical form the token match runs on. A bracket class
42
+ # folds to the deduped canonical chars of its members — `voc[êe]` folds
43
+ # to "voce", matching the folded token "voce" (você).
44
+ def self.fold(text)
45
+ text.to_s.gsub(BRACKET_CLASS) do
46
+ Regexp.last_match(1).chars.map { |c| ACCENT_MAP.fetch(c, c) }.uniq.join
47
+ end.gsub(/[#{ACCENT_MAP.keys.join}]/, ACCENT_MAP)
48
+ end
49
+
50
+ TOKEN_WORDS = ["você", "não", "loja", "pedido", "atendente", "obrigad",
51
+ "conversa", "cliente"].freeze
52
+ PT_BR_VOCABULARY = /\b(?:#{TOKEN_WORDS.map { |w| fold(w) }.join("|")})\b/i
53
+ PERSONA_NAME = /\bbia\b/i
54
+ PT_BR_TOKENS = /(?:#{PT_BR_VOCABULARY.source}|#{PERSONA_NAME.source})/i
55
+
56
+ # The never-ship set: the studio JS toolchain (only assets/dist ships) and
57
+ # the docs' own Jekyll Gemfile/config.
58
+ NEVER_SHIP_PREFIXES = ["node_modules", "lib/insika/studio/test/", "lib/insika/studio/assets/src/"].freeze
59
+ NEVER_SHIP_PATHS = %w[
60
+ docs/Gemfile docs/Gemfile.lock docs/_config.yml
61
+ lib/insika/studio/README.md lib/insika/studio/package.json
62
+ lib/insika/studio/package-lock.json lib/insika/studio/tailwind.config.js
63
+ ].freeze
64
+
65
+ module_function
66
+
67
+ # -> [String] repo-relative file paths the gem ships, in the gemspec's
68
+ # order: `git ls-files` (fallback: glob where there is no .git), filtered
69
+ # to lib/ + docs/ + the four root files, minus the never-ship set.
70
+ def payload_files(root = Dir.pwd)
71
+ Dir.chdir(root) do
72
+ tracked = `git ls-files -z 2>/dev/null`.split("\x0")
73
+ if tracked.empty?
74
+ tracked = Dir.glob("{lib,docs}/**/*", File::FNM_DOTMATCH).reject { |f| File.directory?(f) } +
75
+ %w[README.md LICENSE CHANGELOG.md bin/insika bin/insika-router]
76
+ end
77
+ tracked.select { |file| payload_path?(file) }.reject { |file| excluded?(file) }
78
+ end
79
+ end
80
+
81
+ # -> bool: is this repo-relative path part of the payload selection?
82
+ def payload_path?(file)
83
+ file.start_with?("lib/", "docs/") || %w[README.md LICENSE CHANGELOG.md bin/insika bin/insika-router].include?(file)
84
+ end
85
+
86
+ # -> bool: is this file in the never-ship set?
87
+ def excluded?(file)
88
+ NEVER_SHIP_PREFIXES.any? { |p| file.include?(p) } || NEVER_SHIP_PATHS.include?(file)
89
+ end
90
+
91
+ # -> bool: does a payload file hold pt-BR domain content? Best-effort:
92
+ # - Ruby files: comments stripped, string literals / regex literals /
93
+ # heredoc bodies scanned;
94
+ # - everything else: scanned as-is.
95
+ # The token set is the pt-BR vocabulary + the demo persona name; both the
96
+ # text and the vocabulary are FOLDED before matching (accents stripped,
97
+ # bracket classes collapsed), so the corpus data's `voc[êe]`-style
98
+ # patterns are caught by the same table.
99
+ def domain_content?(path)
100
+ text = path.to_s.end_with?(".rb") ? ruby_text(path) : File.read(path)
101
+ fold(text).match?(PT_BR_TOKENS)
102
+ end
103
+
104
+ # -> bool: does a payload file mention the demo persona name (`bia`)?
105
+ def persona_content?(path)
106
+ text = path.to_s.end_with?(".rb") ? ruby_text(path) : File.read(path)
107
+ text.match?(PERSONA_NAME)
108
+ end
109
+
110
+ # The Ruby source as the audit reads it: heredoc bodies verbatim, string +
111
+ # regex literals, comments stripped. Best-effort by design (D1).
112
+ def ruby_text(path)
113
+ lines = File.readlines(path)
114
+ out = +""
115
+ heredoc = nil
116
+ lines.each do |line|
117
+ if heredoc
118
+ out << line
119
+ heredoc = nil if line.strip == heredoc
120
+ next
121
+ end
122
+ if (m = line.match(HEREDOC_OPEN))
123
+ heredoc = m[1] || m[2] || m[3]
124
+ next
125
+ end
126
+ code = strip_comments(line)
127
+ out << code.scan(STRING_LITERAL).join("\n")
128
+ out << code.scan(REGEX_LITERAL).join("\n")
129
+ end
130
+ out
131
+ end
132
+
133
+ # Cuts a line at its first `#` that starts a COMMENT — i.e. a `#` NOT
134
+ # inside a string/regex literal (an interpolation `#{…}` or a literal `#`
135
+ # must survive; a naive strip would corrupt the literal and hide content).
136
+ def strip_comments(line)
137
+ spans = literal_spans(line)
138
+ cut = nil
139
+ line.chars.each_index do |i|
140
+ next unless line[i] == "#"
141
+ next if spans.any? { |b, e| i >= b && i < e }
142
+
143
+ cut = i
144
+ break
145
+ end
146
+ cut ? line[0...cut] : line
147
+ end
148
+
149
+ # -> [[begin, end), …] spans of the complete string/regex literals in a line.
150
+ def literal_spans(line)
151
+ spans = []
152
+ line.scan(STRING_LITERAL) { spans << [Regexp.last_match.begin(0), Regexp.last_match.end(0)] }
153
+ line.scan(REGEX_LITERAL) { spans << [Regexp.last_match.begin(0), Regexp.last_match.end(0)] }
154
+ spans
155
+ end
156
+
157
+ HEREDOC_OPEN = /<<[~-]\s*(?:"([^"]+)"|'([^']+)'|([A-Za-z_]\w*))(?:\s*\.\w+)*\s*\z/
158
+ STRING_LITERAL = /"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'/
159
+ REGEX_LITERAL = %r{\/(?:[^\/\\\n]|\\.)+\/[a-z]*}
160
+
161
+ private_class_method :payload_path?, :excluded?, :ruby_text, :strip_comments, :literal_spans
162
+ end
163
+ end
@@ -0,0 +1,79 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "digest"
4
+ require "yaml"
5
+
6
+ module Insika
7
+ module Parity
8
+ # C5 — the frozen criterion, parsed from the deployment's criterion file.
9
+ # The prose a human reads and the yaml block the machine applies are the SAME
10
+ # file, so there is exactly one place to edit — and the file's WHOLE bytes are
11
+ # hashed, so an edit to the rationale invalidates the frozen rule too, which is
12
+ # correct: the rationale is what makes the numbers reviewable.
13
+ #
14
+ # Strict by construction: a missing key or an unknown key is refused at load
15
+ # (`ValidationError`), and no key has a default — a number the machine filled in
16
+ # is a number nobody pre-registered.
17
+ class Criterion
18
+ Rule = Data.define(
19
+ :version, :unit, :window_days, :pairs_per_day, :min_decided,
20
+ :min_judge_models, :both_orders,
21
+ :win_or_tie_floor, :estimator,
22
+ :worse_rate_ceiling, :undecided_rate_ceiling, :incomplete_rate_ceiling,
23
+ :per_agent_min_decided, :per_agent_win_or_tie_floor,
24
+ :human_assisted, :silent
25
+ )
26
+
27
+ attr_reader :rule, :path, :sha
28
+
29
+ def initialize(rule:, path:, sha:)
30
+ @rule = rule
31
+ @path = path
32
+ @sha = sha
33
+ end
34
+
35
+ # Reads the file, extracts the FIRST ```yaml fence, validates STRICTLY.
36
+ # Raises Insika::ConfigError when the file is absent or has no block;
37
+ # Insika::ValidationError on an unknown or missing key.
38
+ def self.load(path)
39
+ bytes = read_bytes(path)
40
+ block = extract_yaml_block(bytes)
41
+ parsed = YAML.safe_load(block, permitted_classes: [], aliases: false)
42
+ rule = build_rule(parsed)
43
+ new(rule: rule, path: path, sha: "sha256:#{Digest::SHA256.hexdigest(bytes)}")
44
+ end
45
+
46
+ def to_h = @rule.to_h
47
+
48
+ class << self
49
+ private
50
+
51
+ def read_bytes(path)
52
+ File.read(path, encoding: "UTF-8")
53
+ rescue Errno::ENOENT, Errno::EACCES, Errno::EISDIR => e
54
+ raise Insika::ConfigError, "parity criterion not readable at #{path}: #{e.class}"
55
+ end
56
+
57
+ def extract_yaml_block(bytes)
58
+ match = bytes.match(/```yaml\n(.*?)\n```/m)
59
+ raise Insika::ConfigError, "parity criterion has no ```yaml block (nothing to pre-register)" unless match
60
+
61
+ match[1]
62
+ end
63
+
64
+ def build_rule(parsed)
65
+ raise Insika::ValidationError, "criterion yaml block must be a mapping" unless parsed.is_a?(Hash)
66
+
67
+ raw = parsed.transform_keys(&:to_s)
68
+ keys = raw.keys.sort
69
+ missing = Rule.members.map(&:to_s) - keys
70
+ unknown = keys - Rule.members.map(&:to_s)
71
+ raise Insika::ValidationError, "criterion is missing key(s): #{missing.join(', ')}" if missing.any?
72
+ raise Insika::ValidationError, "criterion has unknown key(s): #{unknown.join(', ')}" if unknown.any?
73
+
74
+ Rule.new(**Rule.members.to_h { |m| [m, raw[m.to_s]] })
75
+ end
76
+ end
77
+ end
78
+ end
79
+ end