insika 0.2.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (300) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +325 -17
  3. data/README.md +52 -11
  4. data/bin/insika +895 -8
  5. data/bin/insika-router +87 -0
  6. data/docs/AGENTS.md +229 -163
  7. data/docs/API.md +56 -0
  8. data/docs/ARCHITECTURE.md +3 -2
  9. data/docs/ARTIFACTS.md +95 -0
  10. data/docs/BENCHMARK.md +2 -2
  11. data/docs/CHANNELS.md +111 -21
  12. data/docs/CONTEXT.md +132 -19
  13. data/docs/DEMO.md +80 -0
  14. data/docs/DEPLOY.md +75 -4
  15. data/docs/EMBEDDING.md +1 -1
  16. data/docs/EVALS.md +146 -2
  17. data/docs/FACTS.md +135 -0
  18. data/docs/HARVEST.md +116 -0
  19. data/docs/KNOWLEDGE.md +290 -0
  20. data/docs/LOADTEST.md +16 -10
  21. data/docs/MEDIA.md +128 -0
  22. data/docs/OBSERVABILITY.md +65 -6
  23. data/docs/OUTCOMES.md +137 -0
  24. data/docs/PLUGINS.md +51 -6
  25. data/docs/POLICY.md +216 -0
  26. data/docs/REFINEMENT.md +17 -12
  27. data/docs/RELEASING.md +27 -0
  28. data/docs/ROUTER.md +213 -0
  29. data/docs/RUNNING-LOCAL.md +6 -6
  30. data/docs/SCHEDULING.md +121 -0
  31. data/docs/SECURITY.md +79 -2
  32. data/docs/SKILLS.md +13 -4
  33. data/docs/SOAK.md +127 -0
  34. data/docs/TEMPLATES.md +134 -0
  35. data/docs/TOOLS.md +220 -27
  36. data/docs/WHY.md +1 -1
  37. data/docs/WORKFLOWS.md +2 -2
  38. data/docs/_includes/head_custom.html +5 -0
  39. data/docs/_includes/title.html +13 -0
  40. data/docs/_sass/color_schemes/insika.scss +32 -0
  41. data/docs/_sass/custom/custom.scss +199 -0
  42. data/docs/_sass/custom/setup.scss +26 -0
  43. data/docs/assets/img/favicon.svg +7 -0
  44. data/docs/assets/img/insika-mark.svg +7 -0
  45. data/docs/core-concepts.md +21 -0
  46. data/docs/domain.md +115 -0
  47. data/docs/improve.md +20 -0
  48. data/docs/index.md +9 -6
  49. data/docs/integrate.md +20 -0
  50. data/docs/operate.md +13 -6
  51. data/docs/prompts/ADD-TOOL.md +118 -0
  52. data/docs/prompts/DIAGNOSE-TURN.md +65 -0
  53. data/docs/prompts/GO-LIVE.md +138 -0
  54. data/docs/prompts/RUN-EXAMPLES.md +70 -0
  55. data/docs/reference.md +19 -0
  56. data/docs/ship.md +10 -2
  57. data/docs/start-here.md +18 -0
  58. data/lib/insika/agent_profile.rb +202 -10
  59. data/lib/insika/artifact_signing.rb +82 -0
  60. data/lib/insika/artifact_store.rb +160 -0
  61. data/lib/insika/balloon_splitter.rb +102 -0
  62. data/lib/insika/budget_ledger.rb +34 -3
  63. data/lib/insika/cache_series_store.rb +49 -0
  64. data/lib/insika/channel_delivery.rb +119 -14
  65. data/lib/insika/channels/relay.rb +77 -3
  66. data/lib/insika/channels/web.rb +2 -2
  67. data/lib/insika/chat_builder.rb +105 -2
  68. data/lib/insika/checkpoint_store.rb +16 -0
  69. data/lib/insika/commands/agent_payload.rb +3 -3
  70. data/lib/insika/commands/backfill_knowledge.rb +145 -0
  71. data/lib/insika/commands/cancel_followup.rb +49 -0
  72. data/lib/insika/commands/delete_artifact.rb +35 -0
  73. data/lib/insika/commands/delete_concept.rb +34 -0
  74. data/lib/insika/commands/delete_mcp.rb +6 -2
  75. data/lib/insika/commands/delete_tenant_data.rb +107 -0
  76. data/lib/insika/commands/export_customer_memory.rb +48 -0
  77. data/lib/insika/commands/forget_customer.rb +117 -0
  78. data/lib/insika/commands/freeze_funnel_baseline.rb +113 -0
  79. data/lib/insika/commands/gate_harvest.rb +138 -0
  80. data/lib/insika/commands/gate_refinement.rb +1 -1
  81. data/lib/insika/commands/judge_shadow_pairs.rb +124 -0
  82. data/lib/insika/commands/memory_forget_fact.rb +20 -4
  83. data/lib/insika/commands/memory_put_fact.rb +23 -4
  84. data/lib/insika/commands/promote_harvest.rb +130 -0
  85. data/lib/insika/commands/record_outcome.rb +46 -0
  86. data/lib/insika/commands/record_shadow_reply.rb +68 -0
  87. data/lib/insika/commands/refresh_mcp_tools.rb +47 -0
  88. data/lib/insika/commands/reject_harvest.rb +38 -0
  89. data/lib/insika/commands/resolve_proposal.rb +108 -0
  90. data/lib/insika/commands/restore_concept.rb +34 -0
  91. data/lib/insika/commands/revoke_contact.rb +49 -0
  92. data/lib/insika/commands/rollback_harvest.rb +86 -0
  93. data/lib/insika/commands/run_distillation.rb +186 -0
  94. data/lib/insika/commands/run_harvest.rb +393 -0
  95. data/lib/insika/commands/seed_demo_data.rb +31 -0
  96. data/lib/insika/commands/send_message.rb +103 -6
  97. data/lib/insika/commands/session_purge.rb +67 -0
  98. data/lib/insika/commands/upsert_mcp.rb +6 -3
  99. data/lib/insika/commands/write_concept.rb +57 -0
  100. data/lib/insika/contact_store.rb +183 -0
  101. data/lib/insika/context/builder.rb +21 -3
  102. data/lib/insika/context/fragment.rb +7 -3
  103. data/lib/insika/context/priority.rb +5 -0
  104. data/lib/insika/context/provider.rb +17 -3
  105. data/lib/insika/context/providers/briefing.rb +96 -0
  106. data/lib/insika/context/providers/knowledge.rb +108 -0
  107. data/lib/insika/context/providers/memory.rb +16 -7
  108. data/lib/insika/context/providers/prompt.rb +57 -23
  109. data/lib/insika/context/providers/skill.rb +2 -0
  110. data/lib/insika/context/providers/tool_search.rb +2 -0
  111. data/lib/insika/context_trace_store.rb +38 -2
  112. data/lib/insika/cron.rb +189 -0
  113. data/lib/insika/demo/agent_attrs.rb +43 -0
  114. data/lib/insika/demo/golden_cases.rb +81 -0
  115. data/lib/insika/demo/seeder.rb +336 -0
  116. data/lib/insika/distill.rb +224 -0
  117. data/lib/insika/distill_engine.rb +169 -0
  118. data/lib/insika/doctor.rb +889 -11
  119. data/lib/insika/dsl/definition.rb +3 -2
  120. data/lib/insika/dsl/runtime.rb +69 -82
  121. data/lib/insika/dsl/server_boot.rb +92 -1
  122. data/lib/insika/dsl/system.rb +10 -2
  123. data/lib/insika/dsl.rb +186 -5
  124. data/lib/insika/edge_limiter.rb +28 -7
  125. data/lib/insika/env_schema.rb +25 -3
  126. data/lib/insika/errors.rb +11 -0
  127. data/lib/insika/evals/assertions.rb +3 -2
  128. data/lib/insika/evals/golden.rb +41 -4
  129. data/lib/insika/evals/judge.rb +47 -2
  130. data/lib/insika/evals/pairwise.rb +29 -6
  131. data/lib/insika/evals/persona.rb +98 -0
  132. data/lib/insika/evals/runner.rb +9 -0
  133. data/lib/insika/evals/simulator.rb +225 -0
  134. data/lib/insika/evals/transport.rb +83 -1
  135. data/lib/insika/event_stream.rb +10 -0
  136. data/lib/insika/evidence.rb +183 -0
  137. data/lib/insika/executor.rb +869 -76
  138. data/lib/insika/followup_engine.rb +207 -0
  139. data/lib/insika/followup_policy.rb +198 -0
  140. data/lib/insika/followup_store.rb +306 -0
  141. data/lib/insika/funnel_declaration.rb +106 -0
  142. data/lib/insika/funnel_fold.rb +179 -0
  143. data/lib/insika/funnel_store.rb +163 -0
  144. data/lib/insika/golden_store.rb +17 -2
  145. data/lib/insika/grounding/matcher.rb +69 -0
  146. data/lib/insika/grounding.rb +44 -0
  147. data/lib/insika/harvest/conversion_gate.rb +159 -0
  148. data/lib/insika/harvest/criterion.rb +98 -0
  149. data/lib/insika/harvest/gate.rb +194 -0
  150. data/lib/insika/harvest/negative_list.rb +199 -0
  151. data/lib/insika/harvest.rb +241 -0
  152. data/lib/insika/harvest_engine.rb +193 -0
  153. data/lib/insika/harvest_store.rb +548 -0
  154. data/lib/insika/knowledge.rb +680 -0
  155. data/lib/insika/knowledge_store.rb +140 -0
  156. data/lib/insika/mcp_client.rb +94 -0
  157. data/lib/insika/mcp_json.rb +74 -0
  158. data/lib/insika/mcp_live_tool.rb +43 -0
  159. data/lib/insika/mcp_store.rb +98 -26
  160. data/lib/insika/mcp_tool_ingestor.rb +30 -8
  161. data/lib/insika/mcp_tool_registry.rb +100 -0
  162. data/lib/insika/media.rb +382 -0
  163. data/lib/insika/memory_audit_store.rb +85 -0
  164. data/lib/insika/memory_store.rb +264 -23
  165. data/lib/insika/message_origin.rb +7 -2
  166. data/lib/insika/middleware.rb +9 -0
  167. data/lib/insika/model_visible.rb +87 -0
  168. data/lib/insika/model_visible_trace_store.rb +66 -0
  169. data/lib/insika/onboarding.rb +22 -1
  170. data/lib/insika/outbox_store.rb +42 -4
  171. data/lib/insika/outcome_store.rb +147 -0
  172. data/lib/insika/overlay_tool_registry.rb +37 -17
  173. data/lib/insika/packaging.rb +163 -0
  174. data/lib/insika/parity/criterion.rb +79 -0
  175. data/lib/insika/parity/verdict.rb +318 -0
  176. data/lib/insika/prefix_fingerprint.rb +58 -0
  177. data/lib/insika/profile_source.rb +29 -1
  178. data/lib/insika/prompt_catalog.rb +10 -0
  179. data/lib/insika/proposal_store.rb +271 -0
  180. data/lib/insika/queue_policy.rb +4 -1
  181. data/lib/insika/refinement/proposer.rb +1 -1
  182. data/lib/insika/reliability.rb +32 -6
  183. data/lib/insika/retention.rb +316 -0
  184. data/lib/insika/router/app.rb +157 -0
  185. data/lib/insika/router/backend_pool.rb +98 -0
  186. data/lib/insika/router/hash_ring.rb +55 -0
  187. data/lib/insika/router/proxy_body.rb +34 -0
  188. data/lib/insika/router/session_key.rb +54 -0
  189. data/lib/insika/router.rb +18 -0
  190. data/lib/insika/routing.rb +101 -0
  191. data/lib/insika/safety/config.rb +44 -4
  192. data/lib/insika/safety/corpus.rb +255 -0
  193. data/lib/insika/safety/detectors.rb +32 -113
  194. data/lib/insika/safety/factory.rb +15 -2
  195. data/lib/insika/safety/grounding_enforcer.rb +59 -0
  196. data/lib/insika/safety/grounding_validator.rb +49 -0
  197. data/lib/insika/safety/input_guardrail.rb +1 -1
  198. data/lib/insika/safety/moderator.rb +1 -1
  199. data/lib/insika/safety/output_filter.rb +9 -5
  200. data/lib/insika/safety/output_validator.rb +11 -5
  201. data/lib/insika/schedule.rb +177 -0
  202. data/lib/insika/schedule_engine.rb +314 -0
  203. data/lib/insika/schedule_store.rb +208 -0
  204. data/lib/insika/schema_guard.rb +35 -0
  205. data/lib/insika/server/app.rb +247 -17
  206. data/lib/insika/server/rack_app.rb +21 -1
  207. data/lib/insika/server/responses.rb +40 -1
  208. data/lib/insika/session_actor.rb +8 -4
  209. data/lib/insika/session_store.rb +65 -2
  210. data/lib/insika/settings_store.rb +10 -0
  211. data/lib/insika/shadow_pair_store.rb +258 -0
  212. data/lib/insika/skill_catalog.rb +16 -0
  213. data/lib/insika/soak/envelope.rb +140 -0
  214. data/lib/insika/soak/report.rb +392 -0
  215. data/lib/insika/soak/runner.rb +554 -0
  216. data/lib/insika/steer_injector.rb +21 -10
  217. data/lib/insika/store.rb +10 -1
  218. data/lib/insika/stores/memory.rb +6 -0
  219. data/lib/insika/stores/sqlite.rb +8 -0
  220. data/lib/insika/studio/app.rb +1437 -54
  221. data/lib/insika/studio/assets/dist/application.css +1 -1
  222. data/lib/insika/studio/assets/dist/application.js +23 -23
  223. data/lib/insika/studio/assets/dist/favicon.svg +6 -0
  224. data/lib/insika/studio/forms.rb +308 -10
  225. data/lib/insika/studio/nav_icons.rb +19 -1
  226. data/lib/insika/studio/views/_agent_tab_cache.erb +25 -0
  227. data/lib/insika/studio/views/_agent_tab_config.erb +514 -0
  228. data/lib/insika/studio/views/_agent_tab_history.erb +24 -0
  229. data/lib/insika/studio/views/_agent_tab_loops.erb +54 -0
  230. data/lib/insika/studio/views/_agent_tab_memory.erb +51 -0
  231. data/lib/insika/studio/views/_agent_tab_outcomes.erb +31 -0
  232. data/lib/insika/studio/views/_agent_tab_prompts.erb +108 -0
  233. data/lib/insika/studio/views/_agent_tab_skills.erb +38 -0
  234. data/lib/insika/studio/views/_agents_master.erb +44 -0
  235. data/lib/insika/studio/views/_message.erb +49 -32
  236. data/lib/insika/studio/views/agent_detail.erb +62 -278
  237. data/lib/insika/studio/views/agents.erb +71 -54
  238. data/lib/insika/studio/views/approvals.erb +4 -1
  239. data/lib/insika/studio/views/artifact.erb +23 -0
  240. data/lib/insika/studio/views/artifacts.erb +59 -0
  241. data/lib/insika/studio/views/chats.erb +4 -1
  242. data/lib/insika/studio/views/customer.erb +94 -0
  243. data/lib/insika/studio/views/customers.erb +32 -0
  244. data/lib/insika/studio/views/evals.erb +6 -3
  245. data/lib/insika/studio/views/facts.erb +133 -0
  246. data/lib/insika/studio/views/followups.erb +125 -0
  247. data/lib/insika/studio/views/funnel.erb +106 -0
  248. data/lib/insika/studio/views/harvest.erb +234 -0
  249. data/lib/insika/studio/views/home.erb +106 -66
  250. data/lib/insika/studio/views/knowledge.erb +123 -0
  251. data/lib/insika/studio/views/layout.erb +15 -11
  252. data/lib/insika/studio/views/mcp.erb +174 -80
  253. data/lib/insika/studio/views/parity.erb +147 -0
  254. data/lib/insika/studio/views/playground.erb +7 -1
  255. data/lib/insika/studio/views/session.erb +233 -124
  256. data/lib/insika/studio/views/settings.erb +41 -1
  257. data/lib/insika/studio/views/skills.erb +1 -2
  258. data/lib/insika/studio/views/system_files.erb +1 -1
  259. data/lib/insika/studio/views/task.erb +13 -0
  260. data/lib/insika/studio/views/tasks.erb +4 -1
  261. data/lib/insika/studio/views/tools.erb +24 -10
  262. data/lib/insika/task_store.rb +21 -1
  263. data/lib/insika/templates/browser-agent/README.md +36 -0
  264. data/lib/insika/templates/browser-agent/agent.rb +49 -0
  265. data/lib/insika/templates/daily-digest/README.md +38 -0
  266. data/lib/insika/templates/daily-digest/agent.rb +77 -0
  267. data/lib/insika/templates/repo-explorer/README.md +36 -0
  268. data/lib/insika/templates/repo-explorer/agent.rb +45 -0
  269. data/lib/insika/templates/research-analyst/README.md +26 -0
  270. data/lib/insika/templates/research-analyst/agent.rb +58 -0
  271. data/lib/insika/templates/review-panel/README.md +20 -0
  272. data/lib/insika/templates/review-panel/agent.rb +50 -0
  273. data/lib/insika/templates/travel-planner/README.md +35 -0
  274. data/lib/insika/templates/travel-planner/agent.rb +87 -0
  275. data/lib/insika/templates.rb +112 -0
  276. data/lib/insika/testing/store_contract.rb +27 -6
  277. data/lib/insika/tick.rb +47 -11
  278. data/lib/insika/timezone.rb +45 -0
  279. data/lib/insika/tool_definition.rb +17 -7
  280. data/lib/insika/tool_envelope.rb +69 -0
  281. data/lib/insika/tool_manifest.rb +5 -1
  282. data/lib/insika/tools/data_defined_tool.rb +10 -0
  283. data/lib/insika/tools/generate_image.rb +89 -0
  284. data/lib/insika/tools/load_knowledge.rb +74 -0
  285. data/lib/insika/tools/run_persona_eval.rb +328 -0
  286. data/lib/insika/tools/save_artifact.rb +95 -0
  287. data/lib/insika/tools/schedule_followup.rb +164 -0
  288. data/lib/insika/tools/tts.rb +47 -0
  289. data/lib/insika/tools/update_briefing.rb +126 -0
  290. data/lib/insika/turn_output.rb +1 -1
  291. data/lib/insika/turn_state.rb +49 -1
  292. data/lib/insika/turn_timing.rb +22 -2
  293. data/lib/insika/version.rb +1 -1
  294. data/lib/insika/vitals.rb +84 -0
  295. data/lib/insika/wiring/graph.rb +472 -13
  296. data/lib/insika/wiring/graph_chat.rb +102 -0
  297. data/lib/insika.rb +149 -3
  298. metadata +181 -5
  299. data/docs/build.md +0 -14
  300. data/docs/understand.md +0 -10
@@ -0,0 +1,98 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Insika
4
+ module Evals
5
+ # A SIMULATED CUSTOMER — the data that turns a scripted case into a
6
+ # generated conversation. Pure data: goal, style, the opening
7
+ # message, the ONLY facts the persona may assert, and a hard turn cap. The
8
+ # persona is played by a model (the cheap utility_model) with this as its whole
9
+ # instruction; the anti-invention rule below is the soul of the feature — a
10
+ # simulator that invents an order number produces a conversation the agent
11
+ # could never have had, and a case that tests nothing.
12
+ Persona = Struct.new(:goal, :style, :opens_with, :knows, :max_turns, keyword_init: true) do
13
+ def to_h
14
+ { "goal" => goal, "style" => style, "opens_with" => opens_with,
15
+ "knows" => knows, "max_turns" => max_turns }.compact
16
+ end
17
+
18
+ # The persona as a whole instruction. The `knows` facts are the ONLY
19
+ # assertions the persona may make; anything else is answered with
20
+ # ignorance — "não sei", "não tenho isso aqui" — exactly like a real
21
+ # customer who does not have the fact. `transcript` is the conversation
22
+ # so far, in order, as [{ role: "user"|"assistant", text: }].
23
+ def prompt(transcript)
24
+ facts = knows.map { |k, v| "- #{k}: #{v}" }.join("\n")
25
+ <<~PROMPT
26
+ You are simulating a customer in a chat with a store's virtual assistant.
27
+ Stay in character. You are not helping the assistant; you are the person
28
+ it serves.
29
+
30
+ GOAL: #{goal}
31
+ STYLE: #{style || "short, natural messages; answers what is asked"}
32
+
33
+ FACTS YOU KNOW — the ONLY facts you may assert:
34
+ #{facts}
35
+
36
+ RULES:
37
+ 1. You may ONLY assert the facts above. Asked about anything else, you do
38
+ not know it — answer with ignorance, like a real customer without that
39
+ fact (you have no order number, no name, no date, no price beyond the
40
+ facts above). Never invent an order number, a name, a date, a price or
41
+ any other detail that is not in FACTS YOU KNOW.
42
+ 2. Reply with ONLY the customer's next message.
43
+ 3. When your goal has been met, end the message with the marker
44
+ <<goal_met>>. When you give up (the assistant cannot get you there),
45
+ end the message with the marker <<gave_up>>. Otherwise end with no
46
+ marker.
47
+
48
+ CONVERSATION SO FAR:
49
+ #{transcript.map { |m| "#{m[:role]}: #{m[:text]}" }.join("\n")}
50
+
51
+ Your next message:
52
+ PROMPT
53
+ end
54
+ end
55
+
56
+ # Loads + validates a persona mapping (the `persona:` key of a golden, or the
57
+ # `--persona` file of the simulate CLI). Fails LOUD on a malformed persona —
58
+ # a silently relaxed max_turns or a missing knows would produce a simulation
59
+ # that tests nothing.
60
+ module PersonaLoader
61
+ class InvalidPersona < StandardError; end
62
+
63
+ module_function
64
+
65
+ def build(raw, source: "(inline)")
66
+ raise InvalidPersona, "#{source}: persona must be a mapping" unless raw.is_a?(Hash)
67
+
68
+ goal = presence(raw["goal"])
69
+ raise InvalidPersona, "#{source}: persona needs a non-empty 'goal'" if goal.nil?
70
+
71
+ knows = raw["knows"]
72
+ unless knows.is_a?(Hash) && !knows.empty?
73
+ raise InvalidPersona, "#{source}: persona needs a non-empty 'knows' mapping (the only facts it may assert)"
74
+ end
75
+
76
+ opens = presence(raw["opens_with"])
77
+ raise InvalidPersona, "#{source}: persona needs a non-empty 'opens_with'" if opens.nil?
78
+
79
+ max = raw["max_turns"]
80
+ unless max.is_a?(Integer) && max.positive?
81
+ raise InvalidPersona, "#{source}: persona needs 'max_turns' as a positive integer"
82
+ end
83
+
84
+ Persona.new(
85
+ goal: goal, style: presence(raw["style"]),
86
+ opens_with: opens,
87
+ knows: knows.transform_keys(&:to_s).transform_values(&:to_s),
88
+ max_turns: max
89
+ )
90
+ end
91
+
92
+ def presence(v)
93
+ s = v.to_s.strip
94
+ s.empty? ? nil : s
95
+ end
96
+ end
97
+ end
98
+ end
@@ -50,6 +50,15 @@ module Insika
50
50
  end
51
51
 
52
52
  def run_case(golden)
53
+ # A persona case is GENERATED, not replayed: the turns do not exist until a
54
+ # Simulator drives the conversation. The replay Runner cannot run it, and a
55
+ # silent no-op would read as a pass — so it is SKIPPED with the reason, and
56
+ # the Simulator (the simulate CLI) is the only driver.
57
+ if golden.simulated?
58
+ return RunCase.new(result: Assertions.skip(golden, "simulated case (persona) — drive it with `insika evals:simulate`"),
59
+ timings: [])
60
+ end
61
+
53
62
  skip = skip_reason(golden)
54
63
  return RunCase.new(result: Assertions.skip(golden, skip), timings: []) if skip
55
64
 
@@ -0,0 +1,225 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Insika
4
+ module Evals
5
+ # SIMULATED USERS. Two models talking: the target agent
6
+ # (reached through the same Transport seam as the Runner) and the simulated
7
+ # customer (a cheap model — the platform utility_model — playing a `Persona`).
8
+ #
9
+ # Pure over its seams, like the Runner: the Transport is injected (HttpTransport
10
+ # for a remote deployment, GraphTransport for the own graph, A2ATransport for an
11
+ # agent that only speaks A2A) and the persona model is an injected `ask`
12
+ # (prompt -> raw text), so the loop is unit-testable offline.
13
+ #
14
+ # Termination is recorded, never guessed: the persona's `max_turns`, the persona
15
+ # emitting `<<goal_met>>` (its goal is served) or `<<gave_up>>` (it abandons),
16
+ # or an errored agent turn. "Gave up at turn 3" is a finding, not a detail.
17
+ #
18
+ # SAFETY (rule fixed in the spec): a simulated conversation must not write for
19
+ # real. The `Safety` gate refuses the run unless the target is declared
20
+ # `staging` or the run uses an eval profile where the agent's side-effect tools
21
+ # are swapped for fakes — and the swap list is DERIVED from the tool registry
22
+ # (the engine marks `side_effect` on tools; see `EvalProfile`), never
23
+ # hand-maintained. Every run is `simulated: true`, so a report never mixes a
24
+ # generated conversation with real traffic.
25
+ class Simulator
26
+ STOP_GOAL_MET = "<<goal_met>>"
27
+ STOP_GAVE_UP = "<<gave_up>>"
28
+ STOPS = { STOP_GOAL_MET => :goal_met, STOP_GAVE_UP => :gave_up }.freeze
29
+
30
+ # A generated conversation. `transcript` is [{ role: "user"|"assistant",
31
+ # text:, tools: [names] }] in order; `stop` is one of :goal_met | :gave_up |
32
+ # :max_turns | :error; `simulated` is ALWAYS true — the flag that keeps a
33
+ # report from mixing generated traffic with real conversations (rule D).
34
+ SimulatedRun = Struct.new(:transcript, :stop, :turns, :error, keyword_init: true) do
35
+ def simulated = true
36
+ def simulated? = true
37
+
38
+ def to_h
39
+ { "simulated" => true, "stop" => stop.to_s, "turns" => turns, "error" => error,
40
+ "transcript" => transcript }.compact
41
+ end
42
+ end
43
+
44
+ # The target-safety gate. A run is allowed when:
45
+ # · `staging` — the operator declares the target is a staging
46
+ # deployment (real tools, staging data), or
47
+ # · the derived `side_effect_tools` is EMPTY — the agent has nothing
48
+ # that can write, or
49
+ # · `eval_profile` AND `swapped_tools` covers EVERY derived
50
+ # side-effect tool — the eval profile swaps them all for dry-runs.
51
+ # Anything else is refused with the offending tool names. Refusals are loud:
52
+ # `UnsafeTarget` is raised BEFORE a single model call.
53
+ #
54
+ # `side_effect_tools` is the DERIVED list (see EvalProfile) — never a
55
+ # hand-typed claim; `swapped_tools` is what the run's eval profile declares
56
+ # it swaps. A bare `eval_profile: true` with a known side-effect list is
57
+ # REFUSED: an eval profile that leaves a write-capable tool unswapped is a
58
+ # trust-me flag wearing a safety's clothes.
59
+ Safety = Struct.new(:staging, :eval_profile, :side_effect_tools, :swapped_tools, keyword_init: true) do
60
+ def self.staging = new(staging: true)
61
+
62
+ # -> reason to refuse, or nil. `side_effect_tools` is the DERIVED list (the
63
+ # target's reachable tools the registry marks `side_effect`).
64
+ def refusal
65
+ return nil if staging
66
+
67
+ tools = Array(side_effect_tools).map(&:to_s).reject(&:empty?)
68
+ return nil if tools.empty?
69
+
70
+ if eval_profile
71
+ swapped = Array(swapped_tools).map(&:to_s)
72
+ uncovered = tools - swapped
73
+ return nil if uncovered.empty?
74
+
75
+ return "eval profile declares swapped tool(s) (#{swapped.join(', ')}) but the target " \
76
+ "also exposes side-effect tool(s) (#{uncovered.join(', ')}) — an eval profile " \
77
+ "must swap EVERY side-effect tool"
78
+ end
79
+
80
+ "the target agent exposes side-effect tool(s) (#{tools.join(', ')}) — a simulated " \
81
+ "conversation could write for real. Run against staging (--staging) or against an " \
82
+ "eval profile where these tools are swapped for fakes (--eval-profile --eval-tools ...)."
83
+ end
84
+ end
85
+
86
+ # Raised before any model call when the Safety gate refuses.
87
+ class UnsafeTarget < StandardError; end
88
+
89
+ # transport: the target agent seam (TurnOutcome per turn, like the Runner).
90
+ # ask: ->(prompt) { text } — the simulated customer (the cheap model).
91
+ # safety: a Safety — the gate evaluated before every run.
92
+ def initialize(transport:, ask:, safety:)
93
+ @transport = transport
94
+ @ask = ask
95
+ @safety = safety
96
+ end
97
+
98
+ # Runs one simulated conversation. -> SimulatedRun.
99
+ def run(persona:, agent:, conv:)
100
+ reason = @safety.refusal
101
+ raise UnsafeTarget, reason if reason
102
+
103
+ transcript = []
104
+ message = persona.opens_with
105
+ stop = :max_turns
106
+
107
+ 1.upto(persona.max_turns) do |turn|
108
+ transcript << { role: "user", text: message }
109
+ outcome = @transport.turn(agent: agent, conv: conv, message: message)
110
+ if outcome.result.error
111
+ transcript << { role: "assistant", text: "", tools: [] }
112
+ return SimulatedRun.new(transcript: transcript, stop: :error, turns: turn,
113
+ error: outcome.result.error)
114
+ end
115
+ transcript << { role: "assistant", text: outcome.result.output_text,
116
+ tools: outcome.result.tool_names }
117
+ break if turn == persona.max_turns # the persona's budget is spent
118
+
119
+ reply = @ask.call(persona.prompt(transcript)).to_s
120
+ marker, text = strip_stop(reply)
121
+ if marker
122
+ transcript << { role: "user", text: text } unless text.empty?
123
+ return SimulatedRun.new(transcript: transcript, stop: marker, turns: turn)
124
+ end
125
+ if text.empty?
126
+ return SimulatedRun.new(transcript: transcript, stop: :error, turns: turn,
127
+ error: "the persona produced an empty message")
128
+ end
129
+
130
+ message = text
131
+ end
132
+
133
+ SimulatedRun.new(transcript: transcript, stop: stop, turns: persona.max_turns)
134
+ end
135
+
136
+ private
137
+
138
+ # -> [stop_reason | nil, text_without_marker]. A trailing stop marker ends
139
+ # the conversation; the text before it is the customer's final message.
140
+ def strip_stop(reply)
141
+ text = reply.strip
142
+ STOPS.each do |marker, reason|
143
+ return [reason, text.delete_suffix(marker).strip] if text.end_with?(marker)
144
+ end
145
+ [nil, text]
146
+ end
147
+ end
148
+
149
+ # The DERIVED eval profile: which of an agent's reachable tools can write for
150
+ # real, computed from the tool registry — the engine marks `side_effect` on
151
+ # tools (a data-tool's non-GET method, the MCP ingestor's `tools/call`), so the
152
+ # swap list is a fact of the deployment, never a hand-maintained list.
153
+ module EvalProfile
154
+ module_function
155
+
156
+ # profile + registry (answers #names and #side_effect?) ->
157
+ # [tool names] the agent can reach that are marked side-effect, sorted.
158
+ def side_effect_tools(profile, registry)
159
+ allowed = if profile.tools_allow.nil? then Array(registry.names)
160
+ else Array(profile.tools_allow).map(&:to_s)
161
+ end
162
+ denied = Array(profile.tools_deny).map(&:to_s)
163
+ (allowed - denied).select { |name| registry.side_effect?(name) }.sort
164
+ end
165
+
166
+ # -> bool: can a simulated run touch this agent without a staging
167
+ # declaration (no reachable side-effect tool)?
168
+ def safe?(profile, registry)
169
+ side_effect_tools(profile, registry).empty?
170
+ end
171
+
172
+ # A registry overlay that answers the swapped names with a DRY-RUN tool and
173
+ # delegates everything else to the base — the "side_effect -> fake" half of
174
+ # the eval profile, derived from the base registry (nothing hand-maintained).
175
+ # The overlay is a drop-in for the Executor's registry: same
176
+ # entries/resolve/side_effect? surface.
177
+ def registry(base, side_effect_tools:, dry_run: nil)
178
+ swapped = Array(side_effect_tools).map(&:to_s)
179
+ fake = dry_run || ->(name) { Simulator::DryRunTool.new(name) }
180
+ OverlayRegistry.new(base: base, swapped: swapped, fake: fake)
181
+ end
182
+ end
183
+
184
+ # The overlay behind `EvalProfile.registry`. Kept as a named class so the
185
+ # constant is assigned once at load time, not inside the method.
186
+ class EvalProfile::OverlayRegistry
187
+ def initialize(base:, swapped:, fake:)
188
+ @base = base
189
+ @swapped = swapped
190
+ @fake = fake
191
+ end
192
+
193
+ def names = @base.names
194
+ def entries = @base.entries
195
+ def side_effect?(name) = @swapped.include?(name.to_s) ? false : @base.side_effect?(name)
196
+
197
+ def resolve(name)
198
+ key = name.to_s
199
+ return @fake.call(key) if @swapped.include?(key)
200
+
201
+ @base.resolve(key)
202
+ end
203
+ end
204
+ end
205
+ end
206
+
207
+ # The dry-run fake lives with the Simulator (it is its "side_effect -> fake"
208
+ # convention). It answers the same surface as a RubyLLM tool (`call`) and never
209
+ # performs the real side effect. Its result carries `dry_run: true` so a reader
210
+ # can tell a swapped call from a real one in the transcript's tool trace.
211
+ class Insika::Evals::Simulator::DryRunTool
212
+ def initialize(name, description: nil)
213
+ @name = name.to_s
214
+ @description = description ||
215
+ "DRY-RUN of #{@name} — disabled for this simulated run, returns a canned envelope"
216
+ end
217
+
218
+ def name = @name
219
+ def description = @description
220
+
221
+ def call(args)
222
+ { "dry_run" => true, "tool" => @name, "simulated" => true,
223
+ "note" => "side-effect tool disabled by the eval profile — the real call was NOT performed" }
224
+ end
225
+ end
@@ -108,7 +108,8 @@ module Insika
108
108
  return nil unless res.code.to_i == 200
109
109
 
110
110
  body = JSON.parse(res.body)
111
- { "tools" => body["tools"], "capabilities" => Array(body["capabilities"]) }
111
+ { "tools" => body["tools"], "capabilities" => Array(body["capabilities"]),
112
+ "side_effect_tools" => body["side_effect_tools"] }
112
113
  rescue StandardError, JSON::ParserError
113
114
  nil
114
115
  end
@@ -174,5 +175,86 @@ module Insika
174
175
  ttfb: nil, total: (mono - t0) * 1000.0)
175
176
  end
176
177
  end
178
+
179
+ # A transport over the deployment's OWN graph, in-process — no HTTP. This is
180
+ # how a simulated conversation (or a replay) exercises the local agent the
181
+ # way a customer would reach it, tools and guardrails included, without a
182
+ # server in between. `runtime` is anything answering the DSL Runtime contract
183
+ # (#chat(message, session_id:, agent:) -> text, raising on failure) — the DSL
184
+ # Definition/System runtime, or a test double.
185
+ #
186
+ # Tool activity is captured from the graph's event stream (the `:tool_call`
187
+ # events the ChatBuilder emits), so an in-process transcript records the same
188
+ # tool names an HTTP replay would — the Simulator's transcript is not blind to
189
+ # what the local agent called. `event_stream` is optional; when omitted it is
190
+ # read off the runtime's graph when one is reachable.
191
+ class GraphTransport
192
+ def initialize(runtime:, event_stream: nil)
193
+ @runtime = runtime
194
+ @event_stream = event_stream ||
195
+ (runtime.graph.event_stream if runtime.respond_to?(:graph) && runtime.graph)
196
+ end
197
+
198
+ def mono = Process.clock_gettime(Process::CLOCK_MONOTONIC)
199
+
200
+ def turn(agent:, conv:, message:)
201
+ t0 = mono
202
+ sub = @event_stream&.subscribe(types: [:tool_call])
203
+ begin
204
+ text = @runtime.chat(message, session_id: conv, agent: agent)
205
+ TurnOutcome.new(
206
+ result: TurnResult.new(output_text: text.to_s, tool_calls: drain_tools(sub), error: nil),
207
+ ttfb: nil, total: (mono - t0) * 1000.0, usage: nil
208
+ )
209
+ rescue Insika::Error => e
210
+ TurnOutcome.new(
211
+ result: TurnResult.new(output_text: "", tool_calls: drain_tools(sub), error: e.message),
212
+ ttfb: nil, total: (mono - t0) * 1000.0, usage: nil
213
+ )
214
+ ensure
215
+ sub&.close
216
+ end
217
+ end
218
+
219
+ private
220
+
221
+ # The same shape an HTTP replay produces: [{ "name" =>, "status" => nil }]
222
+ # (the stream carries no per-tool status — that lives in the trace store).
223
+ def drain_tools(sub)
224
+ return [] if sub.nil?
225
+
226
+ sub.drain_nonblocking.map { |ev| { "name" => ev.data[:name].to_s, "status" => nil } }
227
+ end
228
+ end
229
+
230
+ # Thin A2A transport: drives a REMOTE A2A agent (an agent that only speaks
231
+ # A2A) through the same `turn` seam the Simulator uses. The outbound A2A
232
+ # client already does the send+poll; this is the wrapper that makes it a
233
+ # Transport. `client` answers `#call(url, text, context_id:)` -> {text:} or
234
+ # {error:} — the `Server::A2A::Client` shape.
235
+ class A2ATransport
236
+ def initialize(client:, url:)
237
+ @client = client
238
+ @url = url
239
+ end
240
+
241
+ def mono = Process.clock_gettime(Process::CLOCK_MONOTONIC)
242
+
243
+ def turn(agent:, conv:, message:)
244
+ t0 = mono
245
+ result = @client.call(@url, message.to_s, context_id: conv)
246
+ if result[:error]
247
+ TurnOutcome.new(
248
+ result: TurnResult.new(output_text: "", tool_calls: [], error: result[:error].to_s),
249
+ ttfb: nil, total: (mono - t0) * 1000.0, usage: nil
250
+ )
251
+ else
252
+ TurnOutcome.new(
253
+ result: TurnResult.new(output_text: result[:text].to_s, tool_calls: [], error: nil),
254
+ ttfb: nil, total: (mono - t0) * 1000.0, usage: nil
255
+ )
256
+ end
257
+ end
258
+ end
177
259
  end
178
260
  end
@@ -85,6 +85,16 @@ module Insika
85
85
  end
86
86
  end
87
87
 
88
+ # Drains whatever is ALREADY queued without ever blocking. Safe in a
89
+ # cooperative reactor: between the `empty?` check and the `dequeue` no other
90
+ # fiber runs, so a non-empty dequeue never waits. The eval transports use it
91
+ # to collect the events a turn already emitted, AFTER the turn returned.
92
+ def drain_nonblocking
93
+ drained = []
94
+ drained << @queue.dequeue until @queue.empty?
95
+ drained
96
+ end
97
+
88
98
  # Idempotent: a second CLOSED is harmless (the `each` stops at the first).
89
99
  # `@on_close` fires only once (avoids removing the subscription twice).
90
100
  def close
@@ -0,0 +1,183 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+
5
+ module Insika
6
+ # — the evidence contract (engine half).
7
+ #
8
+ # A tool DECLARES `evidence` in its manifest; the engine then does BOTH jobs
9
+ # from the same declaration: strips the result down to `{items: [{id, line}]}`
10
+ # for the model (the lean envelope), and appends every `id` to the session
11
+ # evidence ledger. No second flag, no "lean but not evidence" mode — a
12
+ # half-configuration cannot exist, which is what keeps "no claim without a
13
+ # tool ID" a tautology at the envelope instead of a convention.
14
+ #
15
+ # Everything here is pure Ruby, no IO: the ToolEnvelope calls it after the real
16
+ # tool returns. It does NOT write anything — the ledger write is the envelope's,
17
+ # via the state.
18
+ module Evidence
19
+ # The declaration (D1/C2.1). Parses three authorable shapes:
20
+ #
21
+ # { "evidence": "products" } // bare kind
22
+ # { "evidence": { "kind": "products" } } // full form
23
+ # { "evidence": { "kind": "products",
24
+ # "items": "results",
25
+ # "attachments": "cards" } } // non-default paths
26
+ #
27
+ # `kind` is the PILOT PACK's value, never a gem constant (the
28
+ # engine owns nothing about product shape).
29
+ Spec = Data.define(:kind, :items_path, :attachments_path) do
30
+ PATH_RE = /\A[a-zA-Z0-9_]+(?:\.[a-zA-Z0-9_]+)*\z/
31
+
32
+ # String | Hash | nil -> Spec | nil. Raises ValidationError on a blank kind
33
+ # or an empty/ill-formed path. All at ingestion, never at the turn.
34
+ def self.parse(raw)
35
+ return nil if raw.nil? || raw == false
36
+
37
+ h = raw.is_a?(String) ? { "kind" => raw } : Coercion.deep_stringify(raw)
38
+ h = h.is_a?(Hash) ? h : {}
39
+ kind = Coercion.presence(h["kind"])
40
+ raise Insika::ValidationError, "evidence.kind is required" if kind.nil?
41
+
42
+ items = presence_path(h["items"], "items")
43
+ attachments = presence_path(h["attachments"], "attachments")
44
+ new(kind: kind, items_path: items, attachments_path: attachments)
45
+ end
46
+
47
+ def to_h
48
+ { "kind" => kind, "items" => items_path, "attachments" => attachments_path }.compact
49
+ end
50
+
51
+ def self.presence_path(value, default)
52
+ s = Coercion.presence(value)
53
+ s = default if s.nil?
54
+ raise Insika::ValidationError, "evidence.#{default}: not a dotted path" unless PATH_RE.match?(s)
55
+
56
+ s
57
+ end
58
+ private_class_method :presence_path
59
+ end
60
+
61
+ # The lean result the model sees — the ONLY thing that survives the envelope.
62
+ MAX_ITEMS = 16
63
+ # Line truncation keeps the transcript lean by force (E1).
64
+ LINE_MAX = 200
65
+ # Attachments are a channel nicety, never the answer; bounded on purpose.
66
+ MAX_ATTACHMENTS = 16
67
+ URL_MAX = 500
68
+
69
+ # The attachments contract, validated for the outbox (channel side, never the
70
+ # model): [{ "type" => "card"|"image", "url" => String, "caption" => String|nil }].
71
+ # Entries without a String url, or beyond MAX_ATTACHMENTS, are DROPPED — never
72
+ # a turn failure (the card is a channel nicety, not the answer).
73
+ def self.valid_attachments(list)
74
+ Array(list).filter_map do |entry|
75
+ next unless entry.is_a?(Hash)
76
+
77
+ url = (entry["url"] || entry[:url]).to_s
78
+ next if url.empty?
79
+
80
+ caption = Coercion.presence(entry["caption"] || entry[:caption])
81
+ { "type" => (entry["type"] || entry[:type]).to_s,
82
+ "url" => url[0, URL_MAX],
83
+ "caption" => caption }
84
+ end.first(MAX_ATTACHMENTS)
85
+ end
86
+
87
+ # Result shaping, stateless — callable from any tool-call fiber (the parallel
88
+ # tool_concurrency path). No shared state in this class.
89
+ class Processor
90
+ class << self
91
+ # -> the parsed Hash the evidence paths dig into. For `evidence_envelope`
92
+ # the raw body lives under `__insika_body` (D3); a code tool returns the
93
+ # object itself. Raises JSON::ParserError on a non-JSON envelope body —
94
+ # the envelope turns that into `{error:}`, nothing recorded (fail closed:
95
+ # no IDs, no claims to make).
96
+ def raw(spec, result)
97
+ return result unless result.is_a?(Hash) && result.key?("__insika_body")
98
+
99
+ JSON.parse(result["__insika_body"].to_s)
100
+ end
101
+
102
+ # -> [lean, attachments]. Assumes the shape already passed
103
+ # SchemaGuard.violation_output. Items are capped and lines truncated; the
104
+ # model must never see a null where the contract says items.
105
+ def build(spec, raw)
106
+ items = SchemaGuard.dig(raw, spec.items_path) || []
107
+ lean_items = items.first(MAX_ITEMS).map do |item|
108
+ { "id" => (item["id"] || item[:id]).to_s,
109
+ "line" => Coercion.utf8((item["line"] || item[:line]).to_s)[0, LINE_MAX] }
110
+ end
111
+ lean = { "items" => lean_items }
112
+ attachments = Insika::Evidence.valid_attachments(SchemaGuard.dig(raw, spec.attachments_path))
113
+ [lean, attachments]
114
+ end
115
+ end
116
+ end
117
+ end
118
+
119
+ # The session evidence ledger (C4). A session-scoped SET with an `ungrounded`
120
+ # counter: records every product id that entered the context via an
121
+ # evidence-declared tool this session , plus the ungrounded
122
+ # counter that feeds the daily metric. It is NOT a policy object — it records
123
+ # and answers `ids` / `ungrounded` / `lines`; it never decides.
124
+ class EvidenceLedger
125
+ # Oldest-evicted cap: a session that outlives it needs a real cap or the row
126
+ # grows forever.
127
+ MAX_IDS = 1_000
128
+
129
+ def initialize(store: nil, session_id: nil)
130
+ @store = store
131
+ @session_id = session_id
132
+ @ids = []
133
+ @ungrounded = 0
134
+ end
135
+
136
+ # The in-memory accumulator (the envelope appends, the validator/enforcer
137
+ # read the union). The PERSISTED list is appended on flush (Executor, stage 8)
138
+ # — the envelope never blocks on the store.
139
+ def record(ids)
140
+ @ids.concat(Array(ids).map(&:to_s).reject(&:empty?))
141
+ self
142
+ end
143
+
144
+ # -> the effective set for THIS turn: persisted session evidence (loaded at
145
+ # build) + the turn's new ids, deduped, capped.
146
+ def ids
147
+ (session_ids + @ids).uniq.last(MAX_IDS)
148
+ end
149
+
150
+ attr_reader :ungrounded
151
+
152
+ def ungrounded_count(claim)
153
+ @ungrounded += 1
154
+ claim
155
+ end
156
+
157
+ # -> self, flushed to the session record. Idempotent. A store OR not-found
158
+ # failure is swallowed (evidence is audit — it must never fail a committed
159
+ # turn; a session purged mid-turn by forget_customer/session_purge reads as
160
+ # "nothing to append", never an explosion).
161
+ def flush!
162
+ return self unless @store && @session_id
163
+
164
+ @store.append_evidence(@session_id, ids: @ids, ungrounded: @ungrounded)
165
+ @ids = []
166
+ @ungrounded = 0
167
+ self
168
+ rescue Insika::Error
169
+ self
170
+ end
171
+
172
+ private
173
+
174
+ def session_ids
175
+ return [] unless @store && @session_id
176
+
177
+ session = @store.find(@session_id)
178
+ Array(session&.evidence&.fetch("ids", [])).map(&:to_s)
179
+ rescue Insika::NotFoundError
180
+ []
181
+ end
182
+ end
183
+ end