insika 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (280) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +199 -5
  3. data/README.md +8 -2
  4. data/bin/insika +231 -13
  5. data/docs/AGENTS.md +505 -6
  6. data/docs/API.md +56 -0
  7. data/docs/CHANNELS.md +100 -10
  8. data/docs/CONTEXT.md +147 -19
  9. data/docs/DEPLOY.md +34 -11
  10. data/docs/EMBEDDING.md +11 -7
  11. data/docs/EVALS.md +20 -1
  12. data/docs/FACTS.md +135 -0
  13. data/docs/HARVEST.md +117 -0
  14. data/docs/LOADTEST.md +17 -10
  15. data/docs/OBSERVABILITY.md +65 -2
  16. data/docs/REFINEMENT.md +9 -9
  17. data/docs/RELEASING.md +34 -7
  18. data/docs/RUNNING-LOCAL.md +4 -4
  19. data/docs/SECURITY.md +85 -11
  20. data/docs/SKILLS.md +189 -3
  21. data/docs/SOAK.md +127 -0
  22. data/docs/TOOLS.md +70 -2
  23. data/docs/WHY.md +1 -1
  24. data/docs/WORKFLOWS.md +2 -2
  25. data/docs/domain.md +115 -0
  26. data/docs/index.md +2 -2
  27. data/docs/onboarding/start.md +1 -1
  28. data/lib/insika/agent_profile.rb +228 -26
  29. data/lib/insika/alert_dispatcher.rb +139 -0
  30. data/lib/insika/balloon_splitter.rb +102 -0
  31. data/lib/insika/baseline_store.rb +2 -2
  32. data/lib/insika/budget_ledger.rb +166 -0
  33. data/lib/insika/cache_series_store.rb +49 -0
  34. data/lib/insika/channel_delivery.rb +132 -24
  35. data/lib/insika/channel_registry.rb +1 -1
  36. data/lib/insika/channels/relay.rb +80 -6
  37. data/lib/insika/channels/web/widget.js +2 -2
  38. data/lib/insika/channels/web.rb +9 -9
  39. data/lib/insika/channels/webhook.rb +58 -0
  40. data/lib/insika/chat_builder.rb +145 -13
  41. data/lib/insika/checkpoint_store.rb +16 -0
  42. data/lib/insika/circuit_state.rb +114 -0
  43. data/lib/insika/coercion.rb +8 -0
  44. data/lib/insika/commands/agent_payload.rb +6 -4
  45. data/lib/insika/commands/cancel_followup.rb +49 -0
  46. data/lib/insika/commands/create_agent.rb +2 -2
  47. data/lib/insika/commands/create_session.rb +1 -1
  48. data/lib/insika/commands/delete_llm_provider.rb +1 -1
  49. data/lib/insika/commands/delete_skill.rb +43 -0
  50. data/lib/insika/commands/delete_tenant_data.rb +95 -0
  51. data/lib/insika/commands/export_customer_memory.rb +48 -0
  52. data/lib/insika/commands/forget_customer.rb +117 -0
  53. data/lib/insika/commands/freeze_funnel_baseline.rb +113 -0
  54. data/lib/insika/commands/gate_harvest.rb +138 -0
  55. data/lib/insika/commands/gate_refinement.rb +12 -12
  56. data/lib/insika/commands/import_mcp_tools.rb +1 -1
  57. data/lib/insika/commands/import_tools.rb +4 -4
  58. data/lib/insika/commands/issue_tenant_token.rb +41 -0
  59. data/lib/insika/commands/judge_shadow_pairs.rb +124 -0
  60. data/lib/insika/commands/memory_forget_fact.rb +20 -4
  61. data/lib/insika/commands/memory_put_fact.rb +23 -4
  62. data/lib/insika/commands/promote_harvest.rb +130 -0
  63. data/lib/insika/commands/record_outcome.rb +46 -0
  64. data/lib/insika/commands/record_shadow_reply.rb +68 -0
  65. data/lib/insika/commands/reject_harvest.rb +38 -0
  66. data/lib/insika/commands/resolve_proposal.rb +108 -0
  67. data/lib/insika/commands/resolve_refinement.rb +1 -1
  68. data/lib/insika/commands/revoke_contact.rb +49 -0
  69. data/lib/insika/commands/revoke_token.rb +39 -0
  70. data/lib/insika/commands/rollback_harvest.rb +86 -0
  71. data/lib/insika/commands/rotate_tenant_token.rb +43 -0
  72. data/lib/insika/commands/run_distillation.rb +186 -0
  73. data/lib/insika/commands/run_harvest.rb +393 -0
  74. data/lib/insika/commands/run_refinement.rb +5 -5
  75. data/lib/insika/commands/send_message.rb +112 -15
  76. data/lib/insika/commands/session_purge.rb +67 -0
  77. data/lib/insika/commands/set_agent_tools.rb +1 -1
  78. data/lib/insika/commands/set_skill_agents.rb +60 -19
  79. data/lib/insika/commands/trigger_workflow.rb +1 -1
  80. data/lib/insika/commands/update_agent.rb +1 -1
  81. data/lib/insika/commands/write_data_tool.rb +1 -1
  82. data/lib/insika/commands/write_golden.rb +1 -1
  83. data/lib/insika/commands/write_skill.rb +19 -9
  84. data/lib/insika/config_store.rb +8 -4
  85. data/lib/insika/contact_store.rb +183 -0
  86. data/lib/insika/context/builder.rb +23 -5
  87. data/lib/insika/context/fragment.rb +31 -3
  88. data/lib/insika/context/priority.rb +6 -2
  89. data/lib/insika/context/provider.rb +17 -3
  90. data/lib/insika/context/providers/briefing.rb +96 -0
  91. data/lib/insika/context/providers/memory.rb +16 -7
  92. data/lib/insika/context/providers/prompt.rb +30 -2
  93. data/lib/insika/context/providers/request.rb +1 -1
  94. data/lib/insika/context/providers/session.rb +17 -2
  95. data/lib/insika/context/providers/skill.rb +7 -1
  96. data/lib/insika/context/providers/skill_trigger.rb +128 -0
  97. data/lib/insika/context/providers/tool_search.rb +2 -0
  98. data/lib/insika/context_trace_store.rb +128 -0
  99. data/lib/insika/delegation_store.rb +2 -2
  100. data/lib/insika/distill.rb +224 -0
  101. data/lib/insika/distill_engine.rb +169 -0
  102. data/lib/insika/doctor.rb +962 -7
  103. data/lib/insika/dsl/runtime.rb +20 -11
  104. data/lib/insika/dsl/server_boot.rb +74 -4
  105. data/lib/insika/dsl/system.rb +1 -1
  106. data/lib/insika/dsl.rb +152 -15
  107. data/lib/insika/edge_limiter.rb +167 -8
  108. data/lib/insika/egress_guard.rb +3 -3
  109. data/lib/insika/env_schema.rb +22 -12
  110. data/lib/insika/errors.rb +72 -5
  111. data/lib/insika/evals/assertions.rb +15 -14
  112. data/lib/insika/evals/baseline.rb +3 -3
  113. data/lib/insika/evals/golden.rb +8 -8
  114. data/lib/insika/evals/judge.rb +7 -7
  115. data/lib/insika/evals/pairwise.rb +21 -9
  116. data/lib/insika/evals/report.rb +2 -2
  117. data/lib/insika/evals/runner.rb +6 -6
  118. data/lib/insika/evals/transport.rb +2 -2
  119. data/lib/insika/event_stream.rb +23 -5
  120. data/lib/insika/evidence.rb +183 -0
  121. data/lib/insika/executor.rb +1092 -160
  122. data/lib/insika/followup_engine.rb +207 -0
  123. data/lib/insika/followup_policy.rb +221 -0
  124. data/lib/insika/followup_store.rb +306 -0
  125. data/lib/insika/frontmatter.rb +1 -1
  126. data/lib/insika/funnel_declaration.rb +106 -0
  127. data/lib/insika/funnel_fold.rb +179 -0
  128. data/lib/insika/funnel_store.rb +163 -0
  129. data/lib/insika/golden_store.rb +3 -3
  130. data/lib/insika/grounding/matcher.rb +69 -0
  131. data/lib/insika/grounding.rb +44 -0
  132. data/lib/insika/harvest/conversion_gate.rb +159 -0
  133. data/lib/insika/harvest/criterion.rb +98 -0
  134. data/lib/insika/harvest/gate.rb +194 -0
  135. data/lib/insika/harvest/negative_list.rb +199 -0
  136. data/lib/insika/harvest.rb +241 -0
  137. data/lib/insika/harvest_engine.rb +193 -0
  138. data/lib/insika/harvest_store.rb +548 -0
  139. data/lib/insika/http_client.rb +3 -3
  140. data/lib/insika/inbound_log.rb +1 -1
  141. data/lib/insika/llm_configurator.rb +3 -3
  142. data/lib/insika/loop_detector.rb +143 -0
  143. data/lib/insika/mcp_http_client.rb +4 -4
  144. data/lib/insika/mcp_tool_ingestor.rb +6 -6
  145. data/lib/insika/media.rb +298 -0
  146. data/lib/insika/memory_audit_store.rb +85 -0
  147. data/lib/insika/memory_store.rb +264 -23
  148. data/lib/insika/message_origin.rb +8 -3
  149. data/lib/insika/model_resolver.rb +1 -1
  150. data/lib/insika/model_selection.rb +5 -4
  151. data/lib/insika/model_visible.rb +87 -0
  152. data/lib/insika/model_visible_trace_store.rb +66 -0
  153. data/lib/insika/onboarding.rb +8 -3
  154. data/lib/insika/outbox_store.rb +44 -6
  155. data/lib/insika/outcome_store.rb +147 -0
  156. data/lib/insika/overlay_tool_registry.rb +3 -4
  157. data/lib/insika/pack.rb +3 -3
  158. data/lib/insika/pack_importer.rb +17 -15
  159. data/lib/insika/packaging.rb +163 -0
  160. data/lib/insika/parity/criterion.rb +79 -0
  161. data/lib/insika/parity/verdict.rb +318 -0
  162. data/lib/insika/pending_action_store.rb +1 -1
  163. data/lib/insika/plugin/loader.rb +2 -2
  164. data/lib/insika/policy/policy.rb +1 -1
  165. data/lib/insika/prefix_fingerprint.rb +58 -0
  166. data/lib/insika/profile_source.rb +34 -7
  167. data/lib/insika/proposal_store.rb +271 -0
  168. data/lib/insika/provider_error_classifier.rb +160 -0
  169. data/lib/insika/queue_policy.rb +6 -3
  170. data/lib/insika/recovery.rb +47 -6
  171. data/lib/insika/refinement/candidate.rb +4 -4
  172. data/lib/insika/refinement/evidence_collector.rb +6 -6
  173. data/lib/insika/refinement/gate.rb +7 -7
  174. data/lib/insika/refinement/panel.rb +7 -7
  175. data/lib/insika/refinement/proposer.rb +10 -10
  176. data/lib/insika/refinement_store.rb +12 -12
  177. data/lib/insika/reliability.rb +211 -0
  178. data/lib/insika/retention.rb +281 -0
  179. data/lib/insika/routing.rb +101 -0
  180. data/lib/insika/safety/config.rb +46 -6
  181. data/lib/insika/safety/corpus.rb +255 -0
  182. data/lib/insika/safety/detectors.rb +34 -115
  183. data/lib/insika/safety/factory.rb +18 -5
  184. data/lib/insika/safety/grounding_enforcer.rb +59 -0
  185. data/lib/insika/safety/grounding_validator.rb +49 -0
  186. data/lib/insika/safety/input_guardrail.rb +20 -5
  187. data/lib/insika/safety/moderator.rb +19 -11
  188. data/lib/insika/safety/output_filter.rb +10 -6
  189. data/lib/insika/safety/output_validator.rb +13 -7
  190. data/lib/insika/safety/safe_responses.rb +1 -1
  191. data/lib/insika/sandbox/boundary.rb +2 -2
  192. data/lib/insika/sandbox.rb +1 -1
  193. data/lib/insika/schema_guard.rb +35 -0
  194. data/lib/insika/server/app.rb +366 -54
  195. data/lib/insika/server/boot.rb +4 -4
  196. data/lib/insika/server/rack_app.rb +31 -7
  197. data/lib/insika/server/responses.rb +58 -9
  198. data/lib/insika/server/tenant_auth.rb +61 -0
  199. data/lib/insika/session_actor.rb +11 -7
  200. data/lib/insika/session_store.rb +66 -3
  201. data/lib/insika/settings_store.rb +15 -5
  202. data/lib/insika/shadow_pair_store.rb +258 -0
  203. data/lib/insika/shutdown.rb +4 -4
  204. data/lib/insika/skill_catalog.rb +131 -20
  205. data/lib/insika/skill_store.rb +70 -22
  206. data/lib/insika/soak/envelope.rb +140 -0
  207. data/lib/insika/soak/report.rb +392 -0
  208. data/lib/insika/soak/runner.rb +554 -0
  209. data/lib/insika/steer_injector.rb +1 -1
  210. data/lib/insika/store.rb +11 -2
  211. data/lib/insika/stores/memory.rb +6 -0
  212. data/lib/insika/stores/sqlite.rb +8 -0
  213. data/lib/insika/studio/app.rb +1058 -75
  214. data/lib/insika/studio/assets/dist/application.css +1 -1
  215. data/lib/insika/studio/assets/dist/application.js +27 -26
  216. data/lib/insika/studio/assets/dist/favicon.svg +6 -0
  217. data/lib/insika/studio/forms.rb +274 -22
  218. data/lib/insika/studio/nav_icons.rb +7 -2
  219. data/lib/insika/studio/views/_message.erb +2 -2
  220. data/lib/insika/studio/views/agent_detail.erb +629 -86
  221. data/lib/insika/studio/views/agents.erb +11 -7
  222. data/lib/insika/studio/views/approvals.erb +4 -1
  223. data/lib/insika/studio/views/chats.erb +4 -1
  224. data/lib/insika/studio/views/customer.erb +94 -0
  225. data/lib/insika/studio/views/customers.erb +32 -0
  226. data/lib/insika/studio/views/evals.erb +4 -1
  227. data/lib/insika/studio/views/facts.erb +133 -0
  228. data/lib/insika/studio/views/followups.erb +125 -0
  229. data/lib/insika/studio/views/funnel.erb +106 -0
  230. data/lib/insika/studio/views/harvest.erb +234 -0
  231. data/lib/insika/studio/views/home.erb +2 -1
  232. data/lib/insika/studio/views/layout.erb +1 -0
  233. data/lib/insika/studio/views/parity.erb +147 -0
  234. data/lib/insika/studio/views/playground.erb +7 -1
  235. data/lib/insika/studio/views/refinement.erb +4 -4
  236. data/lib/insika/studio/views/session.erb +133 -3
  237. data/lib/insika/studio/views/settings.erb +9 -12
  238. data/lib/insika/studio/views/skills.erb +66 -12
  239. data/lib/insika/studio/views/system_files.erb +1 -1
  240. data/lib/insika/studio/views/task.erb +13 -0
  241. data/lib/insika/studio/views/tasks.erb +4 -1
  242. data/lib/insika/studio/views/tools.erb +0 -1
  243. data/lib/insika/subagent_graph.rb +3 -3
  244. data/lib/insika/task_actor.rb +3 -3
  245. data/lib/insika/task_store.rb +22 -2
  246. data/lib/insika/telemetry/pricing.rb +3 -3
  247. data/lib/insika/telemetry/recorder.rb +1 -1
  248. data/lib/insika/telemetry.rb +2 -2
  249. data/lib/insika/testing/store_contract.rb +54 -33
  250. data/lib/insika/tick.rb +146 -0
  251. data/lib/insika/token_store.rb +168 -0
  252. data/lib/insika/tool_assembly.rb +5 -5
  253. data/lib/insika/tool_definition.rb +25 -15
  254. data/lib/insika/tool_envelope.rb +70 -1
  255. data/lib/insika/tool_manifest.rb +11 -7
  256. data/lib/insika/tool_output_compressor.rb +100 -0
  257. data/lib/insika/tool_store.rb +1 -1
  258. data/lib/insika/tool_trace_store.rb +1 -1
  259. data/lib/insika/tools/concurrency.rb +2 -2
  260. data/lib/insika/tools/data_defined_tool.rb +14 -5
  261. data/lib/insika/tools/generate_image.rb +44 -0
  262. data/lib/insika/tools/load_skill.rb +61 -3
  263. data/lib/insika/tools/schedule_followup.rb +164 -0
  264. data/lib/insika/tools/stuck_signal.rb +44 -0
  265. data/lib/insika/tools/subagent.rb +4 -4
  266. data/lib/insika/tools/subagents.rb +1 -1
  267. data/lib/insika/tools/tts.rb +47 -0
  268. data/lib/insika/tools/update_briefing.rb +126 -0
  269. data/lib/insika/turn_output.rb +2 -2
  270. data/lib/insika/turn_state.rb +54 -13
  271. data/lib/insika/turn_timing.rb +24 -4
  272. data/lib/insika/usage_ledger.rb +1 -1
  273. data/lib/insika/version.rb +1 -1
  274. data/lib/insika/vitals.rb +84 -0
  275. data/lib/insika/wiring/graph.rb +372 -34
  276. data/lib/insika/workflow.rb +1 -1
  277. data/lib/insika/workflow_registry.rb +1 -1
  278. data/lib/insika.rb +122 -16
  279. metadata +95 -2
  280. data/lib/insika/server/admin_auth.rb +0 -29
@@ -0,0 +1,143 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Insika
4
+ # loop detection by (tool, args) hash, with a ONE-SHOT intervention.
5
+ #
6
+ # `max_tool_calls` bounds how MANY tool calls a turn makes, not how useful they
7
+ # are: a model retrying the exact same call — same tool, identical arguments —
8
+ # after an empty or error result burns the whole budget doing something that
9
+ # was settled on the first repeat. This detector is the engine saying so, once.
10
+ #
11
+ # The streak is CONSECUTIVE and turn-scoped, like the max_tool_calls counter it
12
+ # sits next to in ChatBuilder#wire_callbacks: a call revisited much later in a
13
+ # long turn is not the pathology being caught, and semantic ("nearly the same")
14
+ # matching is how a guard-rail starts eating legitimate retries.
15
+ #
16
+ # Two invariants, both borrowed from SteerInjector, because the
17
+ # intervention is a `user` message appended mid-loop:
18
+ #
19
+ # · **Batch boundary only.** The append happens after the LAST tool result of a
20
+ # batch closes — a `user` message between two tool results is rejected by
21
+ # Anthropic outright. Same arithmetic: an assistant message opens a batch of
22
+ # N, the Nth `role: tool` message closes it.
23
+ # · **A halted batch receives nothing.** With `halt_when` there is no next
24
+ # model step; a warning appended there would sit unanswered forever.
25
+ #
26
+ # The repeated call itself STILL RUNS — fabricating a synthetic result would
27
+ # teach the model that tools lie (the failure refuses). The
28
+ # warning rides after the truth; only a repeat that arrives AFTER the warning
29
+ # was spent aborts, through the existing TimeoutError(stage: :tool_limit) path.
30
+ class LoopDetector
31
+ # The one intervention text, verbatim — a fixed engine sentence, so a report
32
+ # can identify it without an origin stamp (chat messages carry none).
33
+ def self.intervention(name, streak)
34
+ "You have called `#{name}` with identical arguments #{streak} times in a row and " \
35
+ "received the same result every time. Repeating it will not produce new information. " \
36
+ "Do not call it again with the same arguments — answer with what you already have, " \
37
+ "or change your approach."
38
+ end
39
+
40
+ # chat: the turn's chat — must answer #add_message (the boundary append).
41
+ # limit: the streak that triggers the intervention (profile's
42
+ # max_tool_repeat). Values < 2 mean OFF: a "streak of 1" is every
43
+ # call, which is meaningless.
44
+ # emit: ->(type, data) — the Executor's emitter, bound to the task.
45
+ def initialize(chat:, limit:, emit:)
46
+ @chat = chat
47
+ @limit = limit
48
+ @emit = emit
49
+ @last = nil # fingerprint of the previous call (nil = none yet)
50
+ @streak = 0
51
+ @intervened = false # the ONE warning of this turn has been delivered
52
+ @pending = false # detection fired; waiting for the batch boundary
53
+ @expected = nil # tool calls announced by the batch in flight
54
+ @seen = 0
55
+ @halted = false
56
+ end
57
+
58
+ # From ChatBuilder's before_tool_call. Raises BEFORE the call executes once
59
+ # the warning is spent — bounded spend is the point of aborting here.
60
+ def tool_call(name, arguments)
61
+ fingerprint = [name.to_s, canonical(arguments)]
62
+ if fingerprint == @last
63
+ @streak += 1
64
+ else
65
+ # A different call broke the run: the loop resolved itself, so a warning
66
+ # armed earlier is moot — it must not fire later naming the WRONG call.
67
+ @streak = 1
68
+ @pending = false
69
+ end
70
+ @last = fingerprint
71
+ return if @streak < @limit
72
+
73
+ if @intervened
74
+ raise Insika::TimeoutError.new(
75
+ "tool loop detected (#{name} repeated with identical arguments after a warning)",
76
+ stage: :tool_limit)
77
+ end
78
+ @pending = true
79
+ end
80
+
81
+ # From ChatBuilder's after_tool_result, with the RAW result — the only place
82
+ # a Tool::Halt is still recognizable (SteerInjector's comment applies here).
83
+ def tool_result(result)
84
+ @halted = true if defined?(RubyLLM::Tool::Halt) && result.is_a?(RubyLLM::Tool::Halt)
85
+ end
86
+
87
+ # RubyLLM after_message. An assistant message carrying tool calls OPENS a
88
+ # batch; the Nth tool result CLOSES it — the one boundary where appending
89
+ # is valid.
90
+ def message_ended(message)
91
+ role = field(message, :role).to_s
92
+ return open_batch(message) if role == "assistant"
93
+ return unless role == "tool" && @expected
94
+
95
+ @seen += 1
96
+ intervene! if @seen >= @expected
97
+ end
98
+
99
+ private
100
+
101
+ def open_batch(message)
102
+ calls = field(message, :tool_calls)
103
+ size = calls.respond_to?(:size) ? calls.size : 0
104
+ # No tool call = the model talking; the turn is ending and a pending
105
+ # warning is moot — the loop resolved itself.
106
+ return @expected = nil if size.zero?
107
+
108
+ @expected = size
109
+ @seen = 0
110
+ @halted = false
111
+ end
112
+
113
+ def intervene!
114
+ @expected = nil
115
+ return unless @pending
116
+ @pending = false
117
+ return if @halted # nothing will read it (halt_when): drop, never deliver
118
+
119
+ @intervened = true
120
+ name, = @last
121
+ @chat.add_message(role: :user, content: self.class.intervention(name, @streak))
122
+ # Counts and the tool name, never the arguments — order numbers are PII.
123
+ @emit.call(:tool_loop_intervened, { name: name, streak: @streak })
124
+ end
125
+
126
+ # (name, args) hash: symbols vs strings and key order must not split an
127
+ # identical call into two fingerprints. Compared with ==, never hashed.
128
+ def canonical(value)
129
+ case value
130
+ when Hash then value.map { |k, v| [k.to_s, canonical(v)] }.sort_by(&:first)
131
+ when Array then value.map { |v| canonical(v) }
132
+ else value
133
+ end
134
+ end
135
+
136
+ def field(message, name)
137
+ return message.public_send(name) if message.respond_to?(name)
138
+ return message[name] || message[name.to_s] if message.respond_to?(:[])
139
+
140
+ nil
141
+ end
142
+ end
143
+ end
@@ -3,19 +3,19 @@
3
3
  require "json"
4
4
 
5
5
  module Insika
6
- # MINIMAL MCP client over HTTP JSON-RPC (Phase 7, Stage E). Discovers the tools
6
+ # MINIMAL MCP client over HTTP JSON-RPC. Discovers the tools
7
7
  # of an MCP instance with HTTP transport by making a JSON-RPC 2.0 `tools/list`
8
8
  # POST to the instance endpoint, behind the EgressGuard (SSRF — the url comes
9
- # from editable config, NF4). It is the DEFAULT client injected into the
9
+ # from editable config). It is the DEFAULT client injected into the
10
10
  # McpToolIngestor; tests pass a Fake (duck-typed) in its place.
11
11
  #
12
12
  # Contract (MCP client duck-type): `#list_tools -> [{name, description,
13
13
  # inputSchema}]` — the same MCP envelope that the ToolManifest adapter normalizes.
14
14
  #
15
- # SCOPE (bounded, D8): only the minimal handshake of ONE stateless `tools/list`
15
+ # SCOPE (bounded): only the minimal handshake of ONE stateless `tools/list`
16
16
  # POST. Does NOT implement the full MCP session lifecycle (initialize/protocol
17
17
  # negotiation/session-id/notifications) nor the stdio transport — that is the
18
- # "real MCP transport", later work (out-of-scope, see spec §4 D8). It serves
18
+ # "real MCP transport", later work (out-of-scope, see spec). It serves
19
19
  # simple HTTP MCP servers (direct JSON-RPC) and proves the ingestion seam.
20
20
  class McpHttpClient
21
21
  JSONRPC_VERSION = "2.0"
@@ -3,16 +3,16 @@
3
3
  require "json"
4
4
 
5
5
  module Insika
6
- # LIVE MCP ingestion (Phase 7, Stage E / spec §4 D8): discovers the tools of an
6
+ # LIVE MCP ingestion (/ spec): discovers the tools of an
7
7
  # MCP instance at RUNTIME (no hand-written manifest) and ingests them as
8
8
  # data-tools. Given an McpStore instance + an INJECTABLE MCP client
9
9
  # (duck-typed: `#list_tools -> [{name, description, inputSchema}]`), it builds a
10
- # ToolManifest and REUSES the Stage B ingestion path (the :import_tools Command:
10
+ # ToolManifest and REUSES the ingestion path (the:import_tools Command:
11
11
  # batch upsert into the ToolStore + hot reload + per-tool report + partial-
12
12
  # failure isolation R4). The ToolManifest MCP adapter (`inputSchema`) is reused
13
13
  # — no schema parsing here.
14
14
  #
15
- # GENERIC (NF1): nothing here mentions achei/openclaw. The MCP instance is DATA in the store.
15
+ # GENERIC: nothing here mentions a consumer/gateway. The MCP instance is DATA in the store.
16
16
  #
17
17
  # BINDING STRATEGY (this stage's choice, bounded):
18
18
  # Each discovered tool becomes an HTTP data-tool that makes a JSON-RPC 2.0
@@ -23,10 +23,10 @@ module Insika
23
23
  # runs through the SAME HTTP path as the other data-tools (egress guard, secret
24
24
  # headers, hot reload) — no new execution code.
25
25
  #
26
- # Each tool gets `group: "mcp:<instance>"` so the Stage C per-group gating
26
+ # Each tool gets `group: "mcp:<instance>"` so the per-group gating
27
27
  # (tools_allow_groups) works for free.
28
28
  #
29
- # DEFERRED / OUT-OF-SCOPE (documented — spec §4 D8):
29
+ # DEFERRED / OUT-OF-SCOPE (documented — spec):
30
30
  # - Real MCP transport: only instances with a `url` (http transport) are ingestible;
31
31
  # stdio has no HTTP endpoint -> raises a clear error (later work).
32
32
  # - MCP session lifecycle (initialize/negotiation/session-id/notifications) and the
@@ -65,7 +65,7 @@ module Insika
65
65
  if url.nil?
66
66
  raise Insika::ValidationError,
67
67
  "MCP instance '#{name}' has no url: live ingestion requires HTTP transport " \
68
- "(stdio is later work — D8)"
68
+ "(stdio is later work)"
69
69
  end
70
70
 
71
71
  tools = Array((client || @client_factory.call(record)).list_tools)
@@ -0,0 +1,298 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Insika
4
+ # WS9: the engine transports MEDIA, never meaning. Content parts ride the
5
+ # message contract — `{ "type": "text", "text": … }`, `{ "type": "image",
6
+ # "url": … }`, `{ "type": "audio", "url": … }` — and the Executor turns them
7
+ # into a turn: audio is transcribed (text marked `source: :voice`), images
8
+ # attach to the model ask and the first URL is `{{ctx.image_url}}` for
9
+ # data/HTTP tools. This class owns the PURE parts (normalization) and
10
+ # the STT SEAM (injectable — specs stub it; the default fetches the audio and
11
+ # transcribes via RubyLLM behind a lazy require, so the core stays gem-free
12
+ # at load). `Media::Output` is the generated-media half (WS9, saída): the
13
+ # turn can PRODUCE an image or an audio clip when the agent opted in
14
+ # (`AgentProfile#outputs`) AND the channel declared it can receive it
15
+ # (`channel.capabilities`) — nothing leaks by default.
16
+ module Media
17
+ # A single content part, normalized.
18
+ Part = Data.define(:type, :text, :url) do
19
+ def audio? = type == "audio"
20
+ def image? = type == "image"
21
+ def text? = type == "text"
22
+ end
23
+
24
+ # -> [Part]: normalize the raw parts (string|symbol keys), skipping anything
25
+ # that is not a well-formed text/image/audio part. Lenient on purpose — the
26
+ # SURFACE validates the contract with `well_formed?` (a malformed part is a
27
+ # 422 before dispatch); here a stray entry must not break the turn.
28
+ def self.parts(raw)
29
+ Array(raw).filter_map do |p|
30
+ next unless p.is_a?(Hash)
31
+
32
+ type = (p[:type] || p["type"]).to_s
33
+ url = (p[:url] || p["url"]).to_s
34
+ text = (p[:text] || p["text"]).to_s
35
+ case type
36
+ when "text" then text.empty? ? nil : Part.new("text", text, nil)
37
+ when "image", "audio" then url.empty? ? nil : Part.new(type, nil, url)
38
+ else nil
39
+ end
40
+ end
41
+ end
42
+
43
+ # The SURFACE's contract check (server edge): true when EVERY entry is a
44
+ # well-formed content part — a Hash whose type is text (with text), image
45
+ # or audio (with url). The edge raises a 422 on the first offender; the
46
+ # engine itself stays lenient (`parts` skips strays so a non-HTTP transport
47
+ # that bypassed the edge cannot break a turn).
48
+ def self.well_formed?(raw)
49
+ Array(raw).all? do |p|
50
+ next false unless p.is_a?(Hash)
51
+
52
+ case (p[:type] || p["type"]).to_s
53
+ when "text" then !(p[:text] || p["text"]).to_s.empty?
54
+ when "image", "audio" then !(p[:url] || p["url"]).to_s.empty?
55
+ # a part WITHOUT a type is admitted only as a bare text part (the
56
+ # shape the input joiner already tolerates) — anything else is refused.
57
+ when "" then !(p[:text] || p["text"]).to_s.empty?
58
+ else false
59
+ end
60
+ end
61
+ end
62
+
63
+ # The OUTPUT media kinds a channel may declare it can receive
64
+ # (`channel.capabilities`). The closed list is the "abstraction admits
65
+ # only what leaks" rule: an unknown value is refused at the edge, never
66
+ # silently ignored.
67
+ OUTPUT_CAPABILITIES = %w[image_output audio_output].freeze
68
+
69
+ # -> [String]: the capabilities a raw `channel` hash declares. Lenient on
70
+ # the key spelling (symbol|string) at both boundaries (request parse vs
71
+ # persisted command payload); [] = the channel declared nothing.
72
+ def self.channel_capabilities(raw)
73
+ channel = raw.is_a?(Hash) ? raw : {}
74
+ Array(channel[:capabilities] || channel["capabilities"]).map(&:to_s)
75
+ end
76
+
77
+ # The ceilings on INBOUND media (a URL a consumer sent us). Both fetches
78
+ # stream into the cap and refuse past it: the bytes land in THIS process,
79
+ # so an uncapped one is a hostile URL away from growing it until it dies.
80
+ MAX_AUDIO_BYTES = 1_000_000 # a voice note, not a warehouse
81
+ MAX_IMAGE_BYTES = 5_000_000 # a photo, not a poster
82
+
83
+ def self.audio_parts(parts) = parts.select(&:audio?)
84
+ def self.image_parts(parts) = parts.select(&:image?)
85
+
86
+ # The STT seam: ->(url) { text } (default: fetch + RubyLLM transcription).
87
+ # Injected so a spec never touches the network; the default is built lazily
88
+ # when the turn first carries audio.
89
+ def self.default_transcriber(stt_model:, stt_language: nil)
90
+ lambda do |url|
91
+ fetch_and_transcribe(url, model: stt_model, language: stt_language)
92
+ end
93
+ end
94
+
95
+ def self.fetch_and_transcribe(url, model:, language:)
96
+ require "net/http"
97
+ require "uri"
98
+ require "ruby_llm" # lazy — the core loads without it (load-guard)
99
+
100
+ bytes = fetch_binary(url)
101
+ audio = RubyLLM::Attachment.new(bytes)
102
+ options = { model: model, assume_model_exists: true }
103
+ options[:language] = language if language
104
+ RubyLLM::Transcription.transcribe(audio, **options).text
105
+ end
106
+
107
+ # Egress-guarded binary fetch of a media URL. Blocked like the webhook: the
108
+ # url is consumer config/input, so a private/loopback/metadata target is
109
+ # refused (SSRF) unless the deployment opts out. Size-capped (the caller
110
+ # picks the ceiling; the default is the audio one).
111
+ def self.fetch_binary(url, max_bytes: MAX_AUDIO_BYTES)
112
+ violation = Insika::EgressGuard.violation(url, **egress_opt_out)
113
+ raise Insika::MediaError, "media egress blocked for #{url}: #{violation}" if violation
114
+
115
+ uri = URI.parse(url)
116
+ opts = { use_ssl: uri.scheme == "https", open_timeout: 30, read_timeout: 60 }
117
+ Net::HTTP.start(uri.host, uri.port, opts) do |http|
118
+ buf = +"".b
119
+ http.request(Net::HTTP::Get.new(uri)) do |resp|
120
+ raise Insika::MediaError, "media fetch HTTP #{resp.code}" unless resp.is_a?(Net::HTTPSuccess)
121
+
122
+ resp.read_body { |chunk| buf << chunk; break if buf.bytesize > max_bytes }
123
+ end
124
+ raise Insika::MediaError, "media exceeds #{max_bytes} bytes" if buf.bytesize > max_bytes
125
+ buf
126
+ end
127
+ rescue URI::InvalidURIError
128
+ raise Insika::MediaError, "invalid media URL"
129
+ end
130
+
131
+ # The opt-out the comment above promises, read from the SAME env the
132
+ # data-tool guard reads (INSIKA_EGRESS_ALLOW_HTTP / _ALLOW_PRIVATE): without
133
+ # this, a local run serving media over http:// ALWAYS failed, however the
134
+ # deployment was configured. INSIKA_EGRESS_HOSTS is deliberately NOT applied:
135
+ # that allowlist pins the handful of hosts a tool may call, while media URLs
136
+ # come from the channel's CDN — honouring it here would break every real
137
+ # deployment that narrows its tools.
138
+ def self.egress_opt_out
139
+ { allow_http: Insika::EnvSchema.truthy?(ENV["INSIKA_EGRESS_ALLOW_HTTP"]),
140
+ allow_private: Insika::EnvSchema.truthy?(ENV["INSIKA_EGRESS_ALLOW_PRIVATE"]) }
141
+ end
142
+
143
+ # WS9 (saída): generated media. The OUTPUT shape is an additive part —
144
+ # `{ "type": "image"|"audio", "mime_type": …, "base64": …, "model": … }`
145
+ # — that rides the turn's `output_parts` (terminal event + /v1/responses
146
+ # envelope), NEVER the answer text: the customer's channel consumes the
147
+ # bytes, the model's prose stays the answer.
148
+ #
149
+ # The GENERATION SEAMS are injectable like the STT seam: each is a
150
+ # `->(content, config) { [ part_hash, usage_hash ] }` (part_hash already
151
+ # carries its "type"), specs stub them, and the defaults hit the provider
152
+ # behind lazy requires:
153
+ # · image — RubyLLM.paint (the gem has vision AND painting), billed
154
+ # tokens merged into the turn's usage like any ask;
155
+ # · tts — RubyLLM still has NO speech API (as of 1.16.0), so the default
156
+ # is a thin POST to the OpenAI-compatible `<base>/audio/speech`
157
+ # endpoint (base + key from the provider config the chat uses — a
158
+ # deployment pointing OpenAI at a gateway keeps TTS pointing there).
159
+ # OpenAI's speech API reports no token usage; the part carries the
160
+ # model so the consumer can price it, and the turn counts the call.
161
+ module Output
162
+ DEFAULT_IMAGE_SIZE = "1024x1024"
163
+ DEFAULT_TTS_MODEL = "tts-1"
164
+ DEFAULT_TTS_VOICE = "alloy"
165
+ DEFAULT_TTS_FORMAT = "mp3"
166
+ # Base64 inlines into the envelope — a cap so a pathological generation
167
+ # cannot blow up the SSE frame. A generated 1024x1024 PNG sits well under.
168
+ MAX_EMBEDDED_BYTES = 8 * 1024 * 1024
169
+
170
+ class << self
171
+ # -> { image: seam, tts: seam } with the DEFAULTS bound to a context
172
+ # (the graph's RubyLLM::Context when it owns credentials — nil = the
173
+ # process-wide RubyLLM constant). Built lazily on first generation so
174
+ # the core loads without ruby_llm (load-guard).
175
+ def defaults(context:)
176
+ {
177
+ image: ->(prompt, config) { generate_image(prompt, config: config, context: context) },
178
+ tts: ->(text, config) { synthesize_speech(text, config: config, context: context) }
179
+ }
180
+ end
181
+
182
+ # -> [Part, usage]: paint via RubyLLM. usage is the provider's token
183
+ # counts ({ input_tokens:, output_tokens: } — merged into the turn's
184
+ # usage by the Executor); a provider without counts reports nothing.
185
+ def generate_image(prompt, config:, context:)
186
+ require "ruby_llm" # lazy — the core loads without it (load-guard)
187
+
188
+ cfg = Insika::Coercion.deep_stringify(config || {})
189
+ model = Insika::Coercion.presence(cfg["model"]) || image_model(context)
190
+ api = context || RubyLLM
191
+ image = api.paint(prompt.to_s, model: model, assume_model_exists: true,
192
+ size: presence(cfg["size"]) || DEFAULT_IMAGE_SIZE)
193
+ data = image.respond_to?(:data) ? image.data : nil
194
+ raise Insika::MediaError, "image generation returned no embeddable data" if data.to_s.empty?
195
+
196
+ enforce_embedded_size!(data, "generated image")
197
+ mime = image.respond_to?(:mime_type) ? image.mime_type : nil
198
+ model_id = image.respond_to?(:model_id) ? image.model_id : nil
199
+ usage = image.respond_to?(:usage) ? token_usage(image.usage) : {}
200
+ part = { "type" => "image", "mime_type" => presence(mime) || "image/png",
201
+ "base64" => data, "model" => presence(model_id) }
202
+ [part.compact, usage]
203
+ end
204
+
205
+ # -> [Part, {}]: synthesize speech via the OpenAI-compatible
206
+ # `<base>/audio/speech` endpoint. `context` supplies the base URL + key
207
+ # (the same config the chat uses — see `speech_endpoint`). The bytes
208
+ # embed base64 in the part; the usage is empty (no token counts on the
209
+ # speech API) and the part carries the model for consumer-side pricing.
210
+ def synthesize_speech(text, config:, context:)
211
+ require "net/http"
212
+ require "uri"
213
+ require "json"
214
+ require "base64"
215
+
216
+ cfg = Insika::Coercion.deep_stringify(config || {})
217
+ model = presence(cfg["model"]) || DEFAULT_TTS_MODEL
218
+ voice = presence(cfg["voice"]) || DEFAULT_TTS_VOICE
219
+ format = presence(cfg["format"]) || DEFAULT_TTS_FORMAT
220
+ base, key = speech_endpoint(context)
221
+ if key.to_s.empty?
222
+ raise Insika::MediaError,
223
+ "TTS needs an OpenAI API key (provider config) — set it on the " \
224
+ "provider the agent uses, or inject a tts seam"
225
+ end
226
+
227
+ uri = URI.parse("#{base}/audio/speech")
228
+ req = Net::HTTP::Post.new(uri)
229
+ req["Authorization"] = "Bearer #{key}"
230
+ req["Content-Type"] = "application/json"
231
+ req.body = JSON.generate(model: model, voice: voice, input: text.to_s,
232
+ response_format: format)
233
+ opts = { use_ssl: uri.scheme == "https", open_timeout: 30, read_timeout: 60 }
234
+ bytes = Net::HTTP.start(uri.host, uri.port, opts) do |http|
235
+ resp = http.request(req)
236
+ raise Insika::MediaError, "TTS HTTP #{resp.code}" unless resp.is_a?(Net::HTTPSuccess)
237
+
238
+ # stream into the cap — a rogue/broken endpoint must not grow the
239
+ # process past MAX_EMBEDDED_BYTES before the refusal.
240
+ buf = +"".b
241
+ resp.read_body do |chunk|
242
+ buf << chunk
243
+ break if buf.bytesize > MAX_EMBEDDED_BYTES
244
+ end
245
+ buf
246
+ end
247
+ enforce_embedded_size!(bytes, "synthesized speech")
248
+ part = { "type" => "audio", "mime_type" => mime_for(format), "base64" => Base64.strict_encode64(bytes), "model" => model }
249
+ [part.compact, {}]
250
+ rescue URI::InvalidURIError
251
+ raise Insika::MediaError, "invalid TTS endpoint"
252
+ end
253
+
254
+ private
255
+
256
+ # The OpenAI-compatible base URL + key behind the chat's provider
257
+ # config. A RubyLLM::Context owns the deployment's credentials; the
258
+ # global config is the fallback (a graph without its own context).
259
+ def speech_endpoint(context)
260
+ config = context.respond_to?(:config) ? context.config : RubyLLM.config
261
+ base = config.respond_to?(:openai_api_base) ? config.openai_api_base : nil
262
+ key = config.respond_to?(:openai_api_key) ? config.openai_api_key : nil
263
+ [presence(base) || "https://api.openai.com/v1", key]
264
+ end
265
+
266
+ def image_model(context)
267
+ config = context.respond_to?(:config) ? context.config : RubyLLM.config
268
+ config.respond_to?(:default_image_model) ? config.default_image_model : nil
269
+ end
270
+
271
+ def token_usage(raw)
272
+ usage = raw.is_a?(Hash) ? raw : {}
273
+ {
274
+ input_tokens: usage[:input_tokens] || usage["input_tokens"] || usage["prompt_tokens"],
275
+ output_tokens: usage[:output_tokens] || usage["output_tokens"] || usage["completion_tokens"]
276
+ }.compact
277
+ end
278
+
279
+ def mime_for(format)
280
+ { "mp3" => "audio/mpeg", "opus" => "audio/opus", "aac" => "audio/aac",
281
+ "wav" => "audio/wav", "flac" => "audio/flac" }[format.to_s] || "audio/mpeg"
282
+ end
283
+
284
+ def enforce_embedded_size!(data, label)
285
+ size = data.respond_to?(:bytesize) ? data.bytesize : data.to_s.bytesize
286
+ return if size <= MAX_EMBEDDED_BYTES
287
+
288
+ raise Insika::MediaError,
289
+ "#{label} too large to embed (#{size} bytes > #{MAX_EMBEDDED_BYTES})"
290
+ end
291
+
292
+ def presence(value)
293
+ Insika::Coercion.presence(value)
294
+ end
295
+ end
296
+ end
297
+ end
298
+ end
@@ -0,0 +1,85 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "digest/sha2"
4
+ require "json"
5
+ require "time"
6
+
7
+ module Insika
8
+ # append-only, per-cell audit of MEMORY mutations. Every
9
+ # operator mutation of a cell appends a line (who, when, what, old->new
10
+ # digests). The entry holds DIGESTS of values, never the values — the forget
11
+ # line records that a deletion happened without the deleted content (the
12
+ # digest is not invertible, and keys are fact names — provenance, not
13
+ # payload). Capped per cell (no retention hook, no delete path: the audit
14
+ # outlives what it describes). `record` rescues EVERYTHING — a failed audit
15
+ # write never fails the mutation it describes.
16
+ #
17
+ # The capped-list RMW caveat (context_trace_store.rb's discipline): one cell
18
+ # key, written on the command's fiber; a cross-process race loses the loser's
19
+ # append — a trace-level loss, not a correctness one.
20
+ class MemoryAuditStore
21
+ SCOPE = "memory_audit" # store key = the memory cell scope
22
+ MAX_PER_CELL = 200 # oldest dropped; the cap bounds growth
23
+
24
+ Entry = Data.define(:at, :action, :actor, :key, :tenant, :customer,
25
+ :old_hash, :new_hash, :note)
26
+
27
+ def initialize(store:, clock: nil)
28
+ @store = store
29
+ @clock = clock # -> Time, injectable for specs
30
+ end
31
+
32
+ # action: "put" | "forget" | "purge". key = the fact name (provenance,
33
+ # not payload). Appends + caps. Rescues EVERYTHING -> Entry | nil (the
34
+ # audit never breaks a command).
35
+ def record(cell:, action:, actor:, key: nil, tenant: nil, customer: nil,
36
+ old_hash: nil, new_hash: nil, note: nil)
37
+ entry = {
38
+ "at" => timestamp,
39
+ "action" => action.to_s,
40
+ "actor" => actor.to_s,
41
+ "key" => Coercion.presence(key),
42
+ "tenant" => Coercion.presence(tenant),
43
+ "customer" => Coercion.presence(customer),
44
+ "old_hash" => Coercion.presence(old_hash),
45
+ "new_hash" => Coercion.presence(new_hash),
46
+ "note" => Coercion.presence(note)
47
+ }
48
+ list = (@store.get(SCOPE, cell.to_s) || []) + [entry]
49
+ @store.set(SCOPE, cell.to_s, list.last(MAX_PER_CELL))
50
+ to_entry(entry)
51
+ rescue StandardError
52
+ nil
53
+ end
54
+
55
+ # -> [Entry] most recent first. [] if none. A broken backend degrades to
56
+ # [] — the audit is read to RENDER, never to gate.
57
+ def for_cell(cell, limit: 100)
58
+ Array(@store.get(SCOPE, cell.to_s)).reverse.first(limit).map { |e| to_entry(e) }
59
+ rescue StandardError
60
+ []
61
+ end
62
+
63
+ # The digest the callers share: SHA-256 hexdigest of JSON.generate(value)
64
+ # (or value.to_s for non-JSON scalars). -> String
65
+ def self.digest(value)
66
+ payload = case value
67
+ when Hash, Array then JSON.generate(value)
68
+ else value.to_s
69
+ end
70
+ Digest::SHA256.hexdigest(payload)
71
+ end
72
+
73
+ private
74
+
75
+ def to_entry(record)
76
+ Entry.new(at: record["at"], action: record["action"], actor: record["actor"],
77
+ key: record["key"], tenant: record["tenant"], customer: record["customer"],
78
+ old_hash: record["old_hash"], new_hash: record["new_hash"], note: record["note"])
79
+ end
80
+
81
+ def timestamp = (clock ? clock.call : Time.now.utc).utc.iso8601(6)
82
+
83
+ def clock = @clock
84
+ end
85
+ end