insika 0.7.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +191 -0
  3. data/README.md +9 -6
  4. data/bin/insika +44 -3
  5. data/docs/AGENTS.md +74 -14
  6. data/docs/API.md +73 -0
  7. data/docs/ARCHITECTURE.md +45 -44
  8. data/docs/ARTIFACTS.md +42 -0
  9. data/docs/CHANNELS.md +19 -2
  10. data/docs/CONTEXT.md +86 -38
  11. data/docs/DEPLOY.md +27 -7
  12. data/docs/EVALS.md +98 -8
  13. data/docs/FACTS.md +4 -0
  14. data/docs/KNOWLEDGE.md +7 -0
  15. data/docs/LOADTEST.md +15 -27
  16. data/docs/MEDIA.md +1 -1
  17. data/docs/OBSERVABILITY.md +48 -4
  18. data/docs/POLICY.md +14 -5
  19. data/docs/RELEASING.md +4 -0
  20. data/docs/RUNNING-LOCAL.md +2 -2
  21. data/docs/SECURITY.md +28 -2
  22. data/docs/SOAK.md +1 -1
  23. data/docs/TOOLS.md +150 -32
  24. data/docs/prompts/ADD-TOOL.md +12 -2
  25. data/docs/prompts/DIAGNOSE-TURN.md +3 -0
  26. data/docs/prompts/GO-LIVE.md +6 -4
  27. data/lib/insika/agent_profile.rb +47 -10
  28. data/lib/insika/channels/web/widget.js +33 -0
  29. data/lib/insika/channels/web.rb +5 -2
  30. data/lib/insika/chat_builder.rb +90 -37
  31. data/lib/insika/commands/agent_payload.rb +1 -1
  32. data/lib/insika/commands/run_distillation.rb +5 -8
  33. data/lib/insika/commands/seed_session.rb +118 -0
  34. data/lib/insika/compaction.rb +196 -0
  35. data/lib/insika/context/builder.rb +35 -11
  36. data/lib/insika/context/fragment.rb +4 -1
  37. data/lib/insika/context/priority.rb +8 -0
  38. data/lib/insika/context/provider.rb +5 -0
  39. data/lib/insika/context/providers/briefing.rb +61 -29
  40. data/lib/insika/context/providers/fence_notice.rb +27 -0
  41. data/lib/insika/context/providers/knowledge.rb +7 -4
  42. data/lib/insika/context/providers/memory.rb +8 -4
  43. data/lib/insika/context/providers/session.rb +50 -10
  44. data/lib/insika/context_trace_store.rb +11 -1
  45. data/lib/insika/doctor.rb +213 -10
  46. data/lib/insika/dsl/runtime.rb +5 -0
  47. data/lib/insika/dsl.rb +6 -0
  48. data/lib/insika/edge_limiter.rb +4 -1
  49. data/lib/insika/env_schema.rb +5 -6
  50. data/lib/insika/errors.rb +1 -0
  51. data/lib/insika/evals/assertions.rb +92 -6
  52. data/lib/insika/evals/golden.rb +91 -2
  53. data/lib/insika/evals/runner.rb +20 -0
  54. data/lib/insika/evals/simulator.rb +11 -2
  55. data/lib/insika/evals/transport.rb +118 -16
  56. data/lib/insika/evidence.rb +79 -12
  57. data/lib/insika/executor.rb +94 -23
  58. data/lib/insika/fence.rb +96 -0
  59. data/lib/insika/golden_store.rb +3 -0
  60. data/lib/insika/loop_detector.rb +5 -34
  61. data/lib/insika/mcp_store.rb +5 -2
  62. data/lib/insika/mcp_tool_registry.rb +8 -1
  63. data/lib/insika/memory_store.rb +12 -0
  64. data/lib/insika/overlay_tool_registry.rb +5 -0
  65. data/lib/insika/prefix_fingerprint.rb +32 -27
  66. data/lib/insika/profile_source.rb +8 -0
  67. data/lib/insika/server/app.rb +43 -1
  68. data/lib/insika/server/rack_app.rb +2 -0
  69. data/lib/insika/server/responses.rb +35 -8
  70. data/lib/insika/session_store.rb +38 -5
  71. data/lib/insika/settings_store.rb +18 -2
  72. data/lib/insika/soak/runner.rb +4 -4
  73. data/lib/insika/spoken_transcript.rb +31 -0
  74. data/lib/insika/studio/app.rb +34 -8
  75. data/lib/insika/studio/forms.rb +29 -3
  76. data/lib/insika/studio/views/_agent_tab_config.erb +5 -1
  77. data/lib/insika/studio/views/session.erb +1 -1
  78. data/lib/insika/studio/views/settings.erb +11 -0
  79. data/lib/insika/studio/views/tool_edit.erb +6 -2
  80. data/lib/insika/telemetry/recorder.rb +61 -1
  81. data/lib/insika/templates/daily-digest/README.md +9 -0
  82. data/lib/insika/templates/research-analyst/agent.rb +10 -0
  83. data/lib/insika/tool_assembly.rb +21 -13
  84. data/lib/insika/tool_batch.rb +67 -0
  85. data/lib/insika/tool_definition.rb +73 -10
  86. data/lib/insika/tool_envelope.rb +102 -2
  87. data/lib/insika/tool_store.rb +9 -4
  88. data/lib/insika/tool_trace_store.rb +1 -1
  89. data/lib/insika/tool_usage_report.rb +172 -0
  90. data/lib/insika/tools/data_defined_tool.rb +1 -0
  91. data/lib/insika/tools/present.rb +122 -0
  92. data/lib/insika/tools/run_persona_eval.rb +6 -1
  93. data/lib/insika/tools/tool_search.rb +4 -2
  94. data/lib/insika/turn_budget.rb +91 -0
  95. data/lib/insika/turn_state.rb +13 -1
  96. data/lib/insika/version.rb +1 -1
  97. data/lib/insika/wiring/graph.rb +7 -0
  98. data/lib/insika/wiring/graph_chat.rb +4 -0
  99. data/lib/insika.rb +11 -0
  100. metadata +10 -1
@@ -35,7 +35,9 @@ module Insika
35
35
  ToolDefinition = Data.define(
36
36
  :name, :description, :parameters, :request, :response,
37
37
  :secret_headers, :side_effect, :timeout, :group, :tags, :halt_when,
38
- :evidence # Insika::Evidence::Spec | nil
38
+ :evidence, # Insika::Evidence::Spec | nil
39
+ :requires_evidence, # { "params" => [String] } | nil
40
+ :presentation # { component:, ids:, max: } | nil — a UI tool, no HTTP
39
41
  )
40
42
 
41
43
  class ToolDefinition
@@ -73,9 +75,9 @@ module Insika
73
75
  # Builds + validates. Raises Insika::ValidationError. Accepts keyword args
74
76
  # (already-normalized symbol keys); use from_h for a raw Hash from the store/UI.
75
77
  # `parameters` accepts JSON Schema (Hash) OR the legacy flat array.
76
- def self.build(name:, description:, request:, parameters: nil, response: nil,
78
+ def self.build(name:, description:, request: nil, parameters: nil, response: nil,
77
79
  secret_headers: nil, side_effect: nil, timeout: nil, group: nil, tags: nil,
78
- halt_when: nil, evidence: nil)
80
+ halt_when: nil, evidence: nil, presentation: nil, requires_evidence: nil)
79
81
  name = name.to_s
80
82
  raise Insika::ValidationError, "name must match #{NAME_RE.inspect}" unless NAME_RE.match?(name)
81
83
 
@@ -83,15 +85,21 @@ module Insika
83
85
  raise Insika::ValidationError, "description is required" if desc.empty?
84
86
 
85
87
  schema = normalize_params(parameters)
86
- req = normalize_request(request, top_level_names(schema))
88
+ # A tool is EITHER an HTTP call or a presentation — never both, never neither.
89
+ if request && presentation
90
+ raise Insika::ValidationError, "a tool declares either 'request' or 'presentation', not both"
91
+ end
92
+
93
+ pres = normalize_presentation(presentation, schema)
94
+ req = pres ? nil : normalize_request(request || {}, top_level_names(schema))
87
95
  resp = normalize_response(response)
88
96
  if resp[:extract] == "evidence_envelope" && evidence.nil?
89
97
  raise Insika::ValidationError,
90
98
  "extract 'evidence_envelope' requires an 'evidence' declaration"
91
99
  end
92
100
 
93
- method = req[:method]
94
- effect = side_effect.nil? ? !IDEMPOTENT.include?(method) : (side_effect ? true : false)
101
+ method = req && req[:method]
102
+ effect = side_effect.nil? ? !(method.nil? || IDEMPOTENT.include?(method)) : (side_effect ? true : false)
95
103
 
96
104
  new(
97
105
  name: name, description: desc, parameters: schema, request: req, response: resp,
@@ -99,7 +107,9 @@ module Insika
99
107
  timeout: timeout.nil? ? nil : Integer(timeout),
100
108
  group: normalize_group(group), tags: normalize_tags(tags),
101
109
  halt_when: normalize_halt_when(halt_when),
102
- evidence: Insika::Evidence::Spec.parse(evidence)
110
+ evidence: Insika::Evidence::Spec.parse(evidence),
111
+ presentation: pres,
112
+ requires_evidence: normalize_requires_evidence(requires_evidence, top_level_names(schema))
103
113
  )
104
114
  end
105
115
 
@@ -108,12 +118,63 @@ module Insika
108
118
  h = deep_symbolize(hash)
109
119
  build(
110
120
  name: h[:name], description: h[:description], parameters: h[:parameters],
111
- request: h[:request] || {}, response: h[:response],
121
+ request: h[:request], response: h[:response],
112
122
  secret_headers: h[:secret_headers], side_effect: h[:side_effect], timeout: h[:timeout],
113
- group: h[:group], tags: h[:tags], halt_when: h[:halt_when], evidence: h[:evidence]
123
+ group: h[:group], tags: h[:tags], halt_when: h[:halt_when], evidence: h[:evidence],
124
+ presentation: h[:presentation], requires_evidence: h[:requires_evidence]
114
125
  )
115
126
  end
116
127
 
128
+ # PRESENTATION: a tool whose job is to SHOW something, not to fetch it. The model
129
+ # picks ids, the engine validates them against the session's evidence ledger and
130
+ # joins the cards an evidence tool already returned (Tools::Present). No HTTP.
131
+ #
132
+ # "presentation" => { "component" => "product_cards", # what the channel renders
133
+ # "ids" => "product_ids", # the array:string param
134
+ # "max" => 8 } # 1..16, the attachment cap
135
+ #
136
+ # -> { component:, ids:, max: } | nil
137
+ def self.normalize_presentation(raw, schema)
138
+ return nil if raw.nil?
139
+
140
+ h = deep_symbolize(raw)
141
+ raise Insika::ValidationError, "presentation must be an object" unless h.is_a?(Hash)
142
+
143
+ component = h[:component].to_s
144
+ unless NAME_RE.match?(component)
145
+ raise Insika::ValidationError, "presentation.component must match #{NAME_RE.inspect}"
146
+ end
147
+
148
+ ids = h[:ids].to_s
149
+ prop = (schema["properties"] || {})[ids]
150
+ unless prop.is_a?(Hash) && prop["type"] == "array" && prop.dig("items", "type") == "string"
151
+ raise Insika::ValidationError,
152
+ "presentation.ids must name a declared array:string parameter (got #{ids.inspect})"
153
+ end
154
+
155
+ max = h[:max].nil? ? Insika::Evidence::MAX_ATTACHMENTS : Integer(h[:max], exception: false)
156
+ unless max.is_a?(Integer) && max.between?(1, Insika::Evidence::MAX_ATTACHMENTS)
157
+ raise Insika::ValidationError, "presentation.max must be 1..#{Insika::Evidence::MAX_ATTACHMENTS}"
158
+ end
159
+
160
+ { component: component, ids: ids, max: max }
161
+ end
162
+ private_class_method :normalize_presentation
163
+
164
+ def presentation? = !presentation.nil?
165
+
166
+ def self.normalize_requires_evidence(value, param_names)
167
+ return nil if value.nil?
168
+
169
+ params = value.is_a?(Hash) ? deep_symbolize(value)[:params] : value
170
+ unless params.is_a?(Array) && !params.empty? && params.all? { |p| param_names.include?(p.to_s) }
171
+ raise Insika::ValidationError, "requires_evidence needs a non-empty list of declared top-level parameters"
172
+ end
173
+
174
+ { "params" => params.map(&:to_s).uniq }
175
+ end
176
+ private_class_method :normalize_requires_evidence
177
+
117
178
  # Group: enablement label by DATA (not name convention),
118
179
  # target of AgentProfile's `tools_allow_groups`. Trimmed; empty/nil -> nil.
119
180
  def self.normalize_group(group)
@@ -414,7 +475,7 @@ module Insika
414
475
  h = {
415
476
  "name" => name, "description" => description,
416
477
  "parameters" => parameters,
417
- "request" => request.transform_keys(&:to_s),
478
+ "request" => request&.transform_keys(&:to_s),
418
479
  "response" => response.transform_keys(&:to_s),
419
480
  "secret_headers" => secret_headers,
420
481
  "side_effect" => side_effect, "timeout" => timeout,
@@ -424,6 +485,8 @@ module Insika
424
485
  # present only when declared — a tool without evidence is byte-identical
425
486
  # to today (no declaration, no envelope processing).
426
487
  h["evidence"] = evidence.to_h if evidence
488
+ h["requires_evidence"] = requires_evidence if requires_evidence
489
+ h["presentation"] = presentation.transform_keys(&:to_s) if presentation
427
490
  h
428
491
  end
429
492
 
@@ -12,6 +12,10 @@ module Insika
12
12
  # The tool loop belongs to RubyLLM; this is a decorator over the instances —
13
13
  # the Executor never drives roundtrips.
14
14
  class ToolEnvelope < SimpleDelegator
15
+ PROVENANCE_INSTRUCTION = "This value was not returned by any tool in this conversation. " \
16
+ "Find it with a tool that returns it — a search or a lookup by id — " \
17
+ "then call this tool again with an id from that result."
18
+
15
19
  # The tool timeout's OWN class: distinct from Async::TimeoutError so that
16
20
  # the rescue below NEVER swallows the TURN timeout (which uses the default of
17
21
  # with_timeout). Without this, a turn overflowing while the fiber is inside a
@@ -20,13 +24,21 @@ module Insika
20
24
  ToolTimeout = Class.new(StandardError)
21
25
  private_constant :ToolTimeout
22
26
 
27
+ # A gate's refusal. A plain Hash subclass: it reaches the model exactly as the
28
+ # `{status:, gate:, ...}` it always was, and the engine's own readers (the
29
+ # :tool_result outcome, the trace) recognize a refusal by CLASS — a data tool
30
+ # answering `{"status":"blocked","gate":"fraud_review"}` for a held order is
31
+ # not one, whatever keys it happens to use.
32
+ class Blocked < Hash; end
33
+
23
34
  def initialize(tool, state:, checkpoint_store:, tool_registry:, timeout:,
24
- skip_side_effects: [], trace_recorder: nil)
35
+ skip_side_effects: [], trace_recorder: nil, event_stream: nil)
25
36
  super(tool)
26
37
  @state = state
27
38
  @checkpoint_store = checkpoint_store
28
39
  @tool_registry = tool_registry
29
40
  @timeout = timeout
41
+ @event_stream = event_stream
30
42
  @skip_side_effects = Array(skip_side_effects) # ids already completed in the interrupted turn
31
43
  @trace_recorder = trace_recorder # duck-type: #record(session_id:, entry:). nil = no trace.
32
44
  end
@@ -41,6 +53,13 @@ module Insika
41
53
  call_id = correlation_id
42
54
  return { "skipped" => "already_executed" } if call_id && @skip_side_effects.include?(call_id)
43
55
 
56
+ started = monotonic
57
+ if (blocked = provenance_block(args))
58
+ trace(call_id, args, blocked, started)
59
+ emit_blocked(blocked)
60
+ return blocked
61
+ end
62
+
44
63
  # Approval gate: a tool marked `approval` suspends the turn in
45
64
  # :waiting until the operator resolves it. Delegates to the coordinator (the
46
65
  # Executor), which creates/queries the PendingAction and blocks via the
@@ -61,6 +80,7 @@ module Insika
61
80
  # on the ledger, hoard the attachments. No evidence = the result passes
62
81
  # through untouched (one nil-check — parity).
63
82
  result = process_evidence(result)
83
+ result = fence(result)
64
84
  record_side_effect!(call_id) if side_effect?
65
85
  trace(call_id, args, result, started)
66
86
  result
@@ -83,7 +103,59 @@ module Insika
83
103
  # wall-clock the model waited. No gate (the default, serial) = straight through.
84
104
  def with_gate(&)
85
105
  gate = @state.respond_to?(:tool_gate) ? @state.tool_gate : nil
86
- gate ? gate.acquire(&) : yield
106
+ return yield unless gate
107
+
108
+ serial = @state.respond_to?(:side_effect_gate) ? @state.side_effect_gate : nil
109
+ return gate.acquire(&) unless side_effect? && serial
110
+
111
+ # Backends may read-modify-write. Serialize writes before taking a slot,
112
+ # so queued writes cannot keep independent reads from running.
113
+ serial.acquire { gate.acquire(&) }
114
+ end
115
+
116
+ def provenance_block(args)
117
+ tool = __getobj__
118
+ requirement = tool.respond_to?(:requires_evidence) ? tool.requires_evidence : nil
119
+ return unless requirement
120
+
121
+ ledger = @state.respond_to?(:evidence_ledger) ? @state.evidence_ledger : nil
122
+ known = ledger ? ledger.ids : []
123
+ optional = optional_params(tool)
124
+ requirement.fetch("params").each do |param|
125
+ value = args.key?(param) ? args[param] : args[param.to_sym]
126
+ # A parameter the schema marks optional and the model left out carries no
127
+ # id to ground — nothing to check (a REQUIRED one left out is still a
128
+ # block: the write would run without the id the gate exists for).
129
+ next if value.nil? && ledger && optional.include?(param)
130
+
131
+ values = value.is_a?(Array) ? value : [value]
132
+ values = [nil] if values.empty? && !ledger
133
+ values.each do |id|
134
+ next if ledger && known.include?(id.to_s)
135
+
136
+ return Blocked[{ "status" => "blocked", "gate" => "provenance", "param" => param,
137
+ "value" => id.to_s, "instruction" => PROVENANCE_INSTRUCTION }]
138
+ end
139
+ end
140
+ nil
141
+ end
142
+
143
+ # The wrapped tool's top-level parameters NOT in the schema's `required`
144
+ # (DataDefinedTool exposes its definition's schema; a code tool exposes none
145
+ # -> every declared parameter is treated as required).
146
+ def optional_params(tool)
147
+ return [] unless tool.respond_to?(:params_schema) && (schema = tool.params_schema).is_a?(Hash)
148
+
149
+ (schema["properties"] || {}).keys.map(&:to_s) - Array(schema["required"]).map(&:to_s)
150
+ end
151
+
152
+ def emit_blocked(result)
153
+ task = @state.task # nil on a one-shot turn, like `trace` already assumes
154
+ @event_stream&.emit(Insika::Event.new(
155
+ type: :tool_blocked,
156
+ data: { name: real_name, gate: result["gate"], param: result["param"] },
157
+ meta: task ? { task_id: task.id, session_id: task.session_id } : {}
158
+ ))
87
159
  end
88
160
 
89
161
  # Records the call for debugging in the Studio (name + model args + result +
@@ -96,6 +168,7 @@ module Insika
96
168
  session_id: @state.task.session_id,
97
169
  entry: { "turn" => @state.turn, "tool" => real_name, "call_id" => call_id.to_s,
98
170
  "args" => args, "result" => result,
171
+ "gate" => result.is_a?(Blocked) ? result["gate"] : nil,
99
172
  "ms" => started ? ((monotonic - started) * 1000).round : nil,
100
173
  "at" => Time.now.utc.iso8601 }
101
174
  )
@@ -142,6 +215,31 @@ module Insika
142
215
  tool_call_id: call_id)
143
216
  end
144
217
 
218
+ # ---- fencing ----------------------------------------------
219
+
220
+ # After the evidence reshape (the lean envelope is already the shape the
221
+ # model reads): every String leaf sanitized, keys and non-strings untouched,
222
+ # each leaf capped at the platform's `fencing.max_chars`. Off = bytes
223
+ # identical to today. An error hash is engine-authored — never touched.
224
+ def fence(result)
225
+ return result unless Insika::Fence.enabled?(@state.profile)
226
+ return result if result.is_a?(Hash) && (result[:error] || result["error"])
227
+
228
+ max = (@state.respond_to?(:fence_max_chars) && @state.fence_max_chars) || Insika::Fence::DEFAULT_MAX_CHARS
229
+ return Insika::Fence.sanitize_value(result, max_chars: max) unless lean_evidence?(result)
230
+
231
+ # A lean evidence result: the LINES are third-party text, the IDS are keys.
232
+ # The ledger recorded the ids byte-exact and a presentation or a write joins
233
+ # on them, so NFKC must not touch them (a fullwidth digit in a SKU would
234
+ # stop matching the moment the model repeated it).
235
+ items = result["items"].map { |i| i.merge("line" => Insika::Fence.sanitize_text(i["line"].to_s, max_chars: max)) }
236
+ result.merge("items" => items)
237
+ end
238
+
239
+ def lean_evidence?(result)
240
+ evidence_spec && result.is_a?(Hash) && result["items"].is_a?(Array)
241
+ end
242
+
145
243
  # ---- evidence ---------------------------------------------
146
244
 
147
245
  # The evidence spec for the wrapped tool (D4). Resolution order:
@@ -204,6 +302,8 @@ module Insika
204
302
 
205
303
  @state.evidence_attachments ||= []
206
304
  @state.evidence_attachments.concat(attachments)
305
+ ledger = @state.respond_to?(:evidence_ledger) ? @state.evidence_ledger : nil
306
+ ledger.record_cards(attachments) if ledger.respond_to?(:record_cards)
207
307
  end
208
308
  end
209
309
  end
@@ -55,10 +55,13 @@ module Insika
55
55
  raise Insika::ValidationError, "tool '#{name}' already exists" if create_only && existing
56
56
 
57
57
  final = definition.to_h
58
- final["request"]["headers"] = reconcile_secret_headers(
59
- final["request"]["headers"], definition.secret_headers,
60
- existing&.dig("definition", "request", "headers")
61
- )
58
+ # A presentation tool makes no request — nothing to reconcile.
59
+ if final["request"]
60
+ final["request"]["headers"] = reconcile_secret_headers(
61
+ final["request"]["headers"], definition.secret_headers,
62
+ existing&.dig("definition", "request", "headers")
63
+ )
64
+ end
62
65
 
63
66
  rec = build_record(final, existing)
64
67
  @cs.put(SCOPE, name, rec)
@@ -110,6 +113,8 @@ module Insika
110
113
  secret = definition["secret_headers"] || []
111
114
  return definition if secret.empty?
112
115
 
116
+ return definition unless definition["request"]
117
+
113
118
  headers = (definition.dig("request", "headers") || {}).each_with_object({}) do |(k, v), acc|
114
119
  acc[k] = secret.include?(k) ? SecretMasking.mask(v) : v
115
120
  end
@@ -52,7 +52,7 @@ module Insika
52
52
  "ok" => ok?(e["result"]),
53
53
  "args" => clip(mask(e["args"])), "result" => clip(mask(e["result"])),
54
54
  "ms" => e["ms"], "at" => e["at"].to_s
55
- }
55
+ }.tap { |trace| trace["gate"] = e["gate"].to_s if e["gate"] }
56
56
  end
57
57
 
58
58
  # Conventional tool error = Hash with key "error"/:error (everything else is ok).
@@ -0,0 +1,172 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "time"
4
+
5
+ module Insika
6
+ # The tool AUDIT the per-session trace cannot answer: `tool_traces` records
7
+ # every call, but one session at a time, and nothing aggregates — so "which
8
+ # tools does this agent carry and never use?" had no surface at all. This
9
+ # report is that surface. Read-only by design: it names the candidates, the
10
+ # OPERATOR removes (a tool the report flags may still be the one a rare but
11
+ # critical flow needs).
12
+ #
13
+ # Attribution rides the task record: a session does not stamp its agent, but
14
+ # every task carries the agent in its command payload — tasks → sessions →
15
+ # tool_traces is the same read the Studio does. Three findings per agent:
16
+ #
17
+ # never_called — in `tools_allow`, zero calls in any stored trace. Dead
18
+ # weight: it costs schema tokens on every request and buys
19
+ # nothing (the pilot's `search_orders` is the known example).
20
+ # error_rate — called inside the window with > 30% conventional errors
21
+ # (the trace's own `ok` flag). Either the tool is broken or
22
+ # the model cannot hold its contract; both are operator work.
23
+ # stale — called at some point, but not once inside the window.
24
+ #
25
+ # Bounded and honest: it reads only what the stores kept (the trace caps at
26
+ # 200 entries/session), so a count here is "at least", never an exact total.
27
+ class ToolUsageReport
28
+ WINDOW_DAYS = 14
29
+ ERROR_RATE_THRESHOLD = 0.30
30
+
31
+ # One finding. kind: "never_called" | "error_rate" | "stale".
32
+ Row = Data.define(:agent, :tool, :kind, :detail) do
33
+ def to_h = { "agent" => agent, "tool" => tool, "kind" => kind, "detail" => detail }
34
+ end
35
+
36
+ Report = Data.define(:generated_at, :days, :agents, :rows) do
37
+ def to_h
38
+ { "generated_at" => generated_at, "days" => days, "agents" => agents,
39
+ "rows" => rows.map(&:to_h) }
40
+ end
41
+
42
+ # Human report, grouped by agent. Silent agents still print their header —
43
+ # "nothing flagged" is a result, not an omission.
44
+ def to_s
45
+ lines = ["tool usage — last #{days} day(s), generated #{generated_at}"]
46
+ agents.each do |agent|
47
+ mine = rows.select { |r| r.agent == agent }
48
+ lines << "" << "#{agent}: #{mine.empty? ? 'nothing flagged' : "#{mine.length} finding(s)"}"
49
+ mine.each { |r| lines << format(" %-13s %s — %s", r.kind, r.tool, r.detail) }
50
+ end
51
+ lines.join("\n")
52
+ end
53
+ end
54
+
55
+ def initialize(task_store:, tool_trace_store:, profile_source:, now: nil)
56
+ @task_store = task_store
57
+ @tool_trace_store = tool_trace_store
58
+ @profile_source = profile_source
59
+ @now = now
60
+ end
61
+
62
+ # -> Report. `agent:` narrows to one agent (must still be a stored profile).
63
+ def generate(days: WINDOW_DAYS, agent: nil)
64
+ now = @now || Time.now.utc
65
+ cutoff = now - (days * 24 * 60 * 60)
66
+ profiles = @profile_source.all_raw
67
+ profiles = profiles.select { |r| r["id"].to_s == agent.to_s } if agent
68
+ sessions = sessions_by_agent
69
+
70
+ rows = profiles.flat_map do |record|
71
+ id = record["id"].to_s
72
+ stats = tool_stats(sessions[id] || [], cutoff)
73
+ never_called_rows(id, record, stats) +
74
+ error_rate_rows(id, stats, days) +
75
+ stale_rows(id, stats) + blocked_rows(id, stats, days)
76
+ end
77
+
78
+ Report.new(generated_at: now.iso8601, days: days,
79
+ agents: profiles.map { |r| r["id"].to_s }.sort,
80
+ rows: rows.sort_by { |r| [r.agent, r.kind, r.tool] }.freeze)
81
+ end
82
+
83
+ private
84
+
85
+ # agent id -> [session ids], via the task records (the only place a session
86
+ # is tied to its agent). A task without agent or session (operator commands,
87
+ # workflows without a chat) contributes nothing.
88
+ def sessions_by_agent
89
+ acc = Hash.new { |h, k| h[k] = [] }
90
+ @task_store.each_id do |task_id|
91
+ task = @task_store.find(task_id) or next
92
+ agent = task.command.is_a?(Hash) ? task.command.dig("payload", "agent") : nil
93
+ next if agent.to_s.empty? || task.session_id.to_s.empty?
94
+
95
+ acc[agent.to_s] << task.session_id
96
+ end
97
+ acc.transform_values(&:uniq)
98
+ end
99
+
100
+ # tool name -> { calls:, errors:, window_calls:, window_errors:, last_at: }
101
+ # over every stored trace entry of the agent's sessions.
102
+ def tool_stats(session_ids, cutoff)
103
+ stats = Hash.new { |h, k| h[k] = { calls: 0, errors: 0, window_calls: 0, window_errors: 0, blocked: Hash.new(0), last_at: nil } }
104
+ session_ids.each do |sid|
105
+ @tool_trace_store.for_session(sid).each do |entry|
106
+ s = stats[entry["tool"].to_s]
107
+ at = parse_time(entry["at"])
108
+ error = entry["ok"] == false
109
+ s[:calls] += 1
110
+ s[:errors] += 1 if error
111
+ s[:last_at] = at if at && (s[:last_at].nil? || at > s[:last_at])
112
+ next unless at && at >= cutoff
113
+
114
+ s[:window_calls] += 1
115
+ s[:window_errors] += 1 if error
116
+ s[:blocked][entry["gate"]] += 1 if entry["gate"]
117
+ end
118
+ end
119
+ stats
120
+ end
121
+
122
+ def never_called_rows(agent, record, stats)
123
+ allow = record["tools_allow"]
124
+ return [] if allow.nil? # no allowlist declared -> nothing to audit against
125
+
126
+ Array(allow).map(&:to_s).reject { |t| stats.key?(t) }.map do |tool|
127
+ Row.new(agent: agent, tool: tool, kind: "never_called",
128
+ detail: "in tools_allow, never called in any stored trace — " \
129
+ "its schema still ships on every request")
130
+ end
131
+ end
132
+
133
+ def error_rate_rows(agent, stats, days)
134
+ stats.filter_map do |tool, s|
135
+ next if s[:window_calls].zero?
136
+
137
+ rate = s[:window_errors].to_f / s[:window_calls]
138
+ next if rate <= ERROR_RATE_THRESHOLD
139
+
140
+ Row.new(agent: agent, tool: tool, kind: "error_rate",
141
+ detail: "#{s[:window_errors]}/#{s[:window_calls]} call(s) errored in the last " \
142
+ "#{days} day(s) (#{(rate * 100).round}%)")
143
+ end
144
+ end
145
+
146
+ def blocked_rows(agent, stats, days)
147
+ stats.flat_map do |tool, s|
148
+ s[:blocked].map do |gate, count|
149
+ Row.new(agent: agent, tool: tool, kind: "blocked",
150
+ detail: "#{count} call(s) blocked by #{gate} in the last #{days} day(s)")
151
+ end
152
+ end
153
+ end
154
+
155
+ def stale_rows(agent, stats)
156
+ stats.filter_map do |tool, s|
157
+ next if s[:window_calls].positive? || s[:last_at].nil?
158
+
159
+ Row.new(agent: agent, tool: tool, kind: "stale",
160
+ detail: "last called #{s[:last_at].iso8601}, not once inside the window")
161
+ end
162
+ end
163
+
164
+ def parse_time(value)
165
+ return nil if value.to_s.empty?
166
+
167
+ Time.parse(value.to_s).utc
168
+ rescue ArgumentError
169
+ nil
170
+ end
171
+ end
172
+ end
@@ -47,6 +47,7 @@ module Insika
47
47
  # envelope's duck-typed resolution checks this FIRST — a data-tool declares
48
48
  # its evidence on its definition, never in the registry metadata.
49
49
  def evidence = @definition.evidence
50
+ def requires_evidence = @definition.requires_evidence
50
51
 
51
52
  # FULL (nested) JSON Schema straight into RubyLLM's params_schema — it is what
52
53
  # the providers serialize (OpenAI/Anthropic/Gemini/Bedrock prefer
@@ -0,0 +1,122 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "ruby_llm"
4
+
5
+ module Insika
6
+ module Tools
7
+ # PRESENTATION tool: the model says WHICH products to show, the engine decides
8
+ # WHAT is shown. Built by the overlay registry from a ToolDefinition that
9
+ # declares `presentation` (one class, N instances — like DataDefinedTool).
10
+ #
11
+ # Cards used to be a side effect of evidence: every attachment a search tool
12
+ # returned rode the channel delivery, all of them, and the model had no way to
13
+ # say "these three, not the ten I searched". The deployments compensated with
14
+ # inline markers in the text, parsed by regex on their side — where most of
15
+ # the hallucinated ids came from. Here text never carries product data: the
16
+ # model picks ids, and the engine
17
+ # 1. keeps only ids the session's evidence ledger has seen (provenance),
18
+ # 2. joins each to the card the evidence tool already hoarded this turn,
19
+ # 3. caps at the declared `max`,
20
+ # 4. records the selection on the turn (the delivery sends THAT),
21
+ # 5. emits `:ui` on the stream (the edge publishes it as `insika.ui`),
22
+ # 6. tells the model what was shown and what was dropped, and why.
23
+ # No HTTP, no model call, deterministic. `execute` never raises.
24
+ class Present < RubyLLM::Tool
25
+ # The one sentence for the model when NOTHING could be shown. Engine-generic:
26
+ # the pack tunes vocabulary through the tool's `description`.
27
+ NOTHING_SHOWN = "no card could be shown; name the products in text or search again"
28
+
29
+ def initialize(definition:, event_stream: nil)
30
+ @definition = definition
31
+ @event_stream = event_stream
32
+ @state = nil
33
+ super()
34
+ end
35
+
36
+ # Deposited by ToolAssembly before the envelope wraps the instance — the
37
+ # ledger and the hoarded cards live on the turn, never on the registry.
38
+ def turn_state=(state)
39
+ @state = state
40
+ end
41
+
42
+ def name = @definition.name
43
+ def description = @definition.description
44
+ def params_schema = @definition.parameters
45
+
46
+ def parameters
47
+ @parameters ||= @definition.top_level_params.each_with_object({}) do |p, acc|
48
+ sym = p[:name].to_sym
49
+ acc[sym] = RubyLLM::Parameter.new(sym, type: p[:type], desc: p[:description], required: p[:required])
50
+ end
51
+ end
52
+
53
+ def execute(**kwargs)
54
+ spec = @definition.presentation
55
+ ids = Array(kwargs[spec[:ids].to_sym]).map(&:to_s).reject(&:empty?).uniq
56
+ title = Coercion.presence(kwargs[:title])
57
+
58
+ shown, dropped = select(ids, spec[:max])
59
+ record(spec[:component], title, shown)
60
+ emit(spec[:component], title, shown, dropped)
61
+
62
+ out = { "shown" => shown.map { |c| c["id"] }, "dropped" => dropped }
63
+ out["instruction"] = NOTHING_SHOWN if shown.empty?
64
+ out
65
+ rescue StandardError => e
66
+ { error: "presentation failed: #{e.message}" }
67
+ end
68
+
69
+ private
70
+
71
+ # -> [shown cards, dropped [{id, reason}]]. Order preserved. Reasons:
72
+ # unknown — the ledger never saw the id (the model made it up, or the
73
+ # customer typed it); no ledger on the state reads as unknown too —
74
+ # a check that cannot verify does not pass.
75
+ # no_card — the id is known but no evidence tool returned a card for it
76
+ # this session (a text-only search result, or a seed of ids alone).
77
+ # max — beyond the declared cap.
78
+ # Cards come from this turn first, then from the ledger (the last few
79
+ # searches of the session) — "show me the second one" a turn later works.
80
+ def select(ids, max)
81
+ ledger = @state.respond_to?(:evidence_ledger) ? @state.evidence_ledger : nil
82
+ known = Array(ledger&.ids)
83
+ turn_cards = @state.respond_to?(:evidence_attachments) ? Array(@state.evidence_attachments) : []
84
+ session_cards = ledger.respond_to?(:cards) ? Array(ledger.cards) : []
85
+ by_id = (turn_cards + session_cards).each_with_object({}) { |c, h| h[c["id"]] ||= c if c["id"] }
86
+
87
+ shown = []
88
+ dropped = []
89
+ ids.each do |id|
90
+ if !known.include?(id) then dropped << { "id" => id, "reason" => "unknown" }
91
+ elsif (card = by_id[id]).nil? then dropped << { "id" => id, "reason" => "no_card" }
92
+ elsif shown.size >= max then dropped << { "id" => id, "reason" => "max" }
93
+ else shown << card
94
+ end
95
+ end
96
+ [shown, dropped]
97
+ end
98
+
99
+ def record(component, title, shown)
100
+ return unless @state.respond_to?(:presentations)
101
+
102
+ @state.presentations ||= []
103
+ @state.presentations << { "component" => component, "title" => title,
104
+ "items" => shown.map { |c| c.slice("id", "type", "url", "caption") } }
105
+ end
106
+
107
+ def emit(component, title, shown, dropped)
108
+ return unless @event_stream
109
+
110
+ task = @state.respond_to?(:task) ? @state.task : nil
111
+ meta = task ? { task_id: task.id, session_id: task.session_id } : {}
112
+ @event_stream.emit(Insika::Event.new(
113
+ type: :ui,
114
+ data: { component: component, title: title,
115
+ items: shown.map { |c| c.slice("id", "type", "url", "caption") },
116
+ count: shown.size, dropped: dropped },
117
+ meta: meta
118
+ ))
119
+ end
120
+ end
121
+ end
122
+ end
@@ -161,7 +161,7 @@ module Insika
161
161
  simulator = Insika::Evals::Simulator.new(transport: transport, ask: persona_ask, safety: safety)
162
162
 
163
163
  conv = "eval-#{golden.id}-#{SecureRandom.hex(4)}" # a fresh session every run (never reused)
164
- run = simulator.run(persona: golden.persona, agent: golden.agent, conv: conv)
164
+ run = simulator.run(persona: golden.persona, agent: golden.agent, conv: conv, state: golden.state)
165
165
  verdict = judge.score_conversation(
166
166
  rubric: golden.rubric, transcript: run.transcript, policy: golden.policy,
167
167
  min_score: golden.min_score || Insika::Evals::Judge::DEFAULT_MIN_SCORE
@@ -199,6 +199,11 @@ module Insika
199
199
  llm: llm_context
200
200
  )
201
201
  bus = Insika::CommandBus.new
202
+ # a golden with `state:` seeds through the same bus the turn runs on
203
+ bus.register(:seed_session, Insika::Commands::SeedSession.new(
204
+ session_store: @graph.session_store, memory_store: @graph.memory_store,
205
+ event_stream: @graph.event_stream
206
+ ))
202
207
  bus.register(:send_message, Insika::Commands::SendMessage.new(
203
208
  profiles: @graph.profiles, session_store: @graph.session_store,
204
209
  task_store: @graph.task_store, executor: executor,