insika 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +191 -0
- data/README.md +9 -6
- data/bin/insika +44 -3
- data/docs/AGENTS.md +74 -14
- data/docs/API.md +73 -0
- data/docs/ARCHITECTURE.md +45 -44
- data/docs/ARTIFACTS.md +42 -0
- data/docs/CHANNELS.md +19 -2
- data/docs/CONTEXT.md +86 -38
- data/docs/DEPLOY.md +27 -7
- data/docs/EVALS.md +98 -8
- data/docs/FACTS.md +4 -0
- data/docs/KNOWLEDGE.md +7 -0
- data/docs/LOADTEST.md +15 -27
- data/docs/MEDIA.md +1 -1
- data/docs/OBSERVABILITY.md +48 -4
- data/docs/POLICY.md +14 -5
- data/docs/RELEASING.md +4 -0
- data/docs/RUNNING-LOCAL.md +2 -2
- data/docs/SECURITY.md +28 -2
- data/docs/SOAK.md +1 -1
- data/docs/TOOLS.md +150 -32
- data/docs/prompts/ADD-TOOL.md +12 -2
- data/docs/prompts/DIAGNOSE-TURN.md +3 -0
- data/docs/prompts/GO-LIVE.md +6 -4
- data/lib/insika/agent_profile.rb +47 -10
- data/lib/insika/channels/web/widget.js +33 -0
- data/lib/insika/channels/web.rb +5 -2
- data/lib/insika/chat_builder.rb +90 -37
- data/lib/insika/commands/agent_payload.rb +1 -1
- data/lib/insika/commands/run_distillation.rb +5 -8
- data/lib/insika/commands/seed_session.rb +118 -0
- data/lib/insika/compaction.rb +196 -0
- data/lib/insika/context/builder.rb +35 -11
- data/lib/insika/context/fragment.rb +4 -1
- data/lib/insika/context/priority.rb +8 -0
- data/lib/insika/context/provider.rb +5 -0
- data/lib/insika/context/providers/briefing.rb +61 -29
- data/lib/insika/context/providers/fence_notice.rb +27 -0
- data/lib/insika/context/providers/knowledge.rb +7 -4
- data/lib/insika/context/providers/memory.rb +8 -4
- data/lib/insika/context/providers/session.rb +50 -10
- data/lib/insika/context_trace_store.rb +11 -1
- data/lib/insika/doctor.rb +213 -10
- data/lib/insika/dsl/runtime.rb +5 -0
- data/lib/insika/dsl.rb +6 -0
- data/lib/insika/edge_limiter.rb +4 -1
- data/lib/insika/env_schema.rb +5 -6
- data/lib/insika/errors.rb +1 -0
- data/lib/insika/evals/assertions.rb +92 -6
- data/lib/insika/evals/golden.rb +91 -2
- data/lib/insika/evals/runner.rb +20 -0
- data/lib/insika/evals/simulator.rb +11 -2
- data/lib/insika/evals/transport.rb +118 -16
- data/lib/insika/evidence.rb +79 -12
- data/lib/insika/executor.rb +94 -23
- data/lib/insika/fence.rb +96 -0
- data/lib/insika/golden_store.rb +3 -0
- data/lib/insika/loop_detector.rb +5 -34
- data/lib/insika/mcp_store.rb +5 -2
- data/lib/insika/mcp_tool_registry.rb +8 -1
- data/lib/insika/memory_store.rb +12 -0
- data/lib/insika/overlay_tool_registry.rb +5 -0
- data/lib/insika/prefix_fingerprint.rb +32 -27
- data/lib/insika/profile_source.rb +8 -0
- data/lib/insika/server/app.rb +43 -1
- data/lib/insika/server/rack_app.rb +2 -0
- data/lib/insika/server/responses.rb +35 -8
- data/lib/insika/session_store.rb +38 -5
- data/lib/insika/settings_store.rb +18 -2
- data/lib/insika/soak/runner.rb +4 -4
- data/lib/insika/spoken_transcript.rb +31 -0
- data/lib/insika/studio/app.rb +34 -8
- data/lib/insika/studio/forms.rb +29 -3
- data/lib/insika/studio/views/_agent_tab_config.erb +5 -1
- data/lib/insika/studio/views/session.erb +1 -1
- data/lib/insika/studio/views/settings.erb +11 -0
- data/lib/insika/studio/views/tool_edit.erb +6 -2
- data/lib/insika/telemetry/recorder.rb +61 -1
- data/lib/insika/templates/daily-digest/README.md +9 -0
- data/lib/insika/templates/research-analyst/agent.rb +10 -0
- data/lib/insika/tool_assembly.rb +21 -13
- data/lib/insika/tool_batch.rb +67 -0
- data/lib/insika/tool_definition.rb +73 -10
- data/lib/insika/tool_envelope.rb +102 -2
- data/lib/insika/tool_store.rb +9 -4
- data/lib/insika/tool_trace_store.rb +1 -1
- data/lib/insika/tool_usage_report.rb +172 -0
- data/lib/insika/tools/data_defined_tool.rb +1 -0
- data/lib/insika/tools/present.rb +122 -0
- data/lib/insika/tools/run_persona_eval.rb +6 -1
- data/lib/insika/tools/tool_search.rb +4 -2
- data/lib/insika/turn_budget.rb +91 -0
- data/lib/insika/turn_state.rb +13 -1
- data/lib/insika/version.rb +1 -1
- data/lib/insika/wiring/graph.rb +7 -0
- data/lib/insika/wiring/graph_chat.rb +4 -0
- data/lib/insika.rb +11 -0
- metadata +10 -1
|
@@ -35,7 +35,9 @@ module Insika
|
|
|
35
35
|
ToolDefinition = Data.define(
|
|
36
36
|
:name, :description, :parameters, :request, :response,
|
|
37
37
|
:secret_headers, :side_effect, :timeout, :group, :tags, :halt_when,
|
|
38
|
-
:evidence
|
|
38
|
+
:evidence, # Insika::Evidence::Spec | nil
|
|
39
|
+
:requires_evidence, # { "params" => [String] } | nil
|
|
40
|
+
:presentation # { component:, ids:, max: } | nil — a UI tool, no HTTP
|
|
39
41
|
)
|
|
40
42
|
|
|
41
43
|
class ToolDefinition
|
|
@@ -73,9 +75,9 @@ module Insika
|
|
|
73
75
|
# Builds + validates. Raises Insika::ValidationError. Accepts keyword args
|
|
74
76
|
# (already-normalized symbol keys); use from_h for a raw Hash from the store/UI.
|
|
75
77
|
# `parameters` accepts JSON Schema (Hash) OR the legacy flat array.
|
|
76
|
-
def self.build(name:, description:, request
|
|
78
|
+
def self.build(name:, description:, request: nil, parameters: nil, response: nil,
|
|
77
79
|
secret_headers: nil, side_effect: nil, timeout: nil, group: nil, tags: nil,
|
|
78
|
-
halt_when: nil, evidence: nil)
|
|
80
|
+
halt_when: nil, evidence: nil, presentation: nil, requires_evidence: nil)
|
|
79
81
|
name = name.to_s
|
|
80
82
|
raise Insika::ValidationError, "name must match #{NAME_RE.inspect}" unless NAME_RE.match?(name)
|
|
81
83
|
|
|
@@ -83,15 +85,21 @@ module Insika
|
|
|
83
85
|
raise Insika::ValidationError, "description is required" if desc.empty?
|
|
84
86
|
|
|
85
87
|
schema = normalize_params(parameters)
|
|
86
|
-
|
|
88
|
+
# A tool is EITHER an HTTP call or a presentation — never both, never neither.
|
|
89
|
+
if request && presentation
|
|
90
|
+
raise Insika::ValidationError, "a tool declares either 'request' or 'presentation', not both"
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
pres = normalize_presentation(presentation, schema)
|
|
94
|
+
req = pres ? nil : normalize_request(request || {}, top_level_names(schema))
|
|
87
95
|
resp = normalize_response(response)
|
|
88
96
|
if resp[:extract] == "evidence_envelope" && evidence.nil?
|
|
89
97
|
raise Insika::ValidationError,
|
|
90
98
|
"extract 'evidence_envelope' requires an 'evidence' declaration"
|
|
91
99
|
end
|
|
92
100
|
|
|
93
|
-
method = req[:method]
|
|
94
|
-
effect = side_effect.nil? ? !IDEMPOTENT.include?(method) : (side_effect ? true : false)
|
|
101
|
+
method = req && req[:method]
|
|
102
|
+
effect = side_effect.nil? ? !(method.nil? || IDEMPOTENT.include?(method)) : (side_effect ? true : false)
|
|
95
103
|
|
|
96
104
|
new(
|
|
97
105
|
name: name, description: desc, parameters: schema, request: req, response: resp,
|
|
@@ -99,7 +107,9 @@ module Insika
|
|
|
99
107
|
timeout: timeout.nil? ? nil : Integer(timeout),
|
|
100
108
|
group: normalize_group(group), tags: normalize_tags(tags),
|
|
101
109
|
halt_when: normalize_halt_when(halt_when),
|
|
102
|
-
evidence: Insika::Evidence::Spec.parse(evidence)
|
|
110
|
+
evidence: Insika::Evidence::Spec.parse(evidence),
|
|
111
|
+
presentation: pres,
|
|
112
|
+
requires_evidence: normalize_requires_evidence(requires_evidence, top_level_names(schema))
|
|
103
113
|
)
|
|
104
114
|
end
|
|
105
115
|
|
|
@@ -108,12 +118,63 @@ module Insika
|
|
|
108
118
|
h = deep_symbolize(hash)
|
|
109
119
|
build(
|
|
110
120
|
name: h[:name], description: h[:description], parameters: h[:parameters],
|
|
111
|
-
request: h[:request]
|
|
121
|
+
request: h[:request], response: h[:response],
|
|
112
122
|
secret_headers: h[:secret_headers], side_effect: h[:side_effect], timeout: h[:timeout],
|
|
113
|
-
group: h[:group], tags: h[:tags], halt_when: h[:halt_when], evidence: h[:evidence]
|
|
123
|
+
group: h[:group], tags: h[:tags], halt_when: h[:halt_when], evidence: h[:evidence],
|
|
124
|
+
presentation: h[:presentation], requires_evidence: h[:requires_evidence]
|
|
114
125
|
)
|
|
115
126
|
end
|
|
116
127
|
|
|
128
|
+
# PRESENTATION: a tool whose job is to SHOW something, not to fetch it. The model
|
|
129
|
+
# picks ids, the engine validates them against the session's evidence ledger and
|
|
130
|
+
# joins the cards an evidence tool already returned (Tools::Present). No HTTP.
|
|
131
|
+
#
|
|
132
|
+
# "presentation" => { "component" => "product_cards", # what the channel renders
|
|
133
|
+
# "ids" => "product_ids", # the array:string param
|
|
134
|
+
# "max" => 8 } # 1..16, the attachment cap
|
|
135
|
+
#
|
|
136
|
+
# -> { component:, ids:, max: } | nil
|
|
137
|
+
def self.normalize_presentation(raw, schema)
|
|
138
|
+
return nil if raw.nil?
|
|
139
|
+
|
|
140
|
+
h = deep_symbolize(raw)
|
|
141
|
+
raise Insika::ValidationError, "presentation must be an object" unless h.is_a?(Hash)
|
|
142
|
+
|
|
143
|
+
component = h[:component].to_s
|
|
144
|
+
unless NAME_RE.match?(component)
|
|
145
|
+
raise Insika::ValidationError, "presentation.component must match #{NAME_RE.inspect}"
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
ids = h[:ids].to_s
|
|
149
|
+
prop = (schema["properties"] || {})[ids]
|
|
150
|
+
unless prop.is_a?(Hash) && prop["type"] == "array" && prop.dig("items", "type") == "string"
|
|
151
|
+
raise Insika::ValidationError,
|
|
152
|
+
"presentation.ids must name a declared array:string parameter (got #{ids.inspect})"
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
max = h[:max].nil? ? Insika::Evidence::MAX_ATTACHMENTS : Integer(h[:max], exception: false)
|
|
156
|
+
unless max.is_a?(Integer) && max.between?(1, Insika::Evidence::MAX_ATTACHMENTS)
|
|
157
|
+
raise Insika::ValidationError, "presentation.max must be 1..#{Insika::Evidence::MAX_ATTACHMENTS}"
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
{ component: component, ids: ids, max: max }
|
|
161
|
+
end
|
|
162
|
+
private_class_method :normalize_presentation
|
|
163
|
+
|
|
164
|
+
def presentation? = !presentation.nil?
|
|
165
|
+
|
|
166
|
+
def self.normalize_requires_evidence(value, param_names)
|
|
167
|
+
return nil if value.nil?
|
|
168
|
+
|
|
169
|
+
params = value.is_a?(Hash) ? deep_symbolize(value)[:params] : value
|
|
170
|
+
unless params.is_a?(Array) && !params.empty? && params.all? { |p| param_names.include?(p.to_s) }
|
|
171
|
+
raise Insika::ValidationError, "requires_evidence needs a non-empty list of declared top-level parameters"
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
{ "params" => params.map(&:to_s).uniq }
|
|
175
|
+
end
|
|
176
|
+
private_class_method :normalize_requires_evidence
|
|
177
|
+
|
|
117
178
|
# Group: enablement label by DATA (not name convention),
|
|
118
179
|
# target of AgentProfile's `tools_allow_groups`. Trimmed; empty/nil -> nil.
|
|
119
180
|
def self.normalize_group(group)
|
|
@@ -414,7 +475,7 @@ module Insika
|
|
|
414
475
|
h = {
|
|
415
476
|
"name" => name, "description" => description,
|
|
416
477
|
"parameters" => parameters,
|
|
417
|
-
"request" => request
|
|
478
|
+
"request" => request&.transform_keys(&:to_s),
|
|
418
479
|
"response" => response.transform_keys(&:to_s),
|
|
419
480
|
"secret_headers" => secret_headers,
|
|
420
481
|
"side_effect" => side_effect, "timeout" => timeout,
|
|
@@ -424,6 +485,8 @@ module Insika
|
|
|
424
485
|
# present only when declared — a tool without evidence is byte-identical
|
|
425
486
|
# to today (no declaration, no envelope processing).
|
|
426
487
|
h["evidence"] = evidence.to_h if evidence
|
|
488
|
+
h["requires_evidence"] = requires_evidence if requires_evidence
|
|
489
|
+
h["presentation"] = presentation.transform_keys(&:to_s) if presentation
|
|
427
490
|
h
|
|
428
491
|
end
|
|
429
492
|
|
data/lib/insika/tool_envelope.rb
CHANGED
|
@@ -12,6 +12,10 @@ module Insika
|
|
|
12
12
|
# The tool loop belongs to RubyLLM; this is a decorator over the instances —
|
|
13
13
|
# the Executor never drives roundtrips.
|
|
14
14
|
class ToolEnvelope < SimpleDelegator
|
|
15
|
+
PROVENANCE_INSTRUCTION = "This value was not returned by any tool in this conversation. " \
|
|
16
|
+
"Find it with a tool that returns it — a search or a lookup by id — " \
|
|
17
|
+
"then call this tool again with an id from that result."
|
|
18
|
+
|
|
15
19
|
# The tool timeout's OWN class: distinct from Async::TimeoutError so that
|
|
16
20
|
# the rescue below NEVER swallows the TURN timeout (which uses the default of
|
|
17
21
|
# with_timeout). Without this, a turn overflowing while the fiber is inside a
|
|
@@ -20,13 +24,21 @@ module Insika
|
|
|
20
24
|
ToolTimeout = Class.new(StandardError)
|
|
21
25
|
private_constant :ToolTimeout
|
|
22
26
|
|
|
27
|
+
# A gate's refusal. A plain Hash subclass: it reaches the model exactly as the
|
|
28
|
+
# `{status:, gate:, ...}` it always was, and the engine's own readers (the
|
|
29
|
+
# :tool_result outcome, the trace) recognize a refusal by CLASS — a data tool
|
|
30
|
+
# answering `{"status":"blocked","gate":"fraud_review"}` for a held order is
|
|
31
|
+
# not one, whatever keys it happens to use.
|
|
32
|
+
class Blocked < Hash; end
|
|
33
|
+
|
|
23
34
|
def initialize(tool, state:, checkpoint_store:, tool_registry:, timeout:,
|
|
24
|
-
skip_side_effects: [], trace_recorder: nil)
|
|
35
|
+
skip_side_effects: [], trace_recorder: nil, event_stream: nil)
|
|
25
36
|
super(tool)
|
|
26
37
|
@state = state
|
|
27
38
|
@checkpoint_store = checkpoint_store
|
|
28
39
|
@tool_registry = tool_registry
|
|
29
40
|
@timeout = timeout
|
|
41
|
+
@event_stream = event_stream
|
|
30
42
|
@skip_side_effects = Array(skip_side_effects) # ids already completed in the interrupted turn
|
|
31
43
|
@trace_recorder = trace_recorder # duck-type: #record(session_id:, entry:). nil = no trace.
|
|
32
44
|
end
|
|
@@ -41,6 +53,13 @@ module Insika
|
|
|
41
53
|
call_id = correlation_id
|
|
42
54
|
return { "skipped" => "already_executed" } if call_id && @skip_side_effects.include?(call_id)
|
|
43
55
|
|
|
56
|
+
started = monotonic
|
|
57
|
+
if (blocked = provenance_block(args))
|
|
58
|
+
trace(call_id, args, blocked, started)
|
|
59
|
+
emit_blocked(blocked)
|
|
60
|
+
return blocked
|
|
61
|
+
end
|
|
62
|
+
|
|
44
63
|
# Approval gate: a tool marked `approval` suspends the turn in
|
|
45
64
|
# :waiting until the operator resolves it. Delegates to the coordinator (the
|
|
46
65
|
# Executor), which creates/queries the PendingAction and blocks via the
|
|
@@ -61,6 +80,7 @@ module Insika
|
|
|
61
80
|
# on the ledger, hoard the attachments. No evidence = the result passes
|
|
62
81
|
# through untouched (one nil-check — parity).
|
|
63
82
|
result = process_evidence(result)
|
|
83
|
+
result = fence(result)
|
|
64
84
|
record_side_effect!(call_id) if side_effect?
|
|
65
85
|
trace(call_id, args, result, started)
|
|
66
86
|
result
|
|
@@ -83,7 +103,59 @@ module Insika
|
|
|
83
103
|
# wall-clock the model waited. No gate (the default, serial) = straight through.
|
|
84
104
|
def with_gate(&)
|
|
85
105
|
gate = @state.respond_to?(:tool_gate) ? @state.tool_gate : nil
|
|
86
|
-
|
|
106
|
+
return yield unless gate
|
|
107
|
+
|
|
108
|
+
serial = @state.respond_to?(:side_effect_gate) ? @state.side_effect_gate : nil
|
|
109
|
+
return gate.acquire(&) unless side_effect? && serial
|
|
110
|
+
|
|
111
|
+
# Backends may read-modify-write. Serialize writes before taking a slot,
|
|
112
|
+
# so queued writes cannot keep independent reads from running.
|
|
113
|
+
serial.acquire { gate.acquire(&) }
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def provenance_block(args)
|
|
117
|
+
tool = __getobj__
|
|
118
|
+
requirement = tool.respond_to?(:requires_evidence) ? tool.requires_evidence : nil
|
|
119
|
+
return unless requirement
|
|
120
|
+
|
|
121
|
+
ledger = @state.respond_to?(:evidence_ledger) ? @state.evidence_ledger : nil
|
|
122
|
+
known = ledger ? ledger.ids : []
|
|
123
|
+
optional = optional_params(tool)
|
|
124
|
+
requirement.fetch("params").each do |param|
|
|
125
|
+
value = args.key?(param) ? args[param] : args[param.to_sym]
|
|
126
|
+
# A parameter the schema marks optional and the model left out carries no
|
|
127
|
+
# id to ground — nothing to check (a REQUIRED one left out is still a
|
|
128
|
+
# block: the write would run without the id the gate exists for).
|
|
129
|
+
next if value.nil? && ledger && optional.include?(param)
|
|
130
|
+
|
|
131
|
+
values = value.is_a?(Array) ? value : [value]
|
|
132
|
+
values = [nil] if values.empty? && !ledger
|
|
133
|
+
values.each do |id|
|
|
134
|
+
next if ledger && known.include?(id.to_s)
|
|
135
|
+
|
|
136
|
+
return Blocked[{ "status" => "blocked", "gate" => "provenance", "param" => param,
|
|
137
|
+
"value" => id.to_s, "instruction" => PROVENANCE_INSTRUCTION }]
|
|
138
|
+
end
|
|
139
|
+
end
|
|
140
|
+
nil
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# The wrapped tool's top-level parameters NOT in the schema's `required`
|
|
144
|
+
# (DataDefinedTool exposes its definition's schema; a code tool exposes none
|
|
145
|
+
# -> every declared parameter is treated as required).
|
|
146
|
+
def optional_params(tool)
|
|
147
|
+
return [] unless tool.respond_to?(:params_schema) && (schema = tool.params_schema).is_a?(Hash)
|
|
148
|
+
|
|
149
|
+
(schema["properties"] || {}).keys.map(&:to_s) - Array(schema["required"]).map(&:to_s)
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def emit_blocked(result)
|
|
153
|
+
task = @state.task # nil on a one-shot turn, like `trace` already assumes
|
|
154
|
+
@event_stream&.emit(Insika::Event.new(
|
|
155
|
+
type: :tool_blocked,
|
|
156
|
+
data: { name: real_name, gate: result["gate"], param: result["param"] },
|
|
157
|
+
meta: task ? { task_id: task.id, session_id: task.session_id } : {}
|
|
158
|
+
))
|
|
87
159
|
end
|
|
88
160
|
|
|
89
161
|
# Records the call for debugging in the Studio (name + model args + result +
|
|
@@ -96,6 +168,7 @@ module Insika
|
|
|
96
168
|
session_id: @state.task.session_id,
|
|
97
169
|
entry: { "turn" => @state.turn, "tool" => real_name, "call_id" => call_id.to_s,
|
|
98
170
|
"args" => args, "result" => result,
|
|
171
|
+
"gate" => result.is_a?(Blocked) ? result["gate"] : nil,
|
|
99
172
|
"ms" => started ? ((monotonic - started) * 1000).round : nil,
|
|
100
173
|
"at" => Time.now.utc.iso8601 }
|
|
101
174
|
)
|
|
@@ -142,6 +215,31 @@ module Insika
|
|
|
142
215
|
tool_call_id: call_id)
|
|
143
216
|
end
|
|
144
217
|
|
|
218
|
+
# ---- fencing ----------------------------------------------
|
|
219
|
+
|
|
220
|
+
# After the evidence reshape (the lean envelope is already the shape the
|
|
221
|
+
# model reads): every String leaf sanitized, keys and non-strings untouched,
|
|
222
|
+
# each leaf capped at the platform's `fencing.max_chars`. Off = bytes
|
|
223
|
+
# identical to today. An error hash is engine-authored — never touched.
|
|
224
|
+
def fence(result)
|
|
225
|
+
return result unless Insika::Fence.enabled?(@state.profile)
|
|
226
|
+
return result if result.is_a?(Hash) && (result[:error] || result["error"])
|
|
227
|
+
|
|
228
|
+
max = (@state.respond_to?(:fence_max_chars) && @state.fence_max_chars) || Insika::Fence::DEFAULT_MAX_CHARS
|
|
229
|
+
return Insika::Fence.sanitize_value(result, max_chars: max) unless lean_evidence?(result)
|
|
230
|
+
|
|
231
|
+
# A lean evidence result: the LINES are third-party text, the IDS are keys.
|
|
232
|
+
# The ledger recorded the ids byte-exact and a presentation or a write joins
|
|
233
|
+
# on them, so NFKC must not touch them (a fullwidth digit in a SKU would
|
|
234
|
+
# stop matching the moment the model repeated it).
|
|
235
|
+
items = result["items"].map { |i| i.merge("line" => Insika::Fence.sanitize_text(i["line"].to_s, max_chars: max)) }
|
|
236
|
+
result.merge("items" => items)
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
def lean_evidence?(result)
|
|
240
|
+
evidence_spec && result.is_a?(Hash) && result["items"].is_a?(Array)
|
|
241
|
+
end
|
|
242
|
+
|
|
145
243
|
# ---- evidence ---------------------------------------------
|
|
146
244
|
|
|
147
245
|
# The evidence spec for the wrapped tool (D4). Resolution order:
|
|
@@ -204,6 +302,8 @@ module Insika
|
|
|
204
302
|
|
|
205
303
|
@state.evidence_attachments ||= []
|
|
206
304
|
@state.evidence_attachments.concat(attachments)
|
|
305
|
+
ledger = @state.respond_to?(:evidence_ledger) ? @state.evidence_ledger : nil
|
|
306
|
+
ledger.record_cards(attachments) if ledger.respond_to?(:record_cards)
|
|
207
307
|
end
|
|
208
308
|
end
|
|
209
309
|
end
|
data/lib/insika/tool_store.rb
CHANGED
|
@@ -55,10 +55,13 @@ module Insika
|
|
|
55
55
|
raise Insika::ValidationError, "tool '#{name}' already exists" if create_only && existing
|
|
56
56
|
|
|
57
57
|
final = definition.to_h
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
58
|
+
# A presentation tool makes no request — nothing to reconcile.
|
|
59
|
+
if final["request"]
|
|
60
|
+
final["request"]["headers"] = reconcile_secret_headers(
|
|
61
|
+
final["request"]["headers"], definition.secret_headers,
|
|
62
|
+
existing&.dig("definition", "request", "headers")
|
|
63
|
+
)
|
|
64
|
+
end
|
|
62
65
|
|
|
63
66
|
rec = build_record(final, existing)
|
|
64
67
|
@cs.put(SCOPE, name, rec)
|
|
@@ -110,6 +113,8 @@ module Insika
|
|
|
110
113
|
secret = definition["secret_headers"] || []
|
|
111
114
|
return definition if secret.empty?
|
|
112
115
|
|
|
116
|
+
return definition unless definition["request"]
|
|
117
|
+
|
|
113
118
|
headers = (definition.dig("request", "headers") || {}).each_with_object({}) do |(k, v), acc|
|
|
114
119
|
acc[k] = secret.include?(k) ? SecretMasking.mask(v) : v
|
|
115
120
|
end
|
|
@@ -52,7 +52,7 @@ module Insika
|
|
|
52
52
|
"ok" => ok?(e["result"]),
|
|
53
53
|
"args" => clip(mask(e["args"])), "result" => clip(mask(e["result"])),
|
|
54
54
|
"ms" => e["ms"], "at" => e["at"].to_s
|
|
55
|
-
}
|
|
55
|
+
}.tap { |trace| trace["gate"] = e["gate"].to_s if e["gate"] }
|
|
56
56
|
end
|
|
57
57
|
|
|
58
58
|
# Conventional tool error = Hash with key "error"/:error (everything else is ok).
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "time"
|
|
4
|
+
|
|
5
|
+
module Insika
|
|
6
|
+
# The tool AUDIT the per-session trace cannot answer: `tool_traces` records
|
|
7
|
+
# every call, but one session at a time, and nothing aggregates — so "which
|
|
8
|
+
# tools does this agent carry and never use?" had no surface at all. This
|
|
9
|
+
# report is that surface. Read-only by design: it names the candidates, the
|
|
10
|
+
# OPERATOR removes (a tool the report flags may still be the one a rare but
|
|
11
|
+
# critical flow needs).
|
|
12
|
+
#
|
|
13
|
+
# Attribution rides the task record: a session does not stamp its agent, but
|
|
14
|
+
# every task carries the agent in its command payload — tasks → sessions →
|
|
15
|
+
# tool_traces is the same read the Studio does. Three findings per agent:
|
|
16
|
+
#
|
|
17
|
+
# never_called — in `tools_allow`, zero calls in any stored trace. Dead
|
|
18
|
+
# weight: it costs schema tokens on every request and buys
|
|
19
|
+
# nothing (the pilot's `search_orders` is the known example).
|
|
20
|
+
# error_rate — called inside the window with > 30% conventional errors
|
|
21
|
+
# (the trace's own `ok` flag). Either the tool is broken or
|
|
22
|
+
# the model cannot hold its contract; both are operator work.
|
|
23
|
+
# stale — called at some point, but not once inside the window.
|
|
24
|
+
#
|
|
25
|
+
# Bounded and honest: it reads only what the stores kept (the trace caps at
|
|
26
|
+
# 200 entries/session), so a count here is "at least", never an exact total.
|
|
27
|
+
class ToolUsageReport
|
|
28
|
+
WINDOW_DAYS = 14
|
|
29
|
+
ERROR_RATE_THRESHOLD = 0.30
|
|
30
|
+
|
|
31
|
+
# One finding. kind: "never_called" | "error_rate" | "stale".
|
|
32
|
+
Row = Data.define(:agent, :tool, :kind, :detail) do
|
|
33
|
+
def to_h = { "agent" => agent, "tool" => tool, "kind" => kind, "detail" => detail }
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
Report = Data.define(:generated_at, :days, :agents, :rows) do
|
|
37
|
+
def to_h
|
|
38
|
+
{ "generated_at" => generated_at, "days" => days, "agents" => agents,
|
|
39
|
+
"rows" => rows.map(&:to_h) }
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# Human report, grouped by agent. Silent agents still print their header —
|
|
43
|
+
# "nothing flagged" is a result, not an omission.
|
|
44
|
+
def to_s
|
|
45
|
+
lines = ["tool usage — last #{days} day(s), generated #{generated_at}"]
|
|
46
|
+
agents.each do |agent|
|
|
47
|
+
mine = rows.select { |r| r.agent == agent }
|
|
48
|
+
lines << "" << "#{agent}: #{mine.empty? ? 'nothing flagged' : "#{mine.length} finding(s)"}"
|
|
49
|
+
mine.each { |r| lines << format(" %-13s %s — %s", r.kind, r.tool, r.detail) }
|
|
50
|
+
end
|
|
51
|
+
lines.join("\n")
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def initialize(task_store:, tool_trace_store:, profile_source:, now: nil)
|
|
56
|
+
@task_store = task_store
|
|
57
|
+
@tool_trace_store = tool_trace_store
|
|
58
|
+
@profile_source = profile_source
|
|
59
|
+
@now = now
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# -> Report. `agent:` narrows to one agent (must still be a stored profile).
|
|
63
|
+
def generate(days: WINDOW_DAYS, agent: nil)
|
|
64
|
+
now = @now || Time.now.utc
|
|
65
|
+
cutoff = now - (days * 24 * 60 * 60)
|
|
66
|
+
profiles = @profile_source.all_raw
|
|
67
|
+
profiles = profiles.select { |r| r["id"].to_s == agent.to_s } if agent
|
|
68
|
+
sessions = sessions_by_agent
|
|
69
|
+
|
|
70
|
+
rows = profiles.flat_map do |record|
|
|
71
|
+
id = record["id"].to_s
|
|
72
|
+
stats = tool_stats(sessions[id] || [], cutoff)
|
|
73
|
+
never_called_rows(id, record, stats) +
|
|
74
|
+
error_rate_rows(id, stats, days) +
|
|
75
|
+
stale_rows(id, stats) + blocked_rows(id, stats, days)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
Report.new(generated_at: now.iso8601, days: days,
|
|
79
|
+
agents: profiles.map { |r| r["id"].to_s }.sort,
|
|
80
|
+
rows: rows.sort_by { |r| [r.agent, r.kind, r.tool] }.freeze)
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
private
|
|
84
|
+
|
|
85
|
+
# agent id -> [session ids], via the task records (the only place a session
|
|
86
|
+
# is tied to its agent). A task without agent or session (operator commands,
|
|
87
|
+
# workflows without a chat) contributes nothing.
|
|
88
|
+
def sessions_by_agent
|
|
89
|
+
acc = Hash.new { |h, k| h[k] = [] }
|
|
90
|
+
@task_store.each_id do |task_id|
|
|
91
|
+
task = @task_store.find(task_id) or next
|
|
92
|
+
agent = task.command.is_a?(Hash) ? task.command.dig("payload", "agent") : nil
|
|
93
|
+
next if agent.to_s.empty? || task.session_id.to_s.empty?
|
|
94
|
+
|
|
95
|
+
acc[agent.to_s] << task.session_id
|
|
96
|
+
end
|
|
97
|
+
acc.transform_values(&:uniq)
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# tool name -> { calls:, errors:, window_calls:, window_errors:, last_at: }
|
|
101
|
+
# over every stored trace entry of the agent's sessions.
|
|
102
|
+
def tool_stats(session_ids, cutoff)
|
|
103
|
+
stats = Hash.new { |h, k| h[k] = { calls: 0, errors: 0, window_calls: 0, window_errors: 0, blocked: Hash.new(0), last_at: nil } }
|
|
104
|
+
session_ids.each do |sid|
|
|
105
|
+
@tool_trace_store.for_session(sid).each do |entry|
|
|
106
|
+
s = stats[entry["tool"].to_s]
|
|
107
|
+
at = parse_time(entry["at"])
|
|
108
|
+
error = entry["ok"] == false
|
|
109
|
+
s[:calls] += 1
|
|
110
|
+
s[:errors] += 1 if error
|
|
111
|
+
s[:last_at] = at if at && (s[:last_at].nil? || at > s[:last_at])
|
|
112
|
+
next unless at && at >= cutoff
|
|
113
|
+
|
|
114
|
+
s[:window_calls] += 1
|
|
115
|
+
s[:window_errors] += 1 if error
|
|
116
|
+
s[:blocked][entry["gate"]] += 1 if entry["gate"]
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
stats
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def never_called_rows(agent, record, stats)
|
|
123
|
+
allow = record["tools_allow"]
|
|
124
|
+
return [] if allow.nil? # no allowlist declared -> nothing to audit against
|
|
125
|
+
|
|
126
|
+
Array(allow).map(&:to_s).reject { |t| stats.key?(t) }.map do |tool|
|
|
127
|
+
Row.new(agent: agent, tool: tool, kind: "never_called",
|
|
128
|
+
detail: "in tools_allow, never called in any stored trace — " \
|
|
129
|
+
"its schema still ships on every request")
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
def error_rate_rows(agent, stats, days)
|
|
134
|
+
stats.filter_map do |tool, s|
|
|
135
|
+
next if s[:window_calls].zero?
|
|
136
|
+
|
|
137
|
+
rate = s[:window_errors].to_f / s[:window_calls]
|
|
138
|
+
next if rate <= ERROR_RATE_THRESHOLD
|
|
139
|
+
|
|
140
|
+
Row.new(agent: agent, tool: tool, kind: "error_rate",
|
|
141
|
+
detail: "#{s[:window_errors]}/#{s[:window_calls]} call(s) errored in the last " \
|
|
142
|
+
"#{days} day(s) (#{(rate * 100).round}%)")
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def blocked_rows(agent, stats, days)
|
|
147
|
+
stats.flat_map do |tool, s|
|
|
148
|
+
s[:blocked].map do |gate, count|
|
|
149
|
+
Row.new(agent: agent, tool: tool, kind: "blocked",
|
|
150
|
+
detail: "#{count} call(s) blocked by #{gate} in the last #{days} day(s)")
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
def stale_rows(agent, stats)
|
|
156
|
+
stats.filter_map do |tool, s|
|
|
157
|
+
next if s[:window_calls].positive? || s[:last_at].nil?
|
|
158
|
+
|
|
159
|
+
Row.new(agent: agent, tool: tool, kind: "stale",
|
|
160
|
+
detail: "last called #{s[:last_at].iso8601}, not once inside the window")
|
|
161
|
+
end
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
def parse_time(value)
|
|
165
|
+
return nil if value.to_s.empty?
|
|
166
|
+
|
|
167
|
+
Time.parse(value.to_s).utc
|
|
168
|
+
rescue ArgumentError
|
|
169
|
+
nil
|
|
170
|
+
end
|
|
171
|
+
end
|
|
172
|
+
end
|
|
@@ -47,6 +47,7 @@ module Insika
|
|
|
47
47
|
# envelope's duck-typed resolution checks this FIRST — a data-tool declares
|
|
48
48
|
# its evidence on its definition, never in the registry metadata.
|
|
49
49
|
def evidence = @definition.evidence
|
|
50
|
+
def requires_evidence = @definition.requires_evidence
|
|
50
51
|
|
|
51
52
|
# FULL (nested) JSON Schema straight into RubyLLM's params_schema — it is what
|
|
52
53
|
# the providers serialize (OpenAI/Anthropic/Gemini/Bedrock prefer
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "ruby_llm"
|
|
4
|
+
|
|
5
|
+
module Insika
|
|
6
|
+
module Tools
|
|
7
|
+
# PRESENTATION tool: the model says WHICH products to show, the engine decides
|
|
8
|
+
# WHAT is shown. Built by the overlay registry from a ToolDefinition that
|
|
9
|
+
# declares `presentation` (one class, N instances — like DataDefinedTool).
|
|
10
|
+
#
|
|
11
|
+
# Cards used to be a side effect of evidence: every attachment a search tool
|
|
12
|
+
# returned rode the channel delivery, all of them, and the model had no way to
|
|
13
|
+
# say "these three, not the ten I searched". The deployments compensated with
|
|
14
|
+
# inline markers in the text, parsed by regex on their side — where most of
|
|
15
|
+
# the hallucinated ids came from. Here text never carries product data: the
|
|
16
|
+
# model picks ids, and the engine
|
|
17
|
+
# 1. keeps only ids the session's evidence ledger has seen (provenance),
|
|
18
|
+
# 2. joins each to the card the evidence tool already hoarded this turn,
|
|
19
|
+
# 3. caps at the declared `max`,
|
|
20
|
+
# 4. records the selection on the turn (the delivery sends THAT),
|
|
21
|
+
# 5. emits `:ui` on the stream (the edge publishes it as `insika.ui`),
|
|
22
|
+
# 6. tells the model what was shown and what was dropped, and why.
|
|
23
|
+
# No HTTP, no model call, deterministic. `execute` never raises.
|
|
24
|
+
class Present < RubyLLM::Tool
|
|
25
|
+
# The one sentence for the model when NOTHING could be shown. Engine-generic:
|
|
26
|
+
# the pack tunes vocabulary through the tool's `description`.
|
|
27
|
+
NOTHING_SHOWN = "no card could be shown; name the products in text or search again"
|
|
28
|
+
|
|
29
|
+
def initialize(definition:, event_stream: nil)
|
|
30
|
+
@definition = definition
|
|
31
|
+
@event_stream = event_stream
|
|
32
|
+
@state = nil
|
|
33
|
+
super()
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# Deposited by ToolAssembly before the envelope wraps the instance — the
|
|
37
|
+
# ledger and the hoarded cards live on the turn, never on the registry.
|
|
38
|
+
def turn_state=(state)
|
|
39
|
+
@state = state
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def name = @definition.name
|
|
43
|
+
def description = @definition.description
|
|
44
|
+
def params_schema = @definition.parameters
|
|
45
|
+
|
|
46
|
+
def parameters
|
|
47
|
+
@parameters ||= @definition.top_level_params.each_with_object({}) do |p, acc|
|
|
48
|
+
sym = p[:name].to_sym
|
|
49
|
+
acc[sym] = RubyLLM::Parameter.new(sym, type: p[:type], desc: p[:description], required: p[:required])
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def execute(**kwargs)
|
|
54
|
+
spec = @definition.presentation
|
|
55
|
+
ids = Array(kwargs[spec[:ids].to_sym]).map(&:to_s).reject(&:empty?).uniq
|
|
56
|
+
title = Coercion.presence(kwargs[:title])
|
|
57
|
+
|
|
58
|
+
shown, dropped = select(ids, spec[:max])
|
|
59
|
+
record(spec[:component], title, shown)
|
|
60
|
+
emit(spec[:component], title, shown, dropped)
|
|
61
|
+
|
|
62
|
+
out = { "shown" => shown.map { |c| c["id"] }, "dropped" => dropped }
|
|
63
|
+
out["instruction"] = NOTHING_SHOWN if shown.empty?
|
|
64
|
+
out
|
|
65
|
+
rescue StandardError => e
|
|
66
|
+
{ error: "presentation failed: #{e.message}" }
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
private
|
|
70
|
+
|
|
71
|
+
# -> [shown cards, dropped [{id, reason}]]. Order preserved. Reasons:
|
|
72
|
+
# unknown — the ledger never saw the id (the model made it up, or the
|
|
73
|
+
# customer typed it); no ledger on the state reads as unknown too —
|
|
74
|
+
# a check that cannot verify does not pass.
|
|
75
|
+
# no_card — the id is known but no evidence tool returned a card for it
|
|
76
|
+
# this session (a text-only search result, or a seed of ids alone).
|
|
77
|
+
# max — beyond the declared cap.
|
|
78
|
+
# Cards come from this turn first, then from the ledger (the last few
|
|
79
|
+
# searches of the session) — "show me the second one" a turn later works.
|
|
80
|
+
def select(ids, max)
|
|
81
|
+
ledger = @state.respond_to?(:evidence_ledger) ? @state.evidence_ledger : nil
|
|
82
|
+
known = Array(ledger&.ids)
|
|
83
|
+
turn_cards = @state.respond_to?(:evidence_attachments) ? Array(@state.evidence_attachments) : []
|
|
84
|
+
session_cards = ledger.respond_to?(:cards) ? Array(ledger.cards) : []
|
|
85
|
+
by_id = (turn_cards + session_cards).each_with_object({}) { |c, h| h[c["id"]] ||= c if c["id"] }
|
|
86
|
+
|
|
87
|
+
shown = []
|
|
88
|
+
dropped = []
|
|
89
|
+
ids.each do |id|
|
|
90
|
+
if !known.include?(id) then dropped << { "id" => id, "reason" => "unknown" }
|
|
91
|
+
elsif (card = by_id[id]).nil? then dropped << { "id" => id, "reason" => "no_card" }
|
|
92
|
+
elsif shown.size >= max then dropped << { "id" => id, "reason" => "max" }
|
|
93
|
+
else shown << card
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
[shown, dropped]
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def record(component, title, shown)
|
|
100
|
+
return unless @state.respond_to?(:presentations)
|
|
101
|
+
|
|
102
|
+
@state.presentations ||= []
|
|
103
|
+
@state.presentations << { "component" => component, "title" => title,
|
|
104
|
+
"items" => shown.map { |c| c.slice("id", "type", "url", "caption") } }
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def emit(component, title, shown, dropped)
|
|
108
|
+
return unless @event_stream
|
|
109
|
+
|
|
110
|
+
task = @state.respond_to?(:task) ? @state.task : nil
|
|
111
|
+
meta = task ? { task_id: task.id, session_id: task.session_id } : {}
|
|
112
|
+
@event_stream.emit(Insika::Event.new(
|
|
113
|
+
type: :ui,
|
|
114
|
+
data: { component: component, title: title,
|
|
115
|
+
items: shown.map { |c| c.slice("id", "type", "url", "caption") },
|
|
116
|
+
count: shown.size, dropped: dropped },
|
|
117
|
+
meta: meta
|
|
118
|
+
))
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
end
|
|
@@ -161,7 +161,7 @@ module Insika
|
|
|
161
161
|
simulator = Insika::Evals::Simulator.new(transport: transport, ask: persona_ask, safety: safety)
|
|
162
162
|
|
|
163
163
|
conv = "eval-#{golden.id}-#{SecureRandom.hex(4)}" # a fresh session every run (never reused)
|
|
164
|
-
run = simulator.run(persona: golden.persona, agent: golden.agent, conv: conv)
|
|
164
|
+
run = simulator.run(persona: golden.persona, agent: golden.agent, conv: conv, state: golden.state)
|
|
165
165
|
verdict = judge.score_conversation(
|
|
166
166
|
rubric: golden.rubric, transcript: run.transcript, policy: golden.policy,
|
|
167
167
|
min_score: golden.min_score || Insika::Evals::Judge::DEFAULT_MIN_SCORE
|
|
@@ -199,6 +199,11 @@ module Insika
|
|
|
199
199
|
llm: llm_context
|
|
200
200
|
)
|
|
201
201
|
bus = Insika::CommandBus.new
|
|
202
|
+
# a golden with `state:` seeds through the same bus the turn runs on
|
|
203
|
+
bus.register(:seed_session, Insika::Commands::SeedSession.new(
|
|
204
|
+
session_store: @graph.session_store, memory_store: @graph.memory_store,
|
|
205
|
+
event_stream: @graph.event_stream
|
|
206
|
+
))
|
|
202
207
|
bus.register(:send_message, Insika::Commands::SendMessage.new(
|
|
203
208
|
profiles: @graph.profiles, session_store: @graph.session_store,
|
|
204
209
|
task_store: @graph.task_store, executor: executor,
|