insika 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +191 -0
- data/README.md +9 -6
- data/bin/insika +44 -3
- data/docs/AGENTS.md +74 -14
- data/docs/API.md +73 -0
- data/docs/ARCHITECTURE.md +45 -44
- data/docs/ARTIFACTS.md +42 -0
- data/docs/CHANNELS.md +19 -2
- data/docs/CONTEXT.md +86 -38
- data/docs/DEPLOY.md +27 -7
- data/docs/EVALS.md +98 -8
- data/docs/FACTS.md +4 -0
- data/docs/KNOWLEDGE.md +7 -0
- data/docs/LOADTEST.md +15 -27
- data/docs/MEDIA.md +1 -1
- data/docs/OBSERVABILITY.md +48 -4
- data/docs/POLICY.md +14 -5
- data/docs/RELEASING.md +4 -0
- data/docs/RUNNING-LOCAL.md +2 -2
- data/docs/SECURITY.md +28 -2
- data/docs/SOAK.md +1 -1
- data/docs/TOOLS.md +150 -32
- data/docs/prompts/ADD-TOOL.md +12 -2
- data/docs/prompts/DIAGNOSE-TURN.md +3 -0
- data/docs/prompts/GO-LIVE.md +6 -4
- data/lib/insika/agent_profile.rb +47 -10
- data/lib/insika/channels/web/widget.js +33 -0
- data/lib/insika/channels/web.rb +5 -2
- data/lib/insika/chat_builder.rb +90 -37
- data/lib/insika/commands/agent_payload.rb +1 -1
- data/lib/insika/commands/run_distillation.rb +5 -8
- data/lib/insika/commands/seed_session.rb +118 -0
- data/lib/insika/compaction.rb +196 -0
- data/lib/insika/context/builder.rb +35 -11
- data/lib/insika/context/fragment.rb +4 -1
- data/lib/insika/context/priority.rb +8 -0
- data/lib/insika/context/provider.rb +5 -0
- data/lib/insika/context/providers/briefing.rb +61 -29
- data/lib/insika/context/providers/fence_notice.rb +27 -0
- data/lib/insika/context/providers/knowledge.rb +7 -4
- data/lib/insika/context/providers/memory.rb +8 -4
- data/lib/insika/context/providers/session.rb +50 -10
- data/lib/insika/context_trace_store.rb +11 -1
- data/lib/insika/doctor.rb +213 -10
- data/lib/insika/dsl/runtime.rb +5 -0
- data/lib/insika/dsl.rb +6 -0
- data/lib/insika/edge_limiter.rb +4 -1
- data/lib/insika/env_schema.rb +5 -6
- data/lib/insika/errors.rb +1 -0
- data/lib/insika/evals/assertions.rb +92 -6
- data/lib/insika/evals/golden.rb +91 -2
- data/lib/insika/evals/runner.rb +20 -0
- data/lib/insika/evals/simulator.rb +11 -2
- data/lib/insika/evals/transport.rb +118 -16
- data/lib/insika/evidence.rb +79 -12
- data/lib/insika/executor.rb +94 -23
- data/lib/insika/fence.rb +96 -0
- data/lib/insika/golden_store.rb +3 -0
- data/lib/insika/loop_detector.rb +5 -34
- data/lib/insika/mcp_store.rb +5 -2
- data/lib/insika/mcp_tool_registry.rb +8 -1
- data/lib/insika/memory_store.rb +12 -0
- data/lib/insika/overlay_tool_registry.rb +5 -0
- data/lib/insika/prefix_fingerprint.rb +32 -27
- data/lib/insika/profile_source.rb +8 -0
- data/lib/insika/server/app.rb +43 -1
- data/lib/insika/server/rack_app.rb +2 -0
- data/lib/insika/server/responses.rb +35 -8
- data/lib/insika/session_store.rb +38 -5
- data/lib/insika/settings_store.rb +18 -2
- data/lib/insika/soak/runner.rb +4 -4
- data/lib/insika/spoken_transcript.rb +31 -0
- data/lib/insika/studio/app.rb +34 -8
- data/lib/insika/studio/forms.rb +29 -3
- data/lib/insika/studio/views/_agent_tab_config.erb +5 -1
- data/lib/insika/studio/views/session.erb +1 -1
- data/lib/insika/studio/views/settings.erb +11 -0
- data/lib/insika/studio/views/tool_edit.erb +6 -2
- data/lib/insika/telemetry/recorder.rb +61 -1
- data/lib/insika/templates/daily-digest/README.md +9 -0
- data/lib/insika/templates/research-analyst/agent.rb +10 -0
- data/lib/insika/tool_assembly.rb +21 -13
- data/lib/insika/tool_batch.rb +67 -0
- data/lib/insika/tool_definition.rb +73 -10
- data/lib/insika/tool_envelope.rb +102 -2
- data/lib/insika/tool_store.rb +9 -4
- data/lib/insika/tool_trace_store.rb +1 -1
- data/lib/insika/tool_usage_report.rb +172 -0
- data/lib/insika/tools/data_defined_tool.rb +1 -0
- data/lib/insika/tools/present.rb +122 -0
- data/lib/insika/tools/run_persona_eval.rb +6 -1
- data/lib/insika/tools/tool_search.rb +4 -2
- data/lib/insika/turn_budget.rb +91 -0
- data/lib/insika/turn_state.rb +13 -1
- data/lib/insika/version.rb +1 -1
- data/lib/insika/wiring/graph.rb +7 -0
- data/lib/insika/wiring/graph_chat.rb +4 -0
- data/lib/insika.rb +11 -0
- metadata +10 -1
data/lib/insika/studio/app.rb
CHANGED
|
@@ -45,8 +45,11 @@ module Studio
|
|
|
45
45
|
}.freeze
|
|
46
46
|
|
|
47
47
|
# Plugins that do NOT depend on the secret (loaded at class definition).
|
|
48
|
+
# The templates are UTF-8 ("Signing in…"). Tilt reads them in the PROCESS
|
|
49
|
+
# locale, and a host with no LANG (a bare container, a systemd unit) reads
|
|
50
|
+
# US-ASCII and 500s on the login page — found by the release install proof.
|
|
48
51
|
plugin :render, views: File.expand_path("views", __dir__), engine: "erb",
|
|
49
|
-
layout: "layout", escape: true
|
|
52
|
+
layout: "layout", escape: true, template_opts: { default_encoding: "UTF-8" }
|
|
50
53
|
plugin :hash_branches
|
|
51
54
|
plugin :h
|
|
52
55
|
|
|
@@ -1976,11 +1979,12 @@ end
|
|
|
1976
1979
|
name: t["name"].to_s, description: t["description"].to_s,
|
|
1977
1980
|
method: req["method"] || "GET", url: req["url"].to_s,
|
|
1978
1981
|
parameters: params_text(t["parameters"]),
|
|
1982
|
+
requires_evidence: Array(t.dig("requires_evidence", "params")).join(", "),
|
|
1979
1983
|
query: env_lines(req["query"]), headers: env_lines(req["headers"]),
|
|
1980
1984
|
secret_headers: Array(t["secret_headers"]).join(", "),
|
|
1981
1985
|
body: req["body"].to_s,
|
|
1982
1986
|
extract: resp["extract"] || "body_raw", path: resp["path"].to_s,
|
|
1983
|
-
timeout: t["timeout"]
|
|
1987
|
+
timeout: t["timeout"], presentation: t["presentation"]
|
|
1984
1988
|
}
|
|
1985
1989
|
end
|
|
1986
1990
|
|
|
@@ -2078,7 +2082,14 @@ end
|
|
|
2078
2082
|
"providers" => insika[:llm_provider_store] ? insika[:llm_provider_store].all.size : 0,
|
|
2079
2083
|
"MCP servers" => insika[:mcp_store] ? insika[:mcp_store].all.size : 0
|
|
2080
2084
|
}
|
|
2081
|
-
|
|
2085
|
+
# UTC, deliberately: sessions stamp `updated_at` with `Time.now.utc.iso8601`,
|
|
2086
|
+
# and every bucket below is keyed by a calendar part (date, hour) of that
|
|
2087
|
+
# stamp. A LOCAL `now` mixes two clocks — on a UTC-3 host, from 21:00 local
|
|
2088
|
+
# onward "today" is already tomorrow in UTC, so the day buckets stopped
|
|
2089
|
+
# matching and the 24h floor was built three hours in the future, silently
|
|
2090
|
+
# emptying both charts. Instant comparisons (`cutoff`) never had the bug;
|
|
2091
|
+
# calendar arithmetic did.
|
|
2092
|
+
now = Time.now.utc
|
|
2082
2093
|
cutoff = now - (5 * 60)
|
|
2083
2094
|
@active_now = sessions.count { |s| (t = parse_time(s.updated_at)) && t >= cutoff }
|
|
2084
2095
|
@recent = sessions.sort_by { |s| s.updated_at.to_s }.reverse.first(8)
|
|
@@ -2089,8 +2100,11 @@ end
|
|
|
2089
2100
|
# Trend affordances on the traffic stat cards: today vs yesterday from
|
|
2090
2101
|
# the 14-day series (config counts — agents/skills/tools/providers —
|
|
2091
2102
|
# have no daily shape and honestly show no trend).
|
|
2092
|
-
today,
|
|
2093
|
-
|
|
2103
|
+
# `@activity` is OLDEST FIRST and ends at today, so the last pair reads
|
|
2104
|
+
# [yesterday, today]. Destructured the other way round it reported a
|
|
2105
|
+
# first-conversation-of-the-day as "−1", every day.
|
|
2106
|
+
yesterday, today = @activity.last(2).map(&:last)
|
|
2107
|
+
@conv_trend = today.to_i - yesterday.to_i
|
|
2094
2108
|
@msg_trend = message_delta(sessions, now)
|
|
2095
2109
|
@persistence = insika.dig(:config, :persistence)
|
|
2096
2110
|
view("home")
|
|
@@ -2113,11 +2127,14 @@ end
|
|
|
2113
2127
|
end
|
|
2114
2128
|
|
|
2115
2129
|
# [[Date, count], …] — one bucket per day over the window, most-recent last.
|
|
2130
|
+
# `now` is UTC (render_home) and so is every `t` — see #utc_time. Both sides
|
|
2131
|
+
# of the bucket key must be read off the same clock or the join silently
|
|
2132
|
+
# misses.
|
|
2116
2133
|
def activity_by_day(sessions, days:, now:)
|
|
2117
2134
|
today = now.to_date
|
|
2118
2135
|
buckets = Hash.new(0)
|
|
2119
2136
|
sessions.each do |s|
|
|
2120
|
-
t =
|
|
2137
|
+
t = utc_time(s.updated_at) or next
|
|
2121
2138
|
buckets[t.to_date] += 1
|
|
2122
2139
|
end
|
|
2123
2140
|
(0...days).to_a.reverse.map { |i| d = today - i; [d, buckets[d]] }
|
|
@@ -2131,7 +2148,7 @@ end
|
|
|
2131
2148
|
floor = Time.utc(now.year, now.month, now.day, now.hour) - (hours - 1) * 3600
|
|
2132
2149
|
buckets = Hash.new(0)
|
|
2133
2150
|
sessions.each do |s|
|
|
2134
|
-
t =
|
|
2151
|
+
t = utc_time(s.updated_at) or next
|
|
2135
2152
|
h = Time.utc(t.year, t.month, t.day, t.hour)
|
|
2136
2153
|
buckets[h] += 1 if h >= floor
|
|
2137
2154
|
end
|
|
@@ -2144,11 +2161,20 @@ end
|
|
|
2144
2161
|
def message_delta(sessions, now)
|
|
2145
2162
|
today = now.to_date
|
|
2146
2163
|
sum = ->(date) do
|
|
2147
|
-
sessions.sum { |s| (t =
|
|
2164
|
+
sessions.sum { |s| (t = utc_time(s.updated_at)) && t.to_date == date ? Array(s.messages).size : 0 }
|
|
2148
2165
|
end
|
|
2149
2166
|
sum.call(today) - sum.call(today - 1)
|
|
2150
2167
|
end
|
|
2151
2168
|
|
|
2169
|
+
# A stamp read for its CALENDAR parts, always in UTC. `Time.parse` honours
|
|
2170
|
+
# whatever offset the string carries — ours are `Z`, but a record written by
|
|
2171
|
+
# anything else would otherwise bucket by its own zone. `getutc`, not `utc`:
|
|
2172
|
+
# the latter mutates the receiver.
|
|
2173
|
+
def utc_time(value)
|
|
2174
|
+
t = parse_time(value)
|
|
2175
|
+
t&.getutc
|
|
2176
|
+
end
|
|
2177
|
+
|
|
2152
2178
|
# --- History -------------------------------------------------------------
|
|
2153
2179
|
|
|
2154
2180
|
# Recent conversations (all agents — the Session doesn't stamp the agent that
|
data/lib/insika/studio/forms.rb
CHANGED
|
@@ -77,6 +77,7 @@ module Studio
|
|
|
77
77
|
# unchecked box saves an explicit false — both spellings of ON collapse.
|
|
78
78
|
tool_persistence: r.params["tool_persistence"] == "1",
|
|
79
79
|
tool_output_compression: r.params["tool_output_compression"] == "1",
|
|
80
|
+
fencing: r.params["fencing"] == "1",
|
|
80
81
|
subagents: list_patch(r, "subagents"),
|
|
81
82
|
capabilities: list_patch(r, "capabilities"),
|
|
82
83
|
tools_deferred: list_patch(r, "tools_deferred"),
|
|
@@ -386,7 +387,7 @@ module Studio
|
|
|
386
387
|
# record on write, so anything missing here is erased — a save that only fixed a
|
|
387
388
|
# typo in the description would silently drop them (the class of bug already
|
|
388
389
|
# paid for once). `stored` carries them through untouched.
|
|
389
|
-
UNEDITED_TOOL_FIELDS = %w[group tags halt_when].freeze
|
|
390
|
+
UNEDITED_TOOL_FIELDS = %w[group tags halt_when evidence presentation side_effect].freeze
|
|
390
391
|
|
|
391
392
|
# :write_data_tool payload from the form. nested request/response;
|
|
392
393
|
# headers/query as "key=value" per line (same idiom as the MCP env —
|
|
@@ -394,12 +395,17 @@ module Studio
|
|
|
394
395
|
# `stored` = the definition being edited (nil when creating).
|
|
395
396
|
def tool_patch(r, stored = nil)
|
|
396
397
|
preserved = (stored || {}).slice(*UNEDITED_TOOL_FIELDS).compact
|
|
397
|
-
|
|
398
|
+
method = presence(r.params["method"]) || "GET"
|
|
399
|
+
# A stored side_effect describes the stored method. When the method changes
|
|
400
|
+
# (a GET turned POST) it is dropped so the store re-derives it — the write
|
|
401
|
+
# gate and the resume skip both read it.
|
|
402
|
+
preserved.delete("side_effect") unless method == (stored || {}).dig("request", "method")
|
|
403
|
+
patch = preserved.merge(
|
|
398
404
|
name: presence(r.params["name"]),
|
|
399
405
|
description: r.params["description"].to_s,
|
|
400
406
|
parameters: parse_parameters(r.params["parameters"]),
|
|
401
407
|
request: {
|
|
402
|
-
method:
|
|
408
|
+
method: method,
|
|
403
409
|
url: r.params["url"].to_s,
|
|
404
410
|
headers: parse_kv_lines(r.params["headers"]),
|
|
405
411
|
query: parse_kv_lines(r.params["query"]),
|
|
@@ -410,8 +416,17 @@ module Studio
|
|
|
410
416
|
path: presence(r.params["path"])
|
|
411
417
|
},
|
|
412
418
|
secret_headers: split_list(r.params["secret_headers"]),
|
|
419
|
+
requires_evidence: if r.params.key?("requires_evidence")
|
|
420
|
+
split_list(r.params["requires_evidence"]).then { |names| names.empty? ? nil : names }
|
|
421
|
+
else
|
|
422
|
+
(stored || {})["requires_evidence"]
|
|
423
|
+
end,
|
|
413
424
|
timeout: presence(r.params["timeout"])
|
|
414
425
|
)
|
|
426
|
+
# A presentation tool makes no request: the form's HTTP fields are blank and
|
|
427
|
+
# the definition refuses both halves at once.
|
|
428
|
+
patch.delete(:request) if preserved["presentation"]
|
|
429
|
+
patch
|
|
415
430
|
end
|
|
416
431
|
|
|
417
432
|
# Parameters, TWO accepted syntaxes in one field (auto-detected, no mode toggle):
|
|
@@ -461,6 +476,17 @@ module Studio
|
|
|
461
476
|
# per-tenant map in the record — the view says so.
|
|
462
477
|
v = presence(r.params["memory_ttl_days"])
|
|
463
478
|
patch["memory_ttl_days"] = v.nil? ? nil : Integer(v)
|
|
479
|
+
# In-session compaction (RFC-0044). The checkbox is authoritative on
|
|
480
|
+
# this form (unchecked = disable); the numbers keep their stored value
|
|
481
|
+
# when cleared (deep_merge — the defaults backstop a fresh record);
|
|
482
|
+
# model blank = nil = the platform utility_model.
|
|
483
|
+
compaction = { "enabled" => r.params["compaction_enabled"] == "1",
|
|
484
|
+
"model" => presence(r.params["compaction_model"]) }
|
|
485
|
+
%w[keep_last compact_after].each do |f|
|
|
486
|
+
v = presence(r.params["compaction_#{f}"])
|
|
487
|
+
compaction[f] = Integer(v) if v
|
|
488
|
+
end
|
|
489
|
+
patch["compaction"] = compaction
|
|
464
490
|
patch
|
|
465
491
|
end
|
|
466
492
|
|
|
@@ -490,12 +490,16 @@
|
|
|
490
490
|
</label>
|
|
491
491
|
<label class="check">
|
|
492
492
|
<input type="checkbox" name="prompt_caching" value="1"<%== " checked" if @agent.prompt_caching %>>
|
|
493
|
-
prompt caching <span class="muted">— Anthropic breakpoint
|
|
493
|
+
prompt caching <span class="muted">— Anthropic breakpoint at the identity layer; memory/knowledge render below it</span>
|
|
494
494
|
</label>
|
|
495
495
|
<label class="check">
|
|
496
496
|
<input type="checkbox" name="tool_output_compression" value="1"<%== " checked" if @agent.tool_output_compression %>>
|
|
497
497
|
tool output compression <span class="muted">— mechanical dedupe of repeated tool results</span>
|
|
498
498
|
</label>
|
|
499
|
+
<label class="check">
|
|
500
|
+
<input type="checkbox" name="fencing" value="1"<%== " checked" if @agent.fencing %>>
|
|
501
|
+
fencing <span class="muted">— sanitize tool results, memory and knowledge before the model reads them (invisible characters, forged turn markers, transcript-shaped tags) and add the "data, not instructions" notice</span>
|
|
502
|
+
</label>
|
|
499
503
|
<label class="check">
|
|
500
504
|
<input type="checkbox" name="tool_persistence" value="1"<%== " checked" unless @agent.tool_persistence == false %>>
|
|
501
505
|
tool persistence <span class="muted">— the engine's "Tool discipline" prompt block (retry weak/empty tool results differently); ON by default</span>
|
|
@@ -185,7 +185,7 @@
|
|
|
185
185
|
<% traces[turn].each do |t| %>
|
|
186
186
|
<details class="trace-item <%= t["ok"] ? "ok" : "err" %>">
|
|
187
187
|
<summary>
|
|
188
|
-
<span class="badge"><%= t["ok"] ? "ok" : "error" %></span>
|
|
188
|
+
<span class="badge"><%= t["gate"] ? "blocked: #{t["gate"]}" : (t["ok"] ? "ok" : "error") %></span>
|
|
189
189
|
<strong><%= t["tool"] %></strong>
|
|
190
190
|
<span class="lat"><%= t["ms"] %>ms</span>
|
|
191
191
|
<span class="at"><%= t["at"] %></span>
|
|
@@ -43,6 +43,17 @@
|
|
|
43
43
|
<label>Memory TTL (days)<input type="text" name="memory_ttl_days" value="<%= @settings["memory_ttl_days"] %>" inputmode="numeric" placeholder="blank = off"></label>
|
|
44
44
|
</div>
|
|
45
45
|
<p class="muted">Memory TTL: how many days a customer memory cell lives before the daily sweep prunes it (per-fact <code>expires_at</code> overrides always win). Editing here sets the platform default for <strong>every</strong> cell; a per-tenant map authored in the settings record is replaced by this save.</p>
|
|
46
|
+
<hr>
|
|
47
|
+
<label class="check">
|
|
48
|
+
<input type="checkbox" name="compaction_enabled" value="1"<%== " checked" if @settings.dig("compaction", "enabled") %>>
|
|
49
|
+
in-session compaction — summarize old turns past the threshold
|
|
50
|
+
</label>
|
|
51
|
+
<div class="grid-2">
|
|
52
|
+
<label>keep_last <span class="muted">(messages kept verbatim)</span><input type="text" name="compaction_keep_last" value="<%= @settings.dig("compaction", "keep_last") %>" inputmode="numeric"></label>
|
|
53
|
+
<label>compact_after <span class="muted">(uncompacted messages that trigger it)</span><input type="text" name="compaction_compact_after" value="<%= @settings.dig("compaction", "compact_after") %>" inputmode="numeric"></label>
|
|
54
|
+
</div>
|
|
55
|
+
<label>compaction model <span class="muted">(blank = the platform utility_model)</span><input type="text" name="compaction_model" value="<%= @settings.dig("compaction", "model") %>" placeholder="deepseek-v4-flash"></label>
|
|
56
|
+
<p class="muted">When a session's uncompacted transcript grows past <code>compact_after</code> messages, everything but the last <code>keep_last</code> is summarized by the cheap model into one <code><conversation_summary></code> fragment; the tail stays verbatim and the boundary is stable (the prompt cache holds after it). Runs post-turn, off the critical path.</p>
|
|
46
57
|
</form>
|
|
47
58
|
</div>
|
|
48
59
|
|
|
@@ -33,7 +33,7 @@
|
|
|
33
33
|
</label>
|
|
34
34
|
|
|
35
35
|
<label>url <span class="muted">(https; use <code>{{param}}</code>)</span>
|
|
36
|
-
<input type="text" name="url" value="<%= @form[:url] %>" placeholder="https://viacep.com.br/ws/{{cep}}/json" required
|
|
36
|
+
<input type="text" name="url" value="<%= @form[:url] %>" placeholder="https://viacep.com.br/ws/{{cep}}/json"<%== " required" unless @form[:presentation] %>>
|
|
37
37
|
</label>
|
|
38
38
|
|
|
39
39
|
<label>parameters <span class="muted">(one per line: <code>name | type | required|optional | description</code> — type is <code>string</code>/<code>number</code>/<code>integer</code>/<code>boolean</code> or <code>array:string</code>. For a list of objects or a nested object, paste a full <strong>JSON Schema</strong> instead: any text starting with <code>{</code> is read as one.)</span>
|
|
@@ -62,7 +62,7 @@
|
|
|
62
62
|
<div class="grid-2">
|
|
63
63
|
<label>response
|
|
64
64
|
<select name="extract">
|
|
65
|
-
<% [["body_raw", "raw body"], ["json_path", "JSON field"], ["status", "status only"]].each do |v, lbl| %>
|
|
65
|
+
<% [["body_raw", "raw body"], ["json_path", "JSON field"], ["status", "status only"], ["evidence_envelope", "evidence envelope"]].each do |v, lbl| %>
|
|
66
66
|
<option value="<%= v %>"<%== " selected" if @form[:extract] == v %>><%= lbl %></option>
|
|
67
67
|
<% end %>
|
|
68
68
|
</select>
|
|
@@ -72,6 +72,10 @@
|
|
|
72
72
|
</label>
|
|
73
73
|
</div>
|
|
74
74
|
|
|
75
|
+
<label>requires_evidence <span class="muted">(parameter names, comma-separated; each ID must first be returned by a tool in this conversation)</span>
|
|
76
|
+
<input type="text" name="requires_evidence" value="<%= @form[:requires_evidence] %>" placeholder="product_id">
|
|
77
|
+
</label>
|
|
78
|
+
|
|
75
79
|
<label>timeout <span class="muted">(seconds, optional)</span>
|
|
76
80
|
<input type="number" name="timeout" value="<%= @form[:timeout] %>" min="1" placeholder="30">
|
|
77
81
|
</label>
|
|
@@ -40,7 +40,8 @@ module Insika
|
|
|
40
40
|
# and units are part of the documented contract (docs/OBSERVABILITY.md) —
|
|
41
41
|
# renaming one breaks every dashboard built on it.
|
|
42
42
|
class Instruments
|
|
43
|
-
attr_reader :turns, :turn_duration, :tokens, :cost, :tool_calls, :tool_duration
|
|
43
|
+
attr_reader :turns, :turn_duration, :tokens, :cost, :tool_calls, :tool_duration,
|
|
44
|
+
:cache_hit_rate, :loop_intervened, :context_compacted, :tool_blocked
|
|
44
45
|
|
|
45
46
|
def initialize(meter)
|
|
46
47
|
@turns = meter.create_counter("insika.turns", unit: "{turn}",
|
|
@@ -55,6 +56,14 @@ module Insika
|
|
|
55
56
|
description: "Tool invocations")
|
|
56
57
|
@tool_duration = meter.create_histogram("insika.tool.duration", unit: "s",
|
|
57
58
|
description: "Wall time of a tool call")
|
|
59
|
+
@cache_hit_rate = meter.create_histogram("insika.cache.hit_rate", unit: "%",
|
|
60
|
+
description: "Prompt-cache hit rate of a turn (cached / billed prompt tokens)")
|
|
61
|
+
@loop_intervened = meter.create_counter("insika.tool.loop_intervened", unit: "{intervention}",
|
|
62
|
+
description: "Loop-detector warnings delivered to the model")
|
|
63
|
+
@tool_blocked = meter.create_counter("insika.tool.blocked", unit: "{call}",
|
|
64
|
+
description: "Tool calls blocked before execution")
|
|
65
|
+
@context_compacted = meter.create_counter("insika.context.compacted", unit: "{compaction}",
|
|
66
|
+
description: "In-session compactions persisted (RFC-0044)")
|
|
58
67
|
end
|
|
59
68
|
end
|
|
60
69
|
|
|
@@ -73,6 +82,9 @@ module Insika
|
|
|
73
82
|
when :tool_call then start_tool(meta, data)
|
|
74
83
|
when :tool_result then finish_tool(meta)
|
|
75
84
|
when :data_tool_call then point_tool(meta, data)
|
|
85
|
+
when :tool_loop_intervened then count_loop(meta, data)
|
|
86
|
+
when :tool_blocked then count_blocked(meta, data)
|
|
87
|
+
when :context_compacted then count_compaction(data)
|
|
76
88
|
when :task_completed then finish_turn(meta, data, :ok)
|
|
77
89
|
when :task_failed then finish_turn(meta, data, :error)
|
|
78
90
|
when :task_cancelled then finish_turn(meta, data, :cancelled)
|
|
@@ -166,6 +178,54 @@ module Insika
|
|
|
166
178
|
@instruments.turns.add(1, attributes: labels)
|
|
167
179
|
@instruments.turn_duration.record(seconds, attributes: labels) if seconds
|
|
168
180
|
count_usage(turn, usage)
|
|
181
|
+
count_cache_hit(turn, usage)
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
# Same arithmetic as the Executor's per-agent series (stamp_cache_hit):
|
|
185
|
+
# the billed prompt is fresh input + cache reads + cache writes, and the
|
|
186
|
+
# hit rate is reads over the whole billed prompt — always in [0,100]. A
|
|
187
|
+
# turn with no billed prompt tokens (no usage, usage without the fields)
|
|
188
|
+
# records nothing: absence is not a 0% hit.
|
|
189
|
+
def count_cache_hit(turn, usage)
|
|
190
|
+
return unless usage
|
|
191
|
+
|
|
192
|
+
billed = usage[:input_tokens].to_i + usage[:cached_tokens].to_i +
|
|
193
|
+
usage[:cache_creation_tokens].to_i
|
|
194
|
+
return unless billed.positive?
|
|
195
|
+
|
|
196
|
+
rate = (usage[:cached_tokens].to_i * 100.0) / billed
|
|
197
|
+
base = turn.labels.merge(attrs("insika.model" => usage[:model]&.to_s))
|
|
198
|
+
@instruments.cache_hit_rate.record(rate, attributes: base)
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
# The loop detector delivered its one-shot warning (`:tool_loop_intervened`,
|
|
202
|
+
# counts and the tool name, never arguments). An orphan event (no open turn)
|
|
203
|
+
# is ignored, like every other consumer of this stream.
|
|
204
|
+
def count_loop(meta, data)
|
|
205
|
+
return unless @instruments
|
|
206
|
+
|
|
207
|
+
turn = @turns[meta[:task_id]] or return
|
|
208
|
+
labels = turn.labels.merge(attrs("insika.tool" => data[:name]&.to_s))
|
|
209
|
+
@instruments.loop_intervened.add(1, attributes: labels)
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
def count_blocked(meta, data)
|
|
213
|
+
return unless @instruments
|
|
214
|
+
|
|
215
|
+
turn = @turns[meta[:task_id]] or return
|
|
216
|
+
labels = turn.labels.merge(attrs("insika.tool" => data[:name]&.to_s,
|
|
217
|
+
"insika.gate" => data[:gate]&.to_s))
|
|
218
|
+
@instruments.tool_blocked.add(1, attributes: labels)
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
# A compaction persisted (`:context_compacted`, RFC-0044) — counted by
|
|
222
|
+
# agent/model, INDEPENDENT of any open turn: it fires post-turn, usually
|
|
223
|
+
# after task_completed already closed the span.
|
|
224
|
+
def count_compaction(data)
|
|
225
|
+
return unless @instruments
|
|
226
|
+
|
|
227
|
+
labels = attrs("insika.agent" => data[:agent]&.to_s, "insika.model" => data[:model]&.to_s)
|
|
228
|
+
@instruments.context_compacted.add(1, attributes: labels)
|
|
169
229
|
end
|
|
170
230
|
|
|
171
231
|
# Tokens ride ONE counter split by `insika.token.type` (instead of four
|
|
@@ -31,6 +31,15 @@ in the Playground — the artifact lands on the Artifacts tab either way.
|
|
|
31
31
|
- The **numbers** are not. Swap the literal string in `agent.rb` for a
|
|
32
32
|
`data_tool` against your own sales API and nothing else changes.
|
|
33
33
|
|
|
34
|
+
## When the report needs real data
|
|
35
|
+
|
|
36
|
+
One agent doing 30–50 tool calls at one reasoning effort is how a report turn
|
|
37
|
+
hits the 300 s timeout. The recipe is to split the phases across agents —
|
|
38
|
+
`thinking: "low"` miners fanned out with `spawn_subagents`, a `thinking: "high"`
|
|
39
|
+
orchestrator that plans and writes. See
|
|
40
|
+
[Artifacts](https://github.com/guizaols/insika/blob/main/docs/ARTIFACTS.md), "Reasoning effort on a report turn",
|
|
41
|
+
and the `research-analyst` template for the fan-out shape.
|
|
42
|
+
|
|
34
43
|
## Edit it
|
|
35
44
|
|
|
36
45
|
The skill's instructions are the actual report spec — change the palette,
|
|
@@ -12,6 +12,12 @@
|
|
|
12
12
|
# of a business idea, IN PARALLEL (each in its own isolated context), then
|
|
13
13
|
# the lead synthesizes one recommendation.
|
|
14
14
|
#
|
|
15
|
+
# Reasoning effort is split by ROLE, not spread evenly: the specialists answer
|
|
16
|
+
# one narrow question each (`thinking: "low"`), the lead plans the delegation
|
|
17
|
+
# and weighs three answers against each other (`thinking: "high"`). A child
|
|
18
|
+
# inherits the environment as a DEFAULT only, so its own `params` wins. See
|
|
19
|
+
# docs/ARTIFACTS.md, "Reasoning effort on a report turn".
|
|
20
|
+
#
|
|
15
21
|
# DEEPSEEK_API_KEY=sk-... ruby research-analyst/agent.rb "a subscription box for specialty coffee"
|
|
16
22
|
# DEEPSEEK_API_KEY=sk-... ruby research-analyst/agent.rb --serve
|
|
17
23
|
require "insika"
|
|
@@ -21,19 +27,23 @@ team = Insika.system do
|
|
|
21
27
|
|
|
22
28
|
agent("market") do
|
|
23
29
|
model "deepseek-v4-flash"
|
|
30
|
+
params thinking: "low"
|
|
24
31
|
instructions "Research the MARKET angle of a business idea: audience, demand, competitors. Three sentences."
|
|
25
32
|
end
|
|
26
33
|
agent("technical") do
|
|
27
34
|
model "deepseek-v4-flash"
|
|
35
|
+
params thinking: "low"
|
|
28
36
|
instructions "Research the TECHNICAL/OPERATIONAL angle of a business idea: what it takes to build and run it. Three sentences."
|
|
29
37
|
end
|
|
30
38
|
agent("risk") do
|
|
31
39
|
model "deepseek-v4-flash"
|
|
40
|
+
params thinking: "low"
|
|
32
41
|
instructions "Research the RISK angle of a business idea: what could make it fail. Three sentences."
|
|
33
42
|
end
|
|
34
43
|
|
|
35
44
|
agent "analyst" do
|
|
36
45
|
model "deepseek-v4-flash"
|
|
46
|
+
params thinking: "high"
|
|
37
47
|
instructions <<~PROMPT
|
|
38
48
|
You are a research LEAD with no expertise of your own — never answer
|
|
39
49
|
from your own knowledge. Given a business idea, call spawn_subagents
|
data/lib/insika/tool_assembly.rb
CHANGED
|
@@ -58,14 +58,13 @@ module Insika
|
|
|
58
58
|
# capability alias.
|
|
59
59
|
def assemble_tool_instances(allowed, state)
|
|
60
60
|
names = state.respond_to?(:capability_names) ? (state.capability_names || {}) : {}
|
|
61
|
-
|
|
62
|
-
return instantiate_tools(allowed, ctx) if names.empty?
|
|
61
|
+
return instantiate_tools(allowed, state) if names.empty?
|
|
63
62
|
|
|
64
63
|
# Dedup by the ENTRY NAME (registry key = impl_name) BEFORE
|
|
65
64
|
# instantiating — the INSTANCE's `.name` (RubyLLM) is not the registration
|
|
66
65
|
# name.
|
|
67
66
|
direct = Array(allowed).reject { |e| e.respond_to?(:name) && names.key?(e.name.to_s) }
|
|
68
|
-
instantiate_tools(direct,
|
|
67
|
+
instantiate_tools(direct, state) + capability_tool_instances(names, state)
|
|
69
68
|
end
|
|
70
69
|
|
|
71
70
|
# Envelopes each allowed tool (per-call timeout + side-effect recording).
|
|
@@ -78,7 +77,7 @@ module Insika
|
|
|
78
77
|
ToolEnvelope.new(tool, state: state, checkpoint_store: @checkpoint_store,
|
|
79
78
|
tool_registry: @tool_registry, timeout: timeout,
|
|
80
79
|
skip_side_effects: skip_side_effects,
|
|
81
|
-
trace_recorder: @tool_trace_store)
|
|
80
|
+
trace_recorder: @tool_trace_store, event_stream: @event_stream)
|
|
82
81
|
end
|
|
83
82
|
end
|
|
84
83
|
|
|
@@ -99,16 +98,19 @@ module Insika
|
|
|
99
98
|
return unless state.respond_to?(:tool_gate) && state.respond_to?(:tool_concurrency)
|
|
100
99
|
|
|
101
100
|
cap = state.tool_concurrency
|
|
102
|
-
state.tool_gate = cap ? Async::Semaphore.new(cap) : nil
|
|
101
|
+
state.tool_gate = cap && cap > 1 ? Async::Semaphore.new(cap) : nil
|
|
102
|
+
if state.respond_to?(:side_effect_gate=)
|
|
103
|
+
state.side_effect_gate = state.tool_gate ? Async::Semaphore.new(1) : nil
|
|
104
|
+
end
|
|
103
105
|
end
|
|
104
106
|
|
|
105
107
|
# Real Engine -> Entries (respond to factory); fakes -> ready instances.
|
|
106
108
|
# `turn_context` is deposited into the instances that expose it
|
|
107
109
|
# (data-tools); the rest ignore it (parity).
|
|
108
|
-
def instantiate_tools(allowed,
|
|
110
|
+
def instantiate_tools(allowed, state = nil)
|
|
109
111
|
Array(allowed).map do |t|
|
|
110
112
|
tool = t.respond_to?(:factory) ? t.factory.call : t
|
|
111
|
-
inject_turn_context(tool,
|
|
113
|
+
inject_turn_context(tool, state)
|
|
112
114
|
tool
|
|
113
115
|
end
|
|
114
116
|
end
|
|
@@ -116,22 +118,28 @@ module Insika
|
|
|
116
118
|
# seam: deposits the turn context into the freshly created instance
|
|
117
119
|
# (same idea as `remember`, which receives tenant/state) BEFORE the
|
|
118
120
|
# ToolEnvelope. Duck-typed: only what exposes `turn_context=` (DataDefinedTool)
|
|
119
|
-
# receives it
|
|
120
|
-
|
|
121
|
-
|
|
121
|
+
# receives it; a tool that exposes `turn_state=` (a presentation tool, which
|
|
122
|
+
# reads the ledger and the hoarded cards) receives the state itself. nil (a
|
|
123
|
+
# state with no turn_context, e.g. a test stub) -> no-op.
|
|
124
|
+
def inject_turn_context(tool, state)
|
|
125
|
+
return if state.nil?
|
|
126
|
+
|
|
127
|
+
tool.turn_state = state if tool.respond_to?(:turn_state=)
|
|
128
|
+
ctx = state.respond_to?(:turn_context) ? state.turn_context : nil
|
|
129
|
+
return if ctx.nil?
|
|
122
130
|
|
|
123
|
-
tool.turn_context =
|
|
131
|
+
tool.turn_context = ctx if tool.respond_to?(:turn_context=)
|
|
124
132
|
end
|
|
125
133
|
|
|
126
134
|
# impl_name -> Capability::ResolvedTool(capability_name:), STILL without
|
|
127
135
|
# ToolEnvelope (the call site's wrap_tools wraps the whole set — same
|
|
128
136
|
# order impl -> ResolvedTool -> ToolEnvelope). entry already validated in
|
|
129
137
|
# resolve_capabilities.
|
|
130
|
-
def capability_tool_instances(names,
|
|
138
|
+
def capability_tool_instances(names, state = nil)
|
|
131
139
|
names.map do |impl_name, capability_name|
|
|
132
140
|
entry = @tool_registry.entries.find { |e| e.name == impl_name }
|
|
133
141
|
tool = entry.factory.call
|
|
134
|
-
inject_turn_context(tool,
|
|
142
|
+
inject_turn_context(tool, state)
|
|
135
143
|
Capability::ResolvedTool.new(tool, capability_name: capability_name,
|
|
136
144
|
impl_name: impl_name)
|
|
137
145
|
end
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Insika
|
|
4
|
+
# The batch arithmetic behind every mid-turn `user` append.
|
|
5
|
+
#
|
|
6
|
+
# A model step that calls tools produces ONE assistant message announcing N
|
|
7
|
+
# tool calls, followed by N `role: tool` messages. Anthropic rejects a `user`
|
|
8
|
+
# message that lands between two of those results outright, so the ONLY valid
|
|
9
|
+
# append point inside a turn is the instant the Nth result closes the batch.
|
|
10
|
+
# SteerInjector discovered this rule; LoopDetector and TurnBudget both live by
|
|
11
|
+
# it, which is why the counting lives here instead of twice.
|
|
12
|
+
#
|
|
13
|
+
# Not a general-purpose helper: it answers one question ("did a batch just
|
|
14
|
+
# close?") and remembers one fact ("was this batch halted"), because a
|
|
15
|
+
# `halt_when` batch has no next model step and anything appended there would
|
|
16
|
+
# sit unread forever.
|
|
17
|
+
class ToolBatch
|
|
18
|
+
def initialize
|
|
19
|
+
@expected = nil # tool calls announced by the batch in flight (nil = none)
|
|
20
|
+
@seen = 0
|
|
21
|
+
@halted = false
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# Feeds a RubyLLM message (duck-typed). True EXACTLY on the message that
|
|
25
|
+
# closes a batch of tool calls — the append boundary. Everything else,
|
|
26
|
+
# including the assistant message that opens the batch, is false.
|
|
27
|
+
def closed?(message)
|
|
28
|
+
role = field(message, :role).to_s
|
|
29
|
+
return open(message) if role == "assistant"
|
|
30
|
+
return false unless role == "tool" && @expected
|
|
31
|
+
|
|
32
|
+
@seen += 1
|
|
33
|
+
return false if @seen < @expected
|
|
34
|
+
|
|
35
|
+
@expected = nil
|
|
36
|
+
true
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# From after_tool_result, with the RAW result: a Tool::Halt is only
|
|
40
|
+
# recognizable there.
|
|
41
|
+
def halt!(result)
|
|
42
|
+
@halted = true if defined?(RubyLLM::Tool::Halt) && result.is_a?(RubyLLM::Tool::Halt)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def halted? = @halted
|
|
46
|
+
|
|
47
|
+
private
|
|
48
|
+
|
|
49
|
+
# An assistant message with no tool calls is the model TALKING: the turn is
|
|
50
|
+
# ending, so nothing is in flight any more.
|
|
51
|
+
def open(message)
|
|
52
|
+
calls = field(message, :tool_calls)
|
|
53
|
+
size = calls.respond_to?(:size) ? calls.size : 0
|
|
54
|
+
@expected = size.zero? ? nil : size
|
|
55
|
+
@seen = 0
|
|
56
|
+
@halted = false
|
|
57
|
+
false
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def field(message, name)
|
|
61
|
+
return message.public_send(name) if message.respond_to?(name)
|
|
62
|
+
return message[name] || message[name.to_s] if message.respond_to?(:[])
|
|
63
|
+
|
|
64
|
+
nil
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
end
|