insika 0.3.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +296 -0
- data/README.md +48 -12
- data/bin/insika +725 -0
- data/bin/insika-router +87 -0
- data/docs/AGENTS.md +116 -406
- data/docs/API.md +5 -5
- data/docs/ARCHITECTURE.md +3 -2
- data/docs/ARTIFACTS.md +137 -0
- data/docs/BENCHMARK.md +2 -2
- data/docs/CHANNELS.md +14 -14
- data/docs/CONTEXT.md +63 -19
- data/docs/DEMO.md +80 -0
- data/docs/DEPLOY.md +87 -10
- data/docs/EMBEDDING.md +1 -1
- data/docs/EVALS.md +128 -3
- data/docs/FACTS.md +3 -3
- data/docs/HARVEST.md +5 -6
- data/docs/KNOWLEDGE.md +290 -0
- data/docs/LOADTEST.md +17 -29
- data/docs/MEDIA.md +128 -0
- data/docs/OBSERVABILITY.md +46 -12
- data/docs/OUTCOMES.md +137 -0
- data/docs/PLUGINS.md +51 -6
- data/docs/POLICY.md +222 -0
- data/docs/REFINEMENT.md +14 -9
- data/docs/RELEASING.md +4 -4
- data/docs/ROUTER.md +213 -0
- data/docs/RUNNING-LOCAL.md +5 -5
- data/docs/SCHEDULING.md +121 -0
- data/docs/SECURITY.md +23 -7
- data/docs/SKILLS.md +11 -2
- data/docs/SOAK.md +3 -3
- data/docs/TEMPLATES.md +134 -0
- data/docs/TOOLS.md +176 -27
- data/docs/WHY.md +1 -1
- data/docs/WORKFLOWS.md +2 -2
- data/docs/_includes/head_custom.html +5 -0
- data/docs/_includes/title.html +13 -0
- data/docs/_sass/color_schemes/insika.scss +32 -0
- data/docs/_sass/custom/custom.scss +199 -0
- data/docs/_sass/custom/setup.scss +26 -0
- data/docs/assets/img/favicon.svg +7 -0
- data/docs/assets/img/insika-mark.svg +7 -0
- data/docs/core-concepts.md +21 -0
- data/docs/domain.md +4 -4
- data/docs/improve.md +20 -0
- data/docs/index.md +8 -5
- data/docs/integrate.md +20 -0
- data/docs/operate.md +13 -6
- data/docs/prompts/ADD-TOOL.md +118 -0
- data/docs/prompts/DIAGNOSE-TURN.md +65 -0
- data/docs/prompts/GO-LIVE.md +138 -0
- data/docs/prompts/RUN-EXAMPLES.md +70 -0
- data/docs/reference.md +19 -0
- data/docs/ship.md +10 -2
- data/docs/start-here.md +18 -0
- data/lib/insika/agent_profile.rb +99 -17
- data/lib/insika/artifact_signing.rb +82 -0
- data/lib/insika/artifact_store.rb +160 -0
- data/lib/insika/channel_delivery.rb +1 -1
- data/lib/insika/chat_builder.rb +50 -19
- data/lib/insika/commands/agent_payload.rb +2 -2
- data/lib/insika/commands/backfill_knowledge.rb +145 -0
- data/lib/insika/commands/delete_artifact.rb +35 -0
- data/lib/insika/commands/delete_concept.rb +34 -0
- data/lib/insika/commands/delete_mcp.rb +6 -2
- data/lib/insika/commands/delete_tenant_data.rb +15 -3
- data/lib/insika/commands/gate_refinement.rb +1 -1
- data/lib/insika/commands/refresh_mcp_tools.rb +47 -0
- data/lib/insika/commands/restore_concept.rb +34 -0
- data/lib/insika/commands/seed_demo_data.rb +31 -0
- data/lib/insika/commands/upsert_mcp.rb +6 -3
- data/lib/insika/commands/write_concept.rb +57 -0
- data/lib/insika/compaction.rb +196 -0
- data/lib/insika/context/builder.rb +6 -2
- data/lib/insika/context/fragment.rb +4 -1
- data/lib/insika/context/priority.rb +8 -0
- data/lib/insika/context/providers/briefing.rb +53 -24
- data/lib/insika/context/providers/knowledge.rb +108 -0
- data/lib/insika/context/providers/prompt.rb +30 -24
- data/lib/insika/context/providers/session.rb +46 -10
- data/lib/insika/context_trace_store.rb +11 -1
- data/lib/insika/cron.rb +189 -0
- data/lib/insika/demo/agent_attrs.rb +43 -0
- data/lib/insika/demo/golden_cases.rb +81 -0
- data/lib/insika/demo/seeder.rb +336 -0
- data/lib/insika/doctor.rb +280 -17
- data/lib/insika/dsl/definition.rb +3 -2
- data/lib/insika/dsl/runtime.rb +64 -79
- data/lib/insika/dsl/server_boot.rb +23 -1
- data/lib/insika/dsl/system.rb +10 -2
- data/lib/insika/dsl.rb +103 -2
- data/lib/insika/env_schema.rb +21 -7
- data/lib/insika/evals/golden.rb +41 -4
- data/lib/insika/evals/judge.rb +47 -2
- data/lib/insika/evals/pairwise.rb +11 -0
- data/lib/insika/evals/persona.rb +98 -0
- data/lib/insika/evals/runner.rb +9 -0
- data/lib/insika/evals/simulator.rb +225 -0
- data/lib/insika/evals/transport.rb +84 -2
- data/lib/insika/event_stream.rb +10 -0
- data/lib/insika/executor.rb +295 -55
- data/lib/insika/followup_policy.rb +2 -25
- data/lib/insika/golden_store.rb +16 -1
- data/lib/insika/grounding/matcher.rb +1 -1
- data/lib/insika/knowledge.rb +680 -0
- data/lib/insika/knowledge_store.rb +140 -0
- data/lib/insika/loop_detector.rb +5 -34
- data/lib/insika/mcp_client.rb +94 -0
- data/lib/insika/mcp_json.rb +74 -0
- data/lib/insika/mcp_live_tool.rb +43 -0
- data/lib/insika/mcp_store.rb +98 -26
- data/lib/insika/mcp_tool_ingestor.rb +30 -8
- data/lib/insika/mcp_tool_registry.rb +100 -0
- data/lib/insika/media.rb +115 -31
- data/lib/insika/message_origin.rb +1 -1
- data/lib/insika/middleware.rb +9 -0
- data/lib/insika/onboarding.rb +17 -1
- data/lib/insika/outcome_store.rb +1 -1
- data/lib/insika/overlay_tool_registry.rb +37 -17
- data/lib/insika/packaging.rb +2 -2
- data/lib/insika/profile_source.rb +15 -1
- data/lib/insika/prompt_catalog.rb +10 -0
- data/lib/insika/retention.rb +36 -1
- data/lib/insika/router/app.rb +157 -0
- data/lib/insika/router/backend_pool.rb +98 -0
- data/lib/insika/router/hash_ring.rb +55 -0
- data/lib/insika/router/proxy_body.rb +34 -0
- data/lib/insika/router/session_key.rb +54 -0
- data/lib/insika/router.rb +18 -0
- data/lib/insika/schedule.rb +177 -0
- data/lib/insika/schedule_engine.rb +314 -0
- data/lib/insika/schedule_store.rb +208 -0
- data/lib/insika/server/app.rb +105 -15
- data/lib/insika/server/rack_app.rb +5 -1
- data/lib/insika/server/responses.rb +5 -5
- data/lib/insika/session_store.rb +34 -4
- data/lib/insika/settings_store.rb +8 -1
- data/lib/insika/skill_catalog.rb +12 -0
- data/lib/insika/soak/runner.rb +4 -4
- data/lib/insika/steer_injector.rb +21 -10
- data/lib/insika/studio/app.rb +591 -47
- data/lib/insika/studio/assets/dist/application.css +1 -1
- data/lib/insika/studio/assets/dist/application.js +21 -21
- data/lib/insika/studio/forms.rb +57 -5
- data/lib/insika/studio/nav_icons.rb +14 -1
- data/lib/insika/studio/views/_agent_tab_cache.erb +25 -0
- data/lib/insika/studio/views/_agent_tab_config.erb +514 -0
- data/lib/insika/studio/views/_agent_tab_history.erb +24 -0
- data/lib/insika/studio/views/_agent_tab_loops.erb +54 -0
- data/lib/insika/studio/views/_agent_tab_memory.erb +51 -0
- data/lib/insika/studio/views/_agent_tab_outcomes.erb +31 -0
- data/lib/insika/studio/views/_agent_tab_prompts.erb +108 -0
- data/lib/insika/studio/views/_agent_tab_skills.erb +38 -0
- data/lib/insika/studio/views/_agents_master.erb +44 -0
- data/lib/insika/studio/views/_message.erb +49 -32
- data/lib/insika/studio/views/agent_detail.erb +61 -820
- data/lib/insika/studio/views/agents.erb +70 -57
- data/lib/insika/studio/views/artifact.erb +23 -0
- data/lib/insika/studio/views/artifacts.erb +59 -0
- data/lib/insika/studio/views/evals.erb +2 -2
- data/lib/insika/studio/views/facts.erb +1 -1
- data/lib/insika/studio/views/funnel.erb +1 -1
- data/lib/insika/studio/views/home.erb +106 -67
- data/lib/insika/studio/views/knowledge.erb +123 -0
- data/lib/insika/studio/views/layout.erb +14 -11
- data/lib/insika/studio/views/mcp.erb +174 -80
- data/lib/insika/studio/views/session.erb +231 -177
- data/lib/insika/studio/views/settings.erb +50 -1
- data/lib/insika/studio/views/skills.erb +1 -1
- data/lib/insika/studio/views/tools.erb +24 -9
- data/lib/insika/telemetry/recorder.rb +49 -1
- data/lib/insika/templates/browser-agent/README.md +36 -0
- data/lib/insika/templates/browser-agent/agent.rb +49 -0
- data/lib/insika/templates/daily-digest/README.md +47 -0
- data/lib/insika/templates/daily-digest/agent.rb +77 -0
- data/lib/insika/templates/repo-explorer/README.md +36 -0
- data/lib/insika/templates/repo-explorer/agent.rb +45 -0
- data/lib/insika/templates/research-analyst/README.md +26 -0
- data/lib/insika/templates/research-analyst/agent.rb +68 -0
- data/lib/insika/templates/review-panel/README.md +20 -0
- data/lib/insika/templates/review-panel/agent.rb +50 -0
- data/lib/insika/templates/travel-planner/README.md +35 -0
- data/lib/insika/templates/travel-planner/agent.rb +87 -0
- data/lib/insika/templates.rb +112 -0
- data/lib/insika/tick.rb +24 -12
- data/lib/insika/timezone.rb +45 -0
- data/lib/insika/tool_batch.rb +67 -0
- data/lib/insika/tool_usage_report.rb +162 -0
- data/lib/insika/tools/generate_image.rb +52 -7
- data/lib/insika/tools/load_knowledge.rb +74 -0
- data/lib/insika/tools/run_persona_eval.rb +328 -0
- data/lib/insika/tools/save_artifact.rb +95 -0
- data/lib/insika/turn_budget.rb +91 -0
- data/lib/insika/turn_output.rb +1 -1
- data/lib/insika/turn_state.rb +15 -4
- data/lib/insika/version.rb +1 -1
- data/lib/insika/wiring/graph.rb +184 -12
- data/lib/insika/wiring/graph_chat.rb +102 -0
- data/lib/insika.rb +64 -0
- metadata +109 -5
- data/docs/build.md +0 -14
- data/docs/understand.md +0 -10
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "securerandom"
|
|
4
|
+
require "ruby_llm"
|
|
5
|
+
require_relative "agent_enum"
|
|
6
|
+
|
|
7
|
+
module Insika
|
|
8
|
+
module Tools
|
|
9
|
+
# `run_persona_eval` — a QA agent's own probe: pick an authored SIMULATED
|
|
10
|
+
# persona case (Evals::GoldenStore) and run it, in-process, against the
|
|
11
|
+
# case's declared target agent — the same Simulator + Judge machinery
|
|
12
|
+
# `insika evals:simulate` drives over HTTP, minus the CLI and the network
|
|
13
|
+
# hop.
|
|
14
|
+
#
|
|
15
|
+
# SAFETY is DERIVED, never a flag: the target's reachable side-effect tools
|
|
16
|
+
# are computed from the live registry (Evals::EvalProfile), the same way
|
|
17
|
+
# the CLI derives them. A read-only target needs no swap:
|
|
18
|
+
# Simulator::Safety's own "side_effect_tools.empty?" branch allows it
|
|
19
|
+
# directly. A target that DOES reach a side-effect tool gets the REAL
|
|
20
|
+
# swap (Evals::EvalProfile.registry — own overlay, wired here):
|
|
21
|
+
# every one of those tools resolves to a Simulator::DryRunTool for the
|
|
22
|
+
# duration of this ONE simulated conversation, run through a THROWAWAY
|
|
23
|
+
# Executor+Bus built fresh per call (`shadow_runtime`) — sharing every
|
|
24
|
+
# OTHER collaborator of the real graph (guardrails, policy, context
|
|
25
|
+
# assembly, skills/prompts, the real session/task/checkpoint stores), so
|
|
26
|
+
# the target is tested as faithfully as `--staging` ever was, minus the
|
|
27
|
+
# one write. Needs the real `graph:` (see `initialize`) — a caller that
|
|
28
|
+
# does not have one (an old-style double) falls back to refusing outright.
|
|
29
|
+
#
|
|
30
|
+
# BUDGET: the persona model + judge model calls are the cost of running the
|
|
31
|
+
# eval, charged to the CALLING agent's turn (never the target's — the
|
|
32
|
+
# target's own turns are billed normally, through the ordinary edge
|
|
33
|
+
# limiter, exactly as if a customer had sent those messages). A hard cap on
|
|
34
|
+
# the calling agent skips the run — visibly, in the tool result — before a
|
|
35
|
+
# cent is spent.
|
|
36
|
+
#
|
|
37
|
+
# TENANT ISOLATION: a persona case belongs to a tenant (Golden#tenant,
|
|
38
|
+
# "platform" by default); this tool only ever lists/runs cases in the
|
|
39
|
+
# CALLING agent's own tenant (calling_tenant, read off turn_context, never
|
|
40
|
+
# the model). One QA agent per store (the C3.2 plan) is what makes this
|
|
41
|
+
# meaningful -- without it, "qa-store-a" could enumerate and run
|
|
42
|
+
# "qa-store-b"'s persona and read its `knows` in the transcript.
|
|
43
|
+
class RunPersonaEval < RubyLLM::Tool
|
|
44
|
+
description "Run an authored simulated-customer persona case, in-process, against " \
|
|
45
|
+
"its target agent and score the whole conversation with the configured " \
|
|
46
|
+
"judge panel. Refuses if the target exposes a tool that could write for " \
|
|
47
|
+
"real."
|
|
48
|
+
param :case_id, desc: "Id of the persona case to run"
|
|
49
|
+
|
|
50
|
+
def name = "run_persona_eval"
|
|
51
|
+
|
|
52
|
+
# golden_store: Evals::GoldenStore — where authored persona cases live,
|
|
53
|
+
# scoped to the CALLING agent's own tenant (never the
|
|
54
|
+
# model's — see `calling_tenant`). A case authored for
|
|
55
|
+
# another tenant is invisible here, not merely undocumented.
|
|
56
|
+
# profiles: ProfileSource — resolves both the target agent (the
|
|
57
|
+
# case's `agent:`) and the CALLING agent (turn_context, for
|
|
58
|
+
# the budget check).
|
|
59
|
+
# tool_registry: the deployment's EFFECTIVE registry — what
|
|
60
|
+
# Evals::EvalProfile derives the target's side-effect tools
|
|
61
|
+
# from (the same registry a real turn resolves tools on).
|
|
62
|
+
# runtime: anything answering `#chat(message, session_id:, agent:)`
|
|
63
|
+
# (raises Insika::Error on failure) — Evals::GraphTransport's
|
|
64
|
+
# contract. Used AS-IS for a read-only target; a target
|
|
65
|
+
# with a reachable side-effect tool needs `graph:` instead
|
|
66
|
+
# (this alone cannot swap anything).
|
|
67
|
+
# graph: the real Wiring::Graph::Result — ONLY consulted to build
|
|
68
|
+
# the throwaway swapped-registry Executor+Bus
|
|
69
|
+
# (`shadow_runtime`) when the target has a reachable
|
|
70
|
+
# side-effect tool. nil (an old-style double) = that case
|
|
71
|
+
# refuses outright, same as before this existed.
|
|
72
|
+
# settings_store: where the platform `utility_model` (persona) and the
|
|
73
|
+
# judge panel (`evals.judges`) are configured.
|
|
74
|
+
# budget_ledger: WS2 counters — read before the run (skip on a hard cap),
|
|
75
|
+
# written after (persona + judge spend only).
|
|
76
|
+
# llm: this graph's own RubyLLM::Context, if it has one (a
|
|
77
|
+
# DSL-built graph's own credentials) — the persona/judge
|
|
78
|
+
# calls' preferred source, ahead of `runtime.llm`
|
|
79
|
+
# (kept for the existing double-based specs) and the
|
|
80
|
+
# process-wide RubyLLM constant.
|
|
81
|
+
def initialize(golden_store:, profiles:, tool_registry:, runtime:, settings_store:,
|
|
82
|
+
graph: nil, budget_ledger: nil, event_stream: nil, llm: nil)
|
|
83
|
+
@golden_store = golden_store
|
|
84
|
+
@profiles = profiles
|
|
85
|
+
@tool_registry = tool_registry
|
|
86
|
+
@runtime = runtime
|
|
87
|
+
@graph = graph
|
|
88
|
+
@settings_store = settings_store
|
|
89
|
+
@budget_ledger = budget_ledger
|
|
90
|
+
@event_stream = event_stream
|
|
91
|
+
@llm = llm
|
|
92
|
+
super()
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# Per-turn bindings, deposited by the Executor (ToolAssembly's
|
|
96
|
+
# `turn_context=` seam — same as save_artifact/data-tools). Only the
|
|
97
|
+
# CALLING agent's id + declared tenant are used here (the budget check);
|
|
98
|
+
# never a tenant/agent the model types.
|
|
99
|
+
attr_reader :turn_context
|
|
100
|
+
|
|
101
|
+
def turn_context=(ctx)
|
|
102
|
+
@turn_context = (ctx || {}).each_with_object({}) { |(k, v), acc| acc[k.to_sym] = v }
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# The runnable case ids, named so the model cannot guess one that does
|
|
106
|
+
# not exist (the Subagent tool's lesson — see AgentEnum).
|
|
107
|
+
def description
|
|
108
|
+
ids = case_ids
|
|
109
|
+
return super if ids.empty?
|
|
110
|
+
|
|
111
|
+
"#{super} Cases you may run: #{ids.join(', ')}."
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def params_schema
|
|
115
|
+
Insika::Tools::AgentEnum.inject(super, case_ids, path: %i[case_id])
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def execute(case_id:)
|
|
119
|
+
golden = @golden_store.find(case_id.to_s)
|
|
120
|
+
# The SAME error, whether the case does not exist or exists under another
|
|
121
|
+
# tenant — a QA agent must not be able to tell the two apart (that
|
|
122
|
+
# distinction is itself a leak: "case exists, just not yours").
|
|
123
|
+
unless golden&.simulated? && golden.tenant == calling_tenant
|
|
124
|
+
return { error: "unknown or invalid persona case '#{case_id}'" }
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
target = @profiles[golden.agent]
|
|
128
|
+
return { error: "persona case '#{case_id}' targets unknown agent '#{golden.agent}'" } unless target
|
|
129
|
+
|
|
130
|
+
derived = Insika::Evals::EvalProfile.side_effect_tools(target, @tool_registry)
|
|
131
|
+
return refuse_side_effects(golden.agent, derived) if !derived.empty? && @graph.nil?
|
|
132
|
+
|
|
133
|
+
skip = budget_skip
|
|
134
|
+
return skip if skip
|
|
135
|
+
|
|
136
|
+
meter = []
|
|
137
|
+
judge = build_judge(meter)
|
|
138
|
+
return { error: "no judge configured for this case — Studio -> Settings -> Evals" } unless judge
|
|
139
|
+
|
|
140
|
+
persona_ask = build_persona_ask(meter)
|
|
141
|
+
return persona_ask if persona_ask.is_a?(Hash) # {error:}
|
|
142
|
+
|
|
143
|
+
run_and_score(golden, derived, persona_ask, judge, meter)
|
|
144
|
+
rescue Insika::Evals::Simulator::UnsafeTarget => e
|
|
145
|
+
{ error: e.message }
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
private
|
|
149
|
+
|
|
150
|
+
def refuse_side_effects(agent, derived)
|
|
151
|
+
{ error: "target agent '#{agent}' exposes side-effect tool(s) (#{derived.join(', ')}) — " \
|
|
152
|
+
"run_persona_eval needs the real graph (no swap available for this caller)" }
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
def run_and_score(golden, derived, persona_ask, judge, meter)
|
|
156
|
+
runtime = derived.empty? ? @runtime : shadow_runtime(derived)
|
|
157
|
+
transport = Insika::Evals::GraphTransport.new(runtime: runtime, event_stream: @event_stream)
|
|
158
|
+
safety = Insika::Evals::Simulator::Safety.new(
|
|
159
|
+
side_effect_tools: derived, eval_profile: !derived.empty?, swapped_tools: derived
|
|
160
|
+
)
|
|
161
|
+
simulator = Insika::Evals::Simulator.new(transport: transport, ask: persona_ask, safety: safety)
|
|
162
|
+
|
|
163
|
+
conv = "eval-#{golden.id}-#{SecureRandom.hex(4)}" # a fresh session every run (never reused)
|
|
164
|
+
run = simulator.run(persona: golden.persona, agent: golden.agent, conv: conv)
|
|
165
|
+
verdict = judge.score_conversation(
|
|
166
|
+
rubric: golden.rubric, transcript: run.transcript, policy: golden.policy,
|
|
167
|
+
min_score: golden.min_score || Insika::Evals::Judge::DEFAULT_MIN_SCORE
|
|
168
|
+
)
|
|
169
|
+
account_spend(meter)
|
|
170
|
+
|
|
171
|
+
{ case: golden.id, agent: golden.agent, stop: run.stop.to_s, turns: run.turns,
|
|
172
|
+
score: verdict&.score, pass: verdict&.pass, reason: verdict&.reason,
|
|
173
|
+
transcript: run.transcript.map { |m| { role: m[:role], text: m[:text] } } }
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
# A THROWAWAY Executor+Bus, built fresh for THIS call, over the SAME
|
|
177
|
+
# session/task/checkpoint stores, guardrails, policy engine, context
|
|
178
|
+
# assembly, skills/prompts and capabilities as the real graph — the
|
|
179
|
+
# ONLY thing different is the tool registry, which resolves every name
|
|
180
|
+
# in `derived` to a Simulator::DryRunTool (Evals::EvalProfile.registry)
|
|
181
|
+
# and everything else exactly as the real one does. Never persisted
|
|
182
|
+
# anywhere NEW: the simulated turn's session/task rows land in the
|
|
183
|
+
# SAME stores a `--staging` run's already do (RunPersonaEval's own
|
|
184
|
+
# never-reused `conv` id is what keeps it from colliding with a real
|
|
185
|
+
# customer session, same as before this existed).
|
|
186
|
+
def shadow_runtime(derived)
|
|
187
|
+
overlay = Insika::Evals::EvalProfile.registry(@tool_registry, side_effect_tools: derived)
|
|
188
|
+
executor = Insika::Executor.new(
|
|
189
|
+
context_builder: @graph.context_builder, policy_engine: @graph.policy_engine,
|
|
190
|
+
middleware: @graph.middleware, hooks: @graph.hooks,
|
|
191
|
+
tool_registry: overlay, skill_catalog: @graph.skill_catalog, profiles: @graph.profiles,
|
|
192
|
+
session_store: @graph.session_store, task_store: @graph.task_store,
|
|
193
|
+
checkpoint_store: @graph.checkpoint_store, event_stream: @graph.event_stream,
|
|
194
|
+
workflow_registry: @graph.workflow_registry, pending_action_store: @graph.pending_action_store,
|
|
195
|
+
capability_registry: @graph.capability_registry,
|
|
196
|
+
tool_catalog: Insika::ToolCatalog.new(tool_registry: overlay),
|
|
197
|
+
memory_store: @graph.memory_store, settings_store: @settings_store,
|
|
198
|
+
delegation_store: @graph.delegation_store, channel_delivery: @graph.channel_delivery,
|
|
199
|
+
llm: llm_context
|
|
200
|
+
)
|
|
201
|
+
bus = Insika::CommandBus.new
|
|
202
|
+
bus.register(:send_message, Insika::Commands::SendMessage.new(
|
|
203
|
+
profiles: @graph.profiles, session_store: @graph.session_store,
|
|
204
|
+
task_store: @graph.task_store, executor: executor,
|
|
205
|
+
inbound_log: @graph.inbound_log, contact_store: @graph.contact_store,
|
|
206
|
+
followup_store: @graph.followup_store, store: @graph.backend
|
|
207
|
+
))
|
|
208
|
+
shadow = @graph.dup
|
|
209
|
+
shadow.bus = bus
|
|
210
|
+
shadow.executor = executor
|
|
211
|
+
Insika::Wiring::GraphChat.new(graph: shadow)
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
# -> [String] every case this tool can run: a valid, SIMULATED (persona:)
|
|
215
|
+
# golden belonging to the CALLING agent's own tenant. A scripted (turns:)
|
|
216
|
+
# case has nothing to simulate — the CLI skips it too — and a case outside
|
|
217
|
+
# `calling_tenant` is not merely un-runnable, it never appears at all (the
|
|
218
|
+
# model cannot even learn another tenant's case ids from the enum).
|
|
219
|
+
def case_ids
|
|
220
|
+
@golden_store.for_tenant(calling_tenant).select(&:simulated?).map(&:id)
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
# The CALLING agent's tenant — never the model's, never the target's. The
|
|
224
|
+
# same "declared command tenant, 'platform' is the single-tenant default"
|
|
225
|
+
# rule `save_artifact`'s `binding_tenant` uses. Every persona-case
|
|
226
|
+
# read/list/run in this tool is scoped to it.
|
|
227
|
+
def calling_tenant
|
|
228
|
+
Insika::Coercion.presence(turn_context&.dig(:command_tenant)) || "platform"
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
# settings["evals"] -> the configured judge panel, scoped to this
|
|
232
|
+
# graph's own RubyLLM credentials (never the process-wide default —
|
|
233
|
+
# a graph's judge spends the graph's own key) and METERED (every judge
|
|
234
|
+
# call's usage lands in `meter`). nil = nobody configured (the CLI's own
|
|
235
|
+
# rule: never guess a judge to spend money on).
|
|
236
|
+
def build_judge(meter)
|
|
237
|
+
judge, = Insika::Evals::JudgePanel.build(@settings_store.get["evals"] || {}, chat_factory: metered_factory(meter))
|
|
238
|
+
judge
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
# The persona model — the platform `utility_model` (never a caller-chosen
|
|
242
|
+
# model: the model does not get to pick what it costs to test itself),
|
|
243
|
+
# metered the same way. -> callable | {error:}.
|
|
244
|
+
def build_persona_ask(meter)
|
|
245
|
+
model = Insika::Coercion.presence(@settings_store.get["utility_model"])
|
|
246
|
+
return { error: "no persona model — set the platform utility_model (Studio -> Settings)" } if model.nil?
|
|
247
|
+
|
|
248
|
+
metered_factory(meter).call(model, nil)
|
|
249
|
+
end
|
|
250
|
+
|
|
251
|
+
# The explicit `llm:` wins (the real wiring passes it — see
|
|
252
|
+
# `Wiring::Graph.register_persona_eval_tool`); `@runtime.llm` is the
|
|
253
|
+
# fallback the existing double-based specs rely on (a fake `runtime`
|
|
254
|
+
# answering `#llm` with no `graph:` at all). nil = the process-wide
|
|
255
|
+
# RubyLLM constant.
|
|
256
|
+
def llm_context
|
|
257
|
+
@llm || (@runtime.respond_to?(:llm) ? @runtime.llm : nil)
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
# ->(model, provider) { ask } — a RubyLLM chat (this graph's own
|
|
261
|
+
# credentials, temperature 0) whose every `#ask` lands its Message in
|
|
262
|
+
# `sink` before handing back the text. Same shape as
|
|
263
|
+
# JudgePanel.ruby_llm_ask, except it does not throw the usage away.
|
|
264
|
+
# `assume_model_exists` is passed ONLY alongside an explicit `provider`
|
|
265
|
+
# (the judge panel's own shape — `settings["evals"]["judges"]` always
|
|
266
|
+
# carries one) — RubyLLM raises ArgumentError on `assume_model_exists:
|
|
267
|
+
# true` with no provider. The persona's own model (the platform
|
|
268
|
+
# `utility_model`, a bare ref with no companion provider setting
|
|
269
|
+
# anywhere in the schema) needs the OPPOSITE: no provider, no
|
|
270
|
+
# assume_model_exists, so RubyLLM resolves it from its own registry —
|
|
271
|
+
# exactly how a bare model ref already works everywhere else this
|
|
272
|
+
# codebase reaches for `utility_model`.
|
|
273
|
+
def metered_factory(sink)
|
|
274
|
+
lambda do |model, provider|
|
|
275
|
+
kwargs = { model: model }
|
|
276
|
+
if provider
|
|
277
|
+
kwargs[:provider] = provider
|
|
278
|
+
kwargs[:assume_model_exists] = true
|
|
279
|
+
end
|
|
280
|
+
chat = (llm_context || RubyLLM).chat(**kwargs).with_temperature(0)
|
|
281
|
+
lambda do |prompt|
|
|
282
|
+
msg = chat.ask(prompt)
|
|
283
|
+
sink << msg
|
|
284
|
+
msg.content
|
|
285
|
+
end
|
|
286
|
+
end
|
|
287
|
+
end
|
|
288
|
+
|
|
289
|
+
# -> truthy (a visible skip result) when the CALLING agent's own hard
|
|
290
|
+
# budget is already at/over a window cap; nil otherwise. Mirrors
|
|
291
|
+
# ScheduleEngine#budget_exhausted? — a HARD cap skips rather than spends
|
|
292
|
+
# (a soft one just runs; the ledger's own alert already warns).
|
|
293
|
+
def budget_skip
|
|
294
|
+
return nil unless @budget_ledger
|
|
295
|
+
|
|
296
|
+
agent_id = turn_context&.dig(:agent_id)
|
|
297
|
+
profile = agent_id && @profiles[agent_id]
|
|
298
|
+
budget = profile&.respond_to?(:budget) ? profile.budget : nil
|
|
299
|
+
return nil if budget.nil? || budget["soft"] == true
|
|
300
|
+
|
|
301
|
+
tenant = turn_context&.dig(:command_tenant)
|
|
302
|
+
window = %w[daily monthly].find do |w|
|
|
303
|
+
cap = budget[w].to_i
|
|
304
|
+
cap.positive? && @budget_ledger.current(tenant: tenant, agent: agent_id)[w.to_sym] >= cap
|
|
305
|
+
end
|
|
306
|
+
{ skipped: true, reason: "budget", window: window } if window
|
|
307
|
+
end
|
|
308
|
+
|
|
309
|
+
# The turn's real billed spend for the persona + judge calls (the A4
|
|
310
|
+
# rule: total + cached + cache_creation), charged to the CALLING agent —
|
|
311
|
+
# never the target, whose own turns are billed normally by the edge
|
|
312
|
+
# limiter, exactly like a real customer's would be.
|
|
313
|
+
def account_spend(meter)
|
|
314
|
+
return if @budget_ledger.nil? || meter.empty?
|
|
315
|
+
|
|
316
|
+
tokens = meter.sum do |m|
|
|
317
|
+
m.input_tokens.to_i + m.output_tokens.to_i +
|
|
318
|
+
(m.respond_to?(:cached_tokens) ? m.cached_tokens.to_i : 0) +
|
|
319
|
+
(m.respond_to?(:cache_creation_tokens) ? m.cache_creation_tokens.to_i : 0)
|
|
320
|
+
end
|
|
321
|
+
return if tokens.zero?
|
|
322
|
+
|
|
323
|
+
@budget_ledger.add(tenant: turn_context&.dig(:command_tenant),
|
|
324
|
+
agent: turn_context&.dig(:agent_id), by: tokens)
|
|
325
|
+
end
|
|
326
|
+
end
|
|
327
|
+
end
|
|
328
|
+
end
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "ruby_llm"
|
|
4
|
+
|
|
5
|
+
module Insika
|
|
6
|
+
module Tools
|
|
7
|
+
# `save_artifact` — the agent delivers a page (a report, a digest) to a
|
|
8
|
+
# destination that survives the channel: title + content in, a URL out,
|
|
9
|
+
# which it can include in a channel message ("today's report: <url>").
|
|
10
|
+
#
|
|
11
|
+
# A REGISTRY tool (not a ChatBuilder system tool): the per-agent
|
|
12
|
+
# `tools_allow` is the switch — registered `optional: true`, so an agent
|
|
13
|
+
# that did not name it cannot call it, and nothing global enables it.
|
|
14
|
+
# The bindings (tenant, agent, task) are the tool INSTANCE's, deposited
|
|
15
|
+
# by the Executor into `turn_context` like a data-tool's — never
|
|
16
|
+
# parameters the model types.
|
|
17
|
+
class SaveArtifact < RubyLLM::Tool
|
|
18
|
+
description "Save an HTML (or markdown / SVG) page — a report, a digest, " \
|
|
19
|
+
"a summary — and get back a URL to share. Use when the answer " \
|
|
20
|
+
"is a document the channel message cannot carry."
|
|
21
|
+
param :title, desc: "Short human-readable title (<= 200 chars)"
|
|
22
|
+
param :content, desc: "The page content: HTML with inline SVG for charts, " \
|
|
23
|
+
"markdown, or SVG. No scripts, no external resources."
|
|
24
|
+
param :mime, desc: "text/html (default), text/markdown, or image/svg+xml",
|
|
25
|
+
required: false
|
|
26
|
+
|
|
27
|
+
def name = "save_artifact"
|
|
28
|
+
|
|
29
|
+
def initialize(artifact_store:, base_url: nil, signing_key: nil, signing_ttl: nil,
|
|
30
|
+
max_bytes: nil, event_stream: nil)
|
|
31
|
+
@artifact_store = artifact_store
|
|
32
|
+
@base_url = base_url.to_s.sub(%r{/\z}, "")
|
|
33
|
+
@signing_key = signing_key
|
|
34
|
+
@signing_ttl = signing_ttl
|
|
35
|
+
@max_bytes = max_bytes
|
|
36
|
+
@event_stream = event_stream
|
|
37
|
+
super()
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Per-turn bindings, deposited by the Executor (ToolAssembly's
|
|
41
|
+
# `turn_context=` seam — the same one data-tools use). From the TURN,
|
|
42
|
+
# never from the model (the tenant-lesson: an id the model types is
|
|
43
|
+
# inventable, and the turn looks right anyway).
|
|
44
|
+
attr_reader :turn_context
|
|
45
|
+
|
|
46
|
+
def turn_context=(ctx)
|
|
47
|
+
@turn_context = (ctx || {}).each_with_object({}) { |(k, v), acc| acc[k.to_sym] = v }
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def execute(title:, content:, mime: "text/html")
|
|
51
|
+
# the deployment's size cap rides the tool instance (INSIKA_ARTIFACT_
|
|
52
|
+
# MAX_BYTES at wiring); absent -> the store's default.
|
|
53
|
+
attrs = { tenant: binding_tenant, agent: @turn_context&.dig(:agent_id),
|
|
54
|
+
task_id: @turn_context&.dig(:task_id), title: title, mime: mime,
|
|
55
|
+
content: content }
|
|
56
|
+
attrs[:max_bytes] = @max_bytes if @max_bytes
|
|
57
|
+
record = @artifact_store.create(**attrs)
|
|
58
|
+
# the authenticated Studio URL is the always-on answer; the signed
|
|
59
|
+
# link exists ONLY when a key was configured (it shares OUTSIDE the
|
|
60
|
+
# Studio and expires).
|
|
61
|
+
url = Insika::ArtifactSigning.url_for(id: record.id, base: @base_url)
|
|
62
|
+
result = { id: record.id, url: url }
|
|
63
|
+
signed = Insika::ArtifactSigning.url_for(
|
|
64
|
+
id: record.id, base: @base_url, key: @signing_key, ttl: @signing_ttl
|
|
65
|
+
)
|
|
66
|
+
result[:signed_url] = signed if signed != url
|
|
67
|
+
emit(record)
|
|
68
|
+
result
|
|
69
|
+
rescue Insika::ValidationError => e
|
|
70
|
+
{ error: e.message }
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
private
|
|
74
|
+
|
|
75
|
+
# The tenant is the AGENT's tenant: the declared command tenant, or the
|
|
76
|
+
# deployment's "platform" in single-tenant — never the chat id (a report
|
|
77
|
+
# belongs to the agent, not to whichever chat ran it), never a parameter
|
|
78
|
+
# the model types.
|
|
79
|
+
def binding_tenant
|
|
80
|
+
declared = @turn_context&.dig(:command_tenant).to_s
|
|
81
|
+
declared.empty? ? "platform" : declared
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# :artifact_saved carries ids + title — NEVER the content (a report is
|
|
85
|
+
# customer data that rides the store, not the events).
|
|
86
|
+
def emit(record)
|
|
87
|
+
@event_stream&.emit(Insika::Event.new(
|
|
88
|
+
type: :artifact_saved,
|
|
89
|
+
data: { id: record.id, title: record.title },
|
|
90
|
+
meta: { task_id: @turn_context&.dig(:task_id) }
|
|
91
|
+
))
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
end
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Insika
|
|
4
|
+
# The turn's tool-call counter — one object, three jobs: count, warn, abort.
|
|
5
|
+
#
|
|
6
|
+
# `max_tool_calls` was enforced but never announced: the model learned of the
|
|
7
|
+
# ceiling only when the turn DIED with `stage: :tool_limit`. A real run in
|
|
8
|
+
# `round1.db` (task d0421891) burned all 50 calls summing a 40-number list one
|
|
9
|
+
# `calc` at a time and delivered nothing — the model had no way to know it was
|
|
10
|
+
# spending a budget at all.
|
|
11
|
+
#
|
|
12
|
+
# So the budget speaks before it kills: at 10 / 5 / 2 calls remaining the
|
|
13
|
+
# engine appends a short notice and the model gets to converge on an answer.
|
|
14
|
+
#
|
|
15
|
+
# · **User turn, never system.** The system prefix stays byte-stable, so the
|
|
16
|
+
# Anthropic cache breakpoint (`chat_builder.rb`'s `apply_instructions`) keeps
|
|
17
|
+
# hitting. A notice in the prompt would bill a cache WRITE every turn.
|
|
18
|
+
# · **Batch boundary only** (ToolBatch), and **a halted batch receives
|
|
19
|
+
# nothing** — the two rules SteerInjector established for mid-turn appends.
|
|
20
|
+
# · **Off when there is no limit.** `max: nil` = count nothing, warn nothing,
|
|
21
|
+
# abort never.
|
|
22
|
+
#
|
|
23
|
+
# KNOWN REACH (measured against DeepSeek, 2026-08-30): a model that announces
|
|
24
|
+
# ONE batch bigger than the whole budget — 30 independent tool calls in a single
|
|
25
|
+
# assistant message — never reaches a boundary before the abort, so it gets no
|
|
26
|
+
# notice. Nothing can be appended mid-batch, so that shape is out of reach by
|
|
27
|
+
# construction; it is also the shape where the model already knows what it asked
|
|
28
|
+
# for. What this catches is the SEQUENTIAL burn, which is the one `round1.db`
|
|
29
|
+
# actually shows: 46 `calc` calls, one per step.
|
|
30
|
+
class TurnBudget
|
|
31
|
+
# The three notices, verbatim and escalating — one constant, so a transcript
|
|
32
|
+
# reader can recognize an engine sentence without an origin stamp (chat
|
|
33
|
+
# messages carry none). Keyed by calls REMAINING after the call being made.
|
|
34
|
+
NOTICES = {
|
|
35
|
+
10 => "Tool budget: 10 of your %<max>d tool calls for this turn are left. " \
|
|
36
|
+
"Start converging — prefer one call that answers the question over several that circle it.",
|
|
37
|
+
5 => "Tool budget: 5 tool calls left in this turn. " \
|
|
38
|
+
"Drop anything optional and gather only what the answer actually needs.",
|
|
39
|
+
2 => "Tool budget: 2 tool calls left in this turn. " \
|
|
40
|
+
"Consolidate what you already have and answer now — do not start new work."
|
|
41
|
+
}.freeze
|
|
42
|
+
|
|
43
|
+
def self.notice(remaining, max) = format(NOTICES.fetch(remaining), max: max)
|
|
44
|
+
|
|
45
|
+
# chat: the turn's chat — must answer #add_message (the boundary append).
|
|
46
|
+
# max: the profile's max_tool_calls. nil = no budget: no notice, no abort.
|
|
47
|
+
# emit: ->(type, data) — the Executor's emitter, bound to the task.
|
|
48
|
+
def initialize(chat:, max:, emit:)
|
|
49
|
+
@chat = chat
|
|
50
|
+
@max = max
|
|
51
|
+
@emit = emit
|
|
52
|
+
@calls = 0
|
|
53
|
+
@batch = ToolBatch.new
|
|
54
|
+
@pending = nil # a threshold was crossed; waiting for the batch boundary
|
|
55
|
+
@warned = [] # thresholds already spent (each fires at most once a turn)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# From ChatBuilder's before_tool_call, FIRST thing: counts the call about to
|
|
59
|
+
# run and raises when it is past the ceiling. The raise is the pre-existing
|
|
60
|
+
# guard-rail, moved here so the count has exactly one owner.
|
|
61
|
+
def tool_call
|
|
62
|
+
return if @max.nil?
|
|
63
|
+
|
|
64
|
+
@calls += 1
|
|
65
|
+
if @calls > @max
|
|
66
|
+
raise Insika::TimeoutError.new("tool call limit exceeded (#{@max})", stage: :tool_limit)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
remaining = @max - @calls
|
|
70
|
+
@pending = remaining if NOTICES.key?(remaining) && !@warned.include?(remaining)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# From ChatBuilder's after_tool_result, with the RAW result.
|
|
74
|
+
def tool_result(result) = @batch.halt!(result)
|
|
75
|
+
|
|
76
|
+
# RubyLLM after_message: delivers the armed notice the moment the batch of
|
|
77
|
+
# tool results closes.
|
|
78
|
+
def message_ended(message)
|
|
79
|
+
return unless @batch.closed?(message)
|
|
80
|
+
return if @pending.nil?
|
|
81
|
+
|
|
82
|
+
remaining = @pending
|
|
83
|
+
@pending = nil
|
|
84
|
+
return if @batch.halted? # nothing will read it (halt_when): drop, never deliver
|
|
85
|
+
|
|
86
|
+
@warned << remaining
|
|
87
|
+
@chat.add_message(role: :user, content: self.class.notice(remaining, @max))
|
|
88
|
+
@emit.call(:tool_budget_warned, { remaining: remaining, max: @max })
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
end
|
data/lib/insika/turn_output.rb
CHANGED
|
@@ -33,7 +33,7 @@ module Insika
|
|
|
33
33
|
# answer; the fragment is still on the stream as `:intermediate` for whoever is
|
|
34
34
|
# debugging it.
|
|
35
35
|
#
|
|
36
|
-
# `halt_when`
|
|
36
|
+
# `halt_when` is the one case where narration IS the turn: the tool
|
|
37
37
|
# already answered the customer, and the model's lead-in ("vou te inscrever
|
|
38
38
|
# agora") is all the turn is worth. That text is retained and published by the
|
|
39
39
|
# Executor's halt branch — see #halt_text.
|
data/lib/insika/turn_state.rb
CHANGED
|
@@ -31,10 +31,15 @@ module Insika
|
|
|
31
31
|
:message_source, # WS9: how the turn's message entered (:voice when it came
|
|
32
32
|
# from a transcribed audio part; nil = typed). Rides the
|
|
33
33
|
# terminal event additively.
|
|
34
|
-
:media_attachments, # WS9: the ask's attachments (image parts);
|
|
35
|
-
# The provider bills them; usage flows like
|
|
36
|
-
# The first
|
|
37
|
-
#
|
|
34
|
+
:media_attachments, # WS9: the ask's attachments (image + document parts);
|
|
35
|
+
# nil = none. The provider bills them; usage flows like
|
|
36
|
+
# any ask. The first URL of each kind also lands on
|
|
37
|
+
# turn_context (`{{ctx.image_url}}` / `{{ctx.document_url}}`).
|
|
38
|
+
:image_attachments, # WS9: the IMAGE subset of media_attachments (nil = no
|
|
39
|
+
# inbound image this turn). The default edit SOURCE for
|
|
40
|
+
# generate_image — kept separate from
|
|
41
|
+
# media_attachments because that one may also hold a
|
|
42
|
+
# document, which is never a paint(with:) source.
|
|
38
43
|
:output_parts, # WS9 (saída): media the turn GENERATED, as additive
|
|
39
44
|
# parts ({ type:, mime_type:, base64:, model: }). Rides
|
|
40
45
|
# the terminal event (output_parts) and the /v1/responses
|
|
@@ -114,6 +119,12 @@ module Insika
|
|
|
114
119
|
# rules the run started under.
|
|
115
120
|
attr_accessor :queue_policy
|
|
116
121
|
|
|
122
|
+
# Internal: the run's SteerInjector, or nil when steering is off (or the chat
|
|
123
|
+
# cannot host the boundary). Rebuilt per ATTEMPT — a WS3 fallback builds a new
|
|
124
|
+
# chat, and the injector belongs to the chat it appends to. The Executor reaches
|
|
125
|
+
# for it at the end of stage 6 to absorb whatever no tool boundary took.
|
|
126
|
+
attr_accessor :steer_injector
|
|
127
|
+
|
|
117
128
|
# Internal: true when this turn re-enters the pipeline via
|
|
118
129
|
# resume_task/recovery. The EdgeLimiter reads it to NEVER re-count or block a
|
|
119
130
|
# turn that was already admitted — a crash/pause under a saturated window must
|
data/lib/insika/version.rb
CHANGED