insika 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +69 -3
- data/README.md +1 -1
- data/bin/insika +22 -7
- data/docs/AGENTS.md +129 -5
- data/docs/CHANNELS.md +1 -1
- data/docs/CONTEXT.md +22 -5
- data/docs/DEPLOY.md +30 -10
- data/docs/EMBEDDING.md +11 -7
- data/docs/EVALS.md +1 -1
- data/docs/LOADTEST.md +3 -2
- data/docs/OBSERVABILITY.md +11 -2
- data/docs/REFINEMENT.md +6 -6
- data/docs/RELEASING.md +7 -7
- data/docs/RUNNING-LOCAL.md +1 -1
- data/docs/SECURITY.md +24 -11
- data/docs/SKILLS.md +189 -3
- data/docs/WHY.md +1 -1
- data/docs/WORKFLOWS.md +2 -2
- data/docs/index.md +1 -1
- data/docs/onboarding/start.md +1 -1
- data/lib/insika/agent_profile.rb +89 -22
- data/lib/insika/alert_dispatcher.rb +139 -0
- data/lib/insika/baseline_store.rb +2 -2
- data/lib/insika/budget_ledger.rb +135 -0
- data/lib/insika/channel_delivery.rb +14 -11
- data/lib/insika/channel_registry.rb +1 -1
- data/lib/insika/channels/relay.rb +3 -3
- data/lib/insika/channels/web/widget.js +2 -2
- data/lib/insika/channels/web.rb +7 -7
- data/lib/insika/channels/webhook.rb +58 -0
- data/lib/insika/chat_builder.rb +62 -13
- data/lib/insika/circuit_state.rb +114 -0
- data/lib/insika/coercion.rb +8 -0
- data/lib/insika/commands/agent_payload.rb +5 -3
- data/lib/insika/commands/create_agent.rb +2 -2
- data/lib/insika/commands/create_session.rb +1 -1
- data/lib/insika/commands/delete_llm_provider.rb +1 -1
- data/lib/insika/commands/delete_skill.rb +43 -0
- data/lib/insika/commands/gate_refinement.rb +12 -12
- data/lib/insika/commands/import_mcp_tools.rb +1 -1
- data/lib/insika/commands/import_tools.rb +4 -4
- data/lib/insika/commands/issue_tenant_token.rb +41 -0
- data/lib/insika/commands/resolve_refinement.rb +1 -1
- data/lib/insika/commands/revoke_token.rb +39 -0
- data/lib/insika/commands/rotate_tenant_token.rb +43 -0
- data/lib/insika/commands/run_refinement.rb +5 -5
- data/lib/insika/commands/send_message.rb +9 -9
- data/lib/insika/commands/set_agent_tools.rb +1 -1
- data/lib/insika/commands/set_skill_agents.rb +60 -19
- data/lib/insika/commands/trigger_workflow.rb +1 -1
- data/lib/insika/commands/update_agent.rb +1 -1
- data/lib/insika/commands/write_data_tool.rb +1 -1
- data/lib/insika/commands/write_golden.rb +1 -1
- data/lib/insika/commands/write_skill.rb +19 -9
- data/lib/insika/config_store.rb +8 -4
- data/lib/insika/context/builder.rb +2 -2
- data/lib/insika/context/fragment.rb +27 -3
- data/lib/insika/context/priority.rb +3 -2
- data/lib/insika/context/providers/memory.rb +1 -1
- data/lib/insika/context/providers/request.rb +1 -1
- data/lib/insika/context/providers/session.rb +17 -2
- data/lib/insika/context/providers/skill.rb +5 -1
- data/lib/insika/context/providers/skill_trigger.rb +128 -0
- data/lib/insika/context_trace_store.rb +92 -0
- data/lib/insika/delegation_store.rb +2 -2
- data/lib/insika/doctor.rb +250 -5
- data/lib/insika/dsl/runtime.rb +12 -9
- data/lib/insika/dsl/server_boot.rb +4 -3
- data/lib/insika/dsl/system.rb +1 -1
- data/lib/insika/dsl.rb +72 -15
- data/lib/insika/edge_limiter.rb +144 -6
- data/lib/insika/egress_guard.rb +3 -3
- data/lib/insika/env_schema.rb +13 -10
- data/lib/insika/errors.rb +61 -5
- data/lib/insika/evals/assertions.rb +12 -12
- data/lib/insika/evals/baseline.rb +3 -3
- data/lib/insika/evals/golden.rb +8 -8
- data/lib/insika/evals/judge.rb +7 -7
- data/lib/insika/evals/pairwise.rb +3 -3
- data/lib/insika/evals/report.rb +2 -2
- data/lib/insika/evals/runner.rb +6 -6
- data/lib/insika/evals/transport.rb +2 -2
- data/lib/insika/event_stream.rb +23 -5
- data/lib/insika/executor.rb +423 -108
- data/lib/insika/frontmatter.rb +1 -1
- data/lib/insika/golden_store.rb +2 -2
- data/lib/insika/http_client.rb +3 -3
- data/lib/insika/inbound_log.rb +1 -1
- data/lib/insika/llm_configurator.rb +3 -3
- data/lib/insika/loop_detector.rb +143 -0
- data/lib/insika/mcp_http_client.rb +4 -4
- data/lib/insika/mcp_tool_ingestor.rb +6 -6
- data/lib/insika/message_origin.rb +2 -2
- data/lib/insika/model_resolver.rb +1 -1
- data/lib/insika/model_selection.rb +5 -4
- data/lib/insika/onboarding.rb +2 -2
- data/lib/insika/outbox_store.rb +2 -2
- data/lib/insika/overlay_tool_registry.rb +3 -4
- data/lib/insika/pack.rb +3 -3
- data/lib/insika/pack_importer.rb +17 -15
- data/lib/insika/pending_action_store.rb +1 -1
- data/lib/insika/plugin/loader.rb +2 -2
- data/lib/insika/policy/policy.rb +1 -1
- data/lib/insika/profile_source.rb +12 -6
- data/lib/insika/provider_error_classifier.rb +160 -0
- data/lib/insika/queue_policy.rb +2 -2
- data/lib/insika/recovery.rb +47 -6
- data/lib/insika/refinement/candidate.rb +4 -4
- data/lib/insika/refinement/evidence_collector.rb +6 -6
- data/lib/insika/refinement/gate.rb +7 -7
- data/lib/insika/refinement/panel.rb +7 -7
- data/lib/insika/refinement/proposer.rb +9 -9
- data/lib/insika/refinement_store.rb +12 -12
- data/lib/insika/reliability.rb +185 -0
- data/lib/insika/safety/config.rb +2 -2
- data/lib/insika/safety/detectors.rb +5 -5
- data/lib/insika/safety/factory.rb +3 -3
- data/lib/insika/safety/input_guardrail.rb +19 -4
- data/lib/insika/safety/moderator.rb +19 -11
- data/lib/insika/safety/output_filter.rb +2 -2
- data/lib/insika/safety/output_validator.rb +2 -2
- data/lib/insika/safety/safe_responses.rb +1 -1
- data/lib/insika/sandbox/boundary.rb +2 -2
- data/lib/insika/sandbox.rb +1 -1
- data/lib/insika/server/app.rb +223 -51
- data/lib/insika/server/boot.rb +4 -4
- data/lib/insika/server/rack_app.rb +15 -7
- data/lib/insika/server/responses.rb +18 -8
- data/lib/insika/server/tenant_auth.rb +61 -0
- data/lib/insika/session_actor.rb +3 -3
- data/lib/insika/session_store.rb +1 -1
- data/lib/insika/settings_store.rb +5 -5
- data/lib/insika/shutdown.rb +4 -4
- data/lib/insika/skill_catalog.rb +127 -20
- data/lib/insika/skill_store.rb +70 -22
- data/lib/insika/steer_injector.rb +1 -1
- data/lib/insika/store.rb +1 -1
- data/lib/insika/studio/app.rb +183 -61
- data/lib/insika/studio/assets/dist/application.js +25 -24
- data/lib/insika/studio/forms.rb +13 -18
- data/lib/insika/studio/nav_icons.rb +1 -1
- data/lib/insika/studio/views/_message.erb +2 -2
- data/lib/insika/studio/views/agent_detail.erb +2 -2
- data/lib/insika/studio/views/agents.erb +1 -1
- data/lib/insika/studio/views/refinement.erb +4 -4
- data/lib/insika/studio/views/session.erb +78 -3
- data/lib/insika/studio/views/settings.erb +7 -12
- data/lib/insika/studio/views/skills.erb +67 -12
- data/lib/insika/subagent_graph.rb +3 -3
- data/lib/insika/task_actor.rb +3 -3
- data/lib/insika/task_store.rb +1 -1
- data/lib/insika/telemetry/pricing.rb +3 -3
- data/lib/insika/telemetry/recorder.rb +1 -1
- data/lib/insika/telemetry.rb +2 -2
- data/lib/insika/testing/store_contract.rb +27 -27
- data/lib/insika/tick.rb +122 -0
- data/lib/insika/token_store.rb +168 -0
- data/lib/insika/tool_assembly.rb +5 -5
- data/lib/insika/tool_definition.rb +8 -8
- data/lib/insika/tool_envelope.rb +1 -1
- data/lib/insika/tool_manifest.rb +6 -6
- data/lib/insika/tool_output_compressor.rb +100 -0
- data/lib/insika/tool_store.rb +1 -1
- data/lib/insika/tool_trace_store.rb +1 -1
- data/lib/insika/tools/concurrency.rb +2 -2
- data/lib/insika/tools/data_defined_tool.rb +4 -5
- data/lib/insika/tools/load_skill.rb +61 -3
- data/lib/insika/tools/stuck_signal.rb +44 -0
- data/lib/insika/tools/subagent.rb +4 -4
- data/lib/insika/tools/subagents.rb +1 -1
- data/lib/insika/turn_output.rb +2 -2
- data/lib/insika/turn_state.rb +17 -13
- data/lib/insika/turn_timing.rb +2 -2
- data/lib/insika/usage_ledger.rb +1 -1
- data/lib/insika/version.rb +1 -1
- data/lib/insika/wiring/graph.rb +77 -26
- data/lib/insika/workflow.rb +1 -1
- data/lib/insika/workflow_registry.rb +1 -1
- data/lib/insika.rb +32 -15
- metadata +19 -2
- data/lib/insika/server/admin_auth.rb +0 -29
data/lib/insika/dsl.rb
CHANGED
|
@@ -3,17 +3,17 @@
|
|
|
3
3
|
require_relative "pack"
|
|
4
4
|
|
|
5
5
|
module Insika
|
|
6
|
-
# Public Ruby DSL
|
|
6
|
+
# Public Ruby DSL — the OSS "business card":
|
|
7
7
|
#
|
|
8
8
|
# agent = Insika.agent("assistant") do
|
|
9
|
-
# model "deepseek-
|
|
9
|
+
# model "deepseek-v4-flash"
|
|
10
10
|
# instructions "You are a concise, friendly assistant."
|
|
11
11
|
# end
|
|
12
12
|
# puts agent.reply("hi, what can you do?") # one turn, in-process
|
|
13
13
|
# agent.serve # control UI + /v1 on :9292
|
|
14
14
|
#
|
|
15
15
|
# It is THIN SUGAR that GENERATES the data (a Insika::Pack), never a bypass of
|
|
16
|
-
# config-over-code (COMPETITIVE-ANALYSIS
|
|
16
|
+
# config-over-code (COMPETITIVE-ANALYSIS). `Insika.agent { … }.to_pack`
|
|
17
17
|
# is the same portable artifact the PackImporter consumes at runtime — the DSL
|
|
18
18
|
# and a hand-written pack produce the SAME profile (the parity spec proves it),
|
|
19
19
|
# because BOTH go through the standard import → StoredProfileSource round-trip.
|
|
@@ -153,7 +153,7 @@ module Insika
|
|
|
153
153
|
|
|
154
154
|
# A DATA-DEFINED (declarative HTTP) tool — pure config-over-code. `defn` is a
|
|
155
155
|
# ToolDefinition hash (name/description/parameters/binding…). Its name is
|
|
156
|
-
# auto-added to the allowlist so the agent can call its own tool
|
|
156
|
+
# auto-added to the allowlist so the agent can call its own tool.
|
|
157
157
|
def data_tool(defn)
|
|
158
158
|
h = defn.transform_keys(&:to_s)
|
|
159
159
|
@tools << h
|
|
@@ -173,9 +173,31 @@ module Insika
|
|
|
173
173
|
n
|
|
174
174
|
end
|
|
175
175
|
|
|
176
|
+
# skills_eager — turns progressive disclosure off for THIS agent, wholly or in
|
|
177
|
+
# part. The body of an eager skill is in the prompt on every turn, so its
|
|
178
|
+
# activation is not a decision and cannot be missed; it is paid for on every
|
|
179
|
+
# turn, so measure the bodies against `context_budget` first (a stable position
|
|
180
|
+
# makes them a cacheable prefix).
|
|
181
|
+
#
|
|
182
|
+
# skills_eager # every allowed skill
|
|
183
|
+
# skills_eager "formato", "markers" # exactly these
|
|
184
|
+
# skills_eager false # none (the default)
|
|
185
|
+
#
|
|
186
|
+
# A LIST and not a per-skill flag because skills are shared: `escalation-to-human`
|
|
187
|
+
# sits in several allowlists, and one flag on the skill would force one decision
|
|
188
|
+
# onto every agent holding it.
|
|
189
|
+
def skills_eager(*names)
|
|
190
|
+
flat = names.flatten
|
|
191
|
+
@config[:skills_eager] =
|
|
192
|
+
if flat.empty? then true
|
|
193
|
+
elsif flat == [true] || flat == [false] then flat.first
|
|
194
|
+
else flat.map(&:to_s)
|
|
195
|
+
end
|
|
196
|
+
end
|
|
197
|
+
|
|
176
198
|
# --- delegation ------------------------------------------------------
|
|
177
199
|
# subagents "security", "performance" → the child agents this one MAY
|
|
178
|
-
# spawn
|
|
200
|
+
# spawn. CAPACITY field: opt-in, never inherited, and the ids
|
|
179
201
|
# must be agents of the same system (`Insika.system { … }`) or already in
|
|
180
202
|
# the store. Present ⇒ the engine wires `spawn_subagent`/`spawn_subagents`.
|
|
181
203
|
def subagents(*ids)
|
|
@@ -185,23 +207,58 @@ module Insika
|
|
|
185
207
|
# --- knobs -----------------------------------------------------------
|
|
186
208
|
def memory(on = true) = @config[:memory] = on
|
|
187
209
|
|
|
188
|
-
#
|
|
210
|
+
# Spend caps per calendar window (WS2): daily/monthly token budgets for
|
|
211
|
+
# this agent, per (tenant, agent) when multi-tenant. HARD is the default:
|
|
212
|
+
# absent `soft:` (or `soft: false`) turns the cap into a hard wall (the
|
|
213
|
+
# turn fails with budget_exceeded + retry_after); `soft: true` warns once
|
|
214
|
+
# per window and keeps running.
|
|
215
|
+
# budget daily: 100_000, monthly: 2_000_000, soft: false
|
|
216
|
+
def budget(hash) = (@config[:budget] ||= {}).merge!(hash.transform_keys(&:to_s))
|
|
217
|
+
|
|
218
|
+
# Provider-interaction reliability, as DATA (WS3): retries + exponential
|
|
219
|
+
# backoff on transient failures, a fallback model chain (mid-turn
|
|
220
|
+
# rotation), and a circuit breaker per (tenant, provider/model) that
|
|
221
|
+
# fail-fasts once the window trips. `fallback`/`circuit_breaker` entries
|
|
222
|
+
# are "provider/model" refs or plain model ids.
|
|
223
|
+
# reliability retries: 3, backoff: "exponential",
|
|
224
|
+
# fallback: ["gpt-4o-mini"], circuit_breaker: { after: 10, within: 60, cooldown: 300 }
|
|
225
|
+
def reliability(hash)
|
|
226
|
+
(@config[:reliability] ||= {}).merge!(hash.transform_keys(&:to_s))
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
# Operator alert delivery (WS6): POST this agent's budget_warning /
|
|
230
|
+
# breaker_open / delivery_failed events to the webhook, as JSON.
|
|
231
|
+
# alerts webhook: "https://ops.example.com/insika-alerts"
|
|
232
|
+
def alerts(hash) = (@config[:alerts] ||= {}).merge!(hash.transform_keys(&:to_s))
|
|
233
|
+
|
|
234
|
+
# The agent may signal it cannot proceed (WS5): when on, the model
|
|
235
|
+
# can call `signal_stuck`, which ends the turn with `outcome: :stuck` + a final
|
|
236
|
+
# message + a `:turn_stuck` event. What "stuck" means is the consumer's call.
|
|
237
|
+
# stuck_signal true
|
|
238
|
+
def stuck_signal(on = true) = @config[:stuck_signal] = on
|
|
239
|
+
|
|
240
|
+
# Mechanical tool-result dedupe in the replayed history
|
|
241
|
+
# (no-LLM compaction, apt for bloated transcripts). CHANGES WHAT THE MODEL
|
|
242
|
+
# SEES: repeated identical tool results collapse to a back-reference.
|
|
243
|
+
def tool_output_compression(on = true) = @config[:tool_output_compression] = on
|
|
244
|
+
|
|
245
|
+
# Content-safety guardrails — opt-in and configurable per agent.
|
|
189
246
|
# Pure config-over-code: the hash is stored on the profile and consumed by
|
|
190
247
|
# Safety::Config.from_profile. Merges, so repeated calls accumulate.
|
|
191
248
|
# guardrails input: true, output: true, strictness: "medium",
|
|
192
|
-
# moderator: "deepseek/deepseek-
|
|
249
|
+
# moderator: "deepseek/deepseek-v4-flash",
|
|
193
250
|
# responses: { "injection" => "I can't help with that." }
|
|
194
251
|
def guardrails(hash) = (@config[:guardrails] ||= {}).merge!(hash.transform_keys(&:to_s))
|
|
195
252
|
|
|
196
|
-
# Refinement
|
|
253
|
+
# Refinement — how the agent's own instruction files may be
|
|
197
254
|
# improved from real traffic. Same config-over-code shape as `guardrails`;
|
|
198
|
-
# omitting it entirely leaves the agent report-only (
|
|
255
|
+
# omitting it entirely leaves the agent report-only (writes nothing).
|
|
199
256
|
# refine mode: "propose", window: { last_sessions: 200 }, files: %w[TOOLS.md],
|
|
200
|
-
# proposers: ["deepseek/deepseek-
|
|
257
|
+
# proposers: ["deepseek/deepseek-v4-flash", "gpt-5-mini"],
|
|
201
258
|
# budget: { tokens: 200_000 }
|
|
202
259
|
def refine(hash) = (@config[:refinement] ||= {}).merge!(hash.transform_keys(&:to_s))
|
|
203
260
|
|
|
204
|
-
# Facts about THIS deployment that are not tools
|
|
261
|
+
# Facts about THIS deployment that are not tools, so an eval
|
|
205
262
|
# case can declare what it needs and be skipped where it is absent instead of
|
|
206
263
|
# failing for the wrong reason.
|
|
207
264
|
# declares "promotions", "human_handoff"
|
|
@@ -217,7 +274,7 @@ module Insika
|
|
|
217
274
|
# edge_stream thinking: true, intermediate: false
|
|
218
275
|
def edge_stream(hash) = (@config[:edge_stream] ||= {}).merge!(hash.transform_keys(&:to_s))
|
|
219
276
|
|
|
220
|
-
# LLM generation params
|
|
277
|
+
# LLM generation params. `param:temperature, 0.2` or `params(...)`.
|
|
221
278
|
def param(key, value) = (@config[:params] ||= {})[key.to_sym] = value
|
|
222
279
|
def params(hash) = (@config[:params] ||= {}).merge!(hash.transform_keys(&:to_sym))
|
|
223
280
|
def temperature(value) = param(:temperature, value)
|
|
@@ -281,14 +338,14 @@ module Insika
|
|
|
281
338
|
DSL.system(&block)
|
|
282
339
|
end
|
|
283
340
|
|
|
284
|
-
#
|
|
341
|
+
# the front door for MOUNTING Insika into an app you already
|
|
285
342
|
# have. Same block as `Insika.system`, one added obligation: the caller names
|
|
286
343
|
# the store, so the graph stops discovering it from `INSIKA_DB` and two graphs
|
|
287
344
|
# in one process can no longer read each other's sessions.
|
|
288
345
|
#
|
|
289
346
|
# INSIKA = Insika.embed(backend: Insika::Stores::SQLite.new(path: "storage/insika.sqlite3")) do
|
|
290
347
|
# agent "support" do
|
|
291
|
-
# model "deepseek-
|
|
348
|
+
# model "deepseek-v4-flash"
|
|
292
349
|
# instructions "…"
|
|
293
350
|
# end
|
|
294
351
|
# end
|
|
@@ -296,7 +353,7 @@ module Insika
|
|
|
296
353
|
# mount Insika::Server.rack_app(INSIKA, token: ENV.fetch("INSIKA_TOKEN")), at: "/ai"
|
|
297
354
|
#
|
|
298
355
|
# It is a thin front door over the SAME assembly `Insika.system` uses — there is
|
|
299
|
-
# one pipeline
|
|
356
|
+
# one pipeline, and the parity spec holds it to that. What an embedded
|
|
300
357
|
# graph owns, and what it still shares with the process, is docs/EMBEDDING.md.
|
|
301
358
|
def embed(backend:, &block)
|
|
302
359
|
DSL.embed(backend: backend, &block)
|
data/lib/insika/edge_limiter.rb
CHANGED
|
@@ -4,7 +4,7 @@ require_relative "middleware"
|
|
|
4
4
|
require_relative "coercion"
|
|
5
5
|
|
|
6
6
|
module Insika
|
|
7
|
-
# The production edge
|
|
7
|
+
# The production edge: THE named place where volume/cost
|
|
8
8
|
# abuse is cut. A Middleware with two independent limits, both OPT-IN
|
|
9
9
|
# (nil/0 = off — a bare wiring behaves exactly as before):
|
|
10
10
|
#
|
|
@@ -13,16 +13,29 @@ module Insika
|
|
|
13
13
|
# · agent token ceiling — total tokens per agent per window. Checked on entry
|
|
14
14
|
# against the accumulated ledger; the turn's own usage is recorded AFTER the
|
|
15
15
|
# terminal returns (the Middleware wraps stages 5-9, so state.usage is set).
|
|
16
|
+
# · calendar budget — WS2: `AgentProfile#budget` caps the spend per
|
|
17
|
+
# (tenant, agent) over CALENDAR windows (daily/monthly), on the
|
|
18
|
+
# BudgetLedger. Hard (default): crossing the cap raises the typed
|
|
19
|
+
# Insika::BudgetExceeded (the envelope quotes budget_exceeded +
|
|
20
|
+
# retry_after); soft: crossing warns instead — ONE budget_warning event
|
|
21
|
+
# per window plus a note injected into the context. Crossing `alert_at`
|
|
22
|
+
# (default 0.8 of the cap) warns the same way, before the wall. The turn's
|
|
23
|
+
# billed spend (input+output+cached+cache_creation — the A4 rule) lands on
|
|
24
|
+
# the windows after the terminal.
|
|
16
25
|
#
|
|
17
26
|
# Config resolution, per turn (configuration over convention):
|
|
18
27
|
# profile.limits[:chat_rate_limit / :agent_token_ceiling] — per-agent override
|
|
19
28
|
# settings["edge"] — platform default
|
|
20
29
|
# A per-agent 0 explicitly disables a platform default for that agent.
|
|
21
30
|
#
|
|
22
|
-
# On breach it uses the graceful-halt contract
|
|
31
|
+
# On breach it uses the graceful-halt contract: halt_response
|
|
23
32
|
# (the safe reply) + guardrail_block (audit -> :guardrail_blocked) and does NOT
|
|
24
33
|
# call `nxt` — the turn completes with ZERO LLM calls. It sits BEFORE the
|
|
25
34
|
# InputGuardrail in the stack so a flood can't spend the LLM moderator either.
|
|
35
|
+
# The BUDGET breach is the ONE deliberate exception: it is a typed failure
|
|
36
|
+
# (BudgetExceeded), not a customer-facing reply — the operator wants the
|
|
37
|
+
# envelope to say "budget" and quote when the window rolls, not to hand the
|
|
38
|
+
# customer a cost message.
|
|
26
39
|
class EdgeLimiter < Middleware
|
|
27
40
|
CHAT_KIND = "chat"
|
|
28
41
|
TOKENS_KIND = "tokens"
|
|
@@ -35,9 +48,13 @@ module Insika
|
|
|
35
48
|
DEFAULT_RESPONSE = "Estou recebendo muitas mensagens agora. Aguarde um " \
|
|
36
49
|
"momento e tente novamente, por favor."
|
|
37
50
|
|
|
38
|
-
def initialize(ledger:, settings_store: nil)
|
|
51
|
+
def initialize(ledger:, settings_store: nil, budget_ledger: nil, event_stream: nil)
|
|
39
52
|
@ledger = ledger
|
|
40
53
|
@settings = settings_store
|
|
54
|
+
# WS2: the calendar-window ledger. nil = budget off (parity — the bare
|
|
55
|
+
# wiring is byte-identical to before).
|
|
56
|
+
@budget_ledger = budget_ledger
|
|
57
|
+
@event_stream = event_stream
|
|
41
58
|
end
|
|
42
59
|
|
|
43
60
|
def call(state, &nxt)
|
|
@@ -69,8 +86,25 @@ module Insika
|
|
|
69
86
|
record_after = token_window
|
|
70
87
|
end
|
|
71
88
|
|
|
72
|
-
|
|
73
|
-
|
|
89
|
+
# WS2: calendar budgets. Entry — a HARD budget at/over the cap raises the
|
|
90
|
+
# typed error (never a customer-facing reply); the alert_at warning and
|
|
91
|
+
# the SOFT over-cap both warn once per window + inject a context note.
|
|
92
|
+
# A resumed turn (crash/pause replay) was already admitted: it is never
|
|
93
|
+
# refused twice — its spend still lands on the ledger below.
|
|
94
|
+
budget_on = budget_configured?(state)
|
|
95
|
+
budget_enforce(state) unless resumed
|
|
96
|
+
|
|
97
|
+
result = begin
|
|
98
|
+
nxt.call(state)
|
|
99
|
+
ensure
|
|
100
|
+
# A turn that FAILED after burning tokens still SPENT them: record the
|
|
101
|
+
# usage the state captured before the error propagates. The ask's usage
|
|
102
|
+
# lands on state.usage before any later stage (guardrail block, tool
|
|
103
|
+
# error, workflow schema) can fail the turn — a failed turn must count
|
|
104
|
+
# against the budget like a completed one (WS2).
|
|
105
|
+
record_usage(state, record_after) if record_after
|
|
106
|
+
record_budget_usage(state) if budget_on
|
|
107
|
+
end
|
|
74
108
|
result
|
|
75
109
|
end
|
|
76
110
|
|
|
@@ -101,7 +135,7 @@ module Insika
|
|
|
101
135
|
# prefix (`Executor#usage_of` reports `cached_tokens`/`cache_creation_tokens`
|
|
102
136
|
# alongside it) — on a cached identity that prefix is ~95% of what the
|
|
103
137
|
# provider actually processed, so a ceiling reading only `total_tokens` is
|
|
104
|
-
# blind
|
|
138
|
+
# blind. Same billed-spend rule as `Evals::Runner#billed_tokens`.
|
|
105
139
|
# nil usage (workflow turn / provider without counts) records nothing.
|
|
106
140
|
def record_usage(state, window)
|
|
107
141
|
usage = state.usage || {}
|
|
@@ -126,5 +160,109 @@ module Insika
|
|
|
126
160
|
v = value.to_i
|
|
127
161
|
v.positive? ? v : nil
|
|
128
162
|
end
|
|
163
|
+
|
|
164
|
+
# --- WS2 calendar budgets ------------------------------------------
|
|
165
|
+
|
|
166
|
+
# -> truthy when a budget is configured AND the ledger is wired.
|
|
167
|
+
def budget_configured?(state)
|
|
168
|
+
budget = state.profile.respond_to?(:budget) ? state.profile.budget : nil
|
|
169
|
+
!budget.nil? && !@budget_ledger.nil?
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
# -> truthy (the budget hash) when budget checks ran. Raises BudgetExceeded
|
|
173
|
+
# on a HARD cap breach.
|
|
174
|
+
def budget_enforce(state, now: Time.now)
|
|
175
|
+
budget = state.profile.respond_to?(:budget) ? state.profile.budget : nil
|
|
176
|
+
return nil if budget.nil? || @budget_ledger.nil?
|
|
177
|
+
|
|
178
|
+
tenant = budget_tenant(state)
|
|
179
|
+
agent = state.profile.id.to_s
|
|
180
|
+
budget_windows(budget).each do |w|
|
|
181
|
+
spent = @budget_ledger.current(tenant: tenant, agent: agent, now: now)[w[:window]]
|
|
182
|
+
if spent >= w[:cap]
|
|
183
|
+
unless w[:soft]
|
|
184
|
+
raise Insika::BudgetExceeded.new(
|
|
185
|
+
window: w[:window],
|
|
186
|
+
retry_after: @budget_ledger.reset_in(w[:window], now: now)
|
|
187
|
+
)
|
|
188
|
+
end
|
|
189
|
+
warn_budget(state, tenant, agent, w, spent, now, level: "cap")
|
|
190
|
+
elsif spent >= w[:alert_at]
|
|
191
|
+
warn_budget(state, tenant, agent, w, spent, now, level: "alert_at")
|
|
192
|
+
end
|
|
193
|
+
end
|
|
194
|
+
budget
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
# The (tenant, agent) scope: the COMMAND's tenant (nil -> the BudgetLedger's
|
|
198
|
+
# "platform" cell) — never state.tenant, which falls back to the session id
|
|
199
|
+
# (a per-chat bucket is not a budget).
|
|
200
|
+
def budget_tenant(state)
|
|
201
|
+
command = state.respond_to?(:task) && state.task&.command
|
|
202
|
+
return nil unless command.is_a?(Hash)
|
|
203
|
+
|
|
204
|
+
meta = command["meta"] || command[:meta] || {}
|
|
205
|
+
meta["tenant"] || meta[:tenant]
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
# -> [{ window:, cap:, soft:, alert_at: }] — one entry per configured window
|
|
209
|
+
# (a 0/absent cap is off). absent `soft` = FALSE (hard): a limit that does
|
|
210
|
+
# not limit is decoration; the alert_at warning is the soft half.
|
|
211
|
+
def budget_windows(budget)
|
|
212
|
+
alert_at = budget["alert_at"].to_f
|
|
213
|
+
alert_at = 0.8 if alert_at <= 0 || alert_at >= 1
|
|
214
|
+
soft = budget["soft"] == true
|
|
215
|
+
%i[daily monthly].filter_map do |window|
|
|
216
|
+
cap = budget[window.to_s].to_i
|
|
217
|
+
cap.positive? ? { window: window, cap: cap, soft: soft,
|
|
218
|
+
alert_at: (cap * alert_at).floor } : nil
|
|
219
|
+
end
|
|
220
|
+
end
|
|
221
|
+
|
|
222
|
+
# The warning: a note in the context (the model sees it, the customer's
|
|
223
|
+
# transcript does not) + the budget_warning event — each LEVEL once per
|
|
224
|
+
# (window) cell: the `alert_at` crossing and the real soft-cap crossing are
|
|
225
|
+
# separate markers, so the cap event is never swallowed by the 80% one that
|
|
226
|
+
# fired earlier (WS2).
|
|
227
|
+
def warn_budget(state, tenant, agent, w, spent, now, level:)
|
|
228
|
+
inject_budget_note(state,
|
|
229
|
+
"[budget: agent '#{agent}' is at #{spent}/#{w[:cap]} tokens this " \
|
|
230
|
+
"#{w[:window]} window — keep this turn cheap]")
|
|
231
|
+
return if @budget_ledger.mark_alert(tenant: tenant, agent: agent, window: w[:window],
|
|
232
|
+
level: level, now: now)
|
|
233
|
+
|
|
234
|
+
@event_stream&.emit(Insika::Event.new(
|
|
235
|
+
type: :budget_warning,
|
|
236
|
+
data: { agent: agent, tenant: tenant, window: w[:window],
|
|
237
|
+
spent: spent, cap: w[:cap], level: level },
|
|
238
|
+
meta: { task_id: state.task&.id, session_id: state.task&.session_id,
|
|
239
|
+
at: Time.now.utc.iso8601 }
|
|
240
|
+
))
|
|
241
|
+
end
|
|
242
|
+
|
|
243
|
+
# Appends the note to the assembled system prompt: the real Data package is
|
|
244
|
+
# immutable (with), the specs' minimal Struct is mutable — both duck-typed.
|
|
245
|
+
def inject_budget_note(state, note)
|
|
246
|
+
ctx = state.context
|
|
247
|
+
return if ctx.nil?
|
|
248
|
+
|
|
249
|
+
if ctx.respond_to?(:with)
|
|
250
|
+
state.context = ctx.with(system: "#{ctx.system}\n\n#{note}")
|
|
251
|
+
elsif ctx.respond_to?(:system=)
|
|
252
|
+
ctx.system = "#{ctx.system}\n\n#{note}"
|
|
253
|
+
end
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
# The turn's REAL billed spend (input + output + cached + cache_creation —
|
|
257
|
+
# the A4 rule) on the calendar windows.
|
|
258
|
+
def record_budget_usage(state, now: Time.now)
|
|
259
|
+
usage = state.usage || {}
|
|
260
|
+
tokens = usage[:total_tokens].to_i + usage[:cached_tokens].to_i +
|
|
261
|
+
usage[:cache_creation_tokens].to_i
|
|
262
|
+
return if tokens.zero?
|
|
263
|
+
|
|
264
|
+
@budget_ledger.add(tenant: budget_tenant(state), agent: state.profile.id.to_s,
|
|
265
|
+
by: tokens, now: now)
|
|
266
|
+
end
|
|
129
267
|
end
|
|
130
268
|
end
|
data/lib/insika/egress_guard.rb
CHANGED
|
@@ -8,14 +8,14 @@ module Insika
|
|
|
8
8
|
# EGRESS guard for data-tools (SSRF). A data-tool makes a server-side HTTP
|
|
9
9
|
# request with a URL coming from UI-editable config — without a guard, it's an
|
|
10
10
|
# SSRF vector (hitting cloud metadata, internal services, localhost). Rules
|
|
11
|
-
# (spec
|
|
11
|
+
# (spec):
|
|
12
12
|
# - https only by default (http requires explicit opt-in);
|
|
13
13
|
# - host required;
|
|
14
14
|
# - optional host allowlist (when present, only it passes);
|
|
15
15
|
# - resolves the host and BLOCKS if ANY address falls into a private/
|
|
16
16
|
# loopback/link-local/metadata network (defense against DNS rebinding);
|
|
17
17
|
# - `allow_private:` (opt-in) ALLOWS the private target — to reach a trusted
|
|
18
|
-
# INTERNAL API (
|
|
18
|
+
# INTERNAL API (the consumer's /api/internal/* comes in via an
|
|
19
19
|
# allowlist). Dangerous without `host_allowlist`: PIN it to a known host.
|
|
20
20
|
# Default false = strict guard.
|
|
21
21
|
#
|
|
@@ -48,7 +48,7 @@ module Insika
|
|
|
48
48
|
addrs = resolve(host)
|
|
49
49
|
return "host did not resolve" if addrs.empty?
|
|
50
50
|
# allow_private skips the private-network block (trusted internal API,
|
|
51
|
-
#
|
|
51
|
+
# Without it, a private/loopback/metadata target is always blocked.
|
|
52
52
|
return "private-network destination blocked" if !allow_private && addrs.any? { |ip| blocked?(ip) }
|
|
53
53
|
|
|
54
54
|
nil
|
data/lib/insika/env_schema.rb
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Insika
|
|
4
|
-
# STRICT config, environment layer
|
|
4
|
+
# STRICT config, environment layer. OpenClaw's config discipline
|
|
5
5
|
# — "recusa boot com chave desconhecida, no silent config compat" — applied to the
|
|
6
6
|
# env vars the engine reads at boot. A declarative registry of the keys the engine
|
|
7
7
|
# OWNS (config over convention: the schema IS data), used two ways:
|
|
@@ -12,7 +12,7 @@ module Insika
|
|
|
12
12
|
# silence), and a DEPRECATED legacy key still set under the old HARNESS_ prefix.
|
|
13
13
|
# Unknown-key detection is scoped to the OWNED prefixes only, so the platform's
|
|
14
14
|
# own vars (Railway's RAILWAY_*, PORT, PATH, the litestream sidecar's
|
|
15
|
-
# LITESTREAM_*, the deployment's DEEPSEEK_*/
|
|
15
|
+
# LITESTREAM_*, the deployment's DEEPSEEK_*/CONSUMER_*) are never flagged.
|
|
16
16
|
# · `enforce!(strict:)` — the boot gate. WARNS on every finding by default and
|
|
17
17
|
# lets the engine come up (last-known-good — a rotated key or a typo must never
|
|
18
18
|
# take the whole service down, same reasoning as the resilient DEEPSEEK boot);
|
|
@@ -92,7 +92,7 @@ module Insika
|
|
|
92
92
|
Spec.new(name: name, type: type, secret: secret, required: required, enum: enum, description: description)
|
|
93
93
|
end
|
|
94
94
|
|
|
95
|
-
# The engine's own keys. Deployment/app keys (DEEPSEEK_*,
|
|
95
|
+
# The engine's own keys. Deployment/app keys (DEEPSEEK_*, CONSUMER_*, …) are NOT
|
|
96
96
|
# here — a root passes them as `extra:`.
|
|
97
97
|
DEFAULT = [
|
|
98
98
|
spec(name: "INSIKA_DB", type: :path, description: "SQLite path; durable config+state. Unset -> ephemeral memory."),
|
|
@@ -107,17 +107,20 @@ module Insika
|
|
|
107
107
|
spec(name: "INSIKA_EGRESS_HOSTS", type: :csv, description: "Comma-separated host allowlist for data-tool egress."),
|
|
108
108
|
spec(name: "INSIKA_OTEL", type: :boolean, description: "Turn on OpenTelemetry export (opt-in)."),
|
|
109
109
|
spec(name: "INSIKA_MODEL_PRICING", description: "JSON rates table (USD per million tokens) for the estimated-cost attribute; unset -> no cost reported."),
|
|
110
|
-
spec(name: "INSIKA_TURN_TIMING", type: :boolean, description: "Emit per-turn TTFB breakdown in responses (opt-in
|
|
110
|
+
spec(name: "INSIKA_TURN_TIMING", type: :boolean, description: "Emit per-turn TTFB breakdown in responses (opt-in)."),
|
|
111
111
|
spec(name: "INSIKA_SUBAGENT_DEPTH_CAP", type: :integer, description: "Max delegation depth in the subagent graph (default 5)."),
|
|
112
112
|
spec(name: "INSIKA_SUBAGENT_FANOUT_CAP", type: :integer, description: "Max parallel children in spawn_subagents (default 8)."),
|
|
113
|
-
spec(name: "INSIKA_CONFIG_STRICT", type: :boolean, description: "Refuse boot on any config finding instead of warning
|
|
114
|
-
spec(name: "INSIKA_BOOT_ID", description: "Boot generation id shared by all workers of one container start; the recovery task sweep runs once per id
|
|
115
|
-
spec(name: "INSIKA_DRAIN_TIMEOUT", type: :integer, description: "Seconds a stopping worker waits for in-flight turns before abandoning them to the next boot's recovery (default 20
|
|
116
|
-
spec(name: "
|
|
117
|
-
spec(name: "
|
|
113
|
+
spec(name: "INSIKA_CONFIG_STRICT", type: :boolean, description: "Refuse boot on any config finding instead of warning."),
|
|
114
|
+
spec(name: "INSIKA_BOOT_ID", description: "Boot generation id shared by all workers of one container start; the recovery task sweep runs once per id. Unset -> every boot sweeps."),
|
|
115
|
+
spec(name: "INSIKA_DRAIN_TIMEOUT", type: :integer, description: "Seconds a stopping worker waits for in-flight turns before abandoning them to the next boot's recovery (default 20)."),
|
|
116
|
+
spec(name: "INSIKA_TICK_INTERVAL", type: :integer, description: "Seconds between tick passes (outbox drain + stale recovery sweep). Default 60; 0 disables."),
|
|
117
|
+
spec(name: "INSIKA_TICK_STALE_AFTER", type: :integer, description: "Seconds a :queued/:running task must sit untouched before the tick sweeps it (default 900). Must exceed the largest turn_timeout of the deployment."),
|
|
118
|
+
spec(name: "INSIKA_TENANCY", enum: %w[single_tenant multi_tenant], description: "single_tenant (default: one operator credential) or multi_tenant (per-tenant + operator tokens resolved from the store)."),
|
|
119
|
+
spec(name: "INSIKA_ONBOARDING", type: :boolean, description: "Expose the public onboarding surface (/start.md, /models.json, /docs) in production (opt-in)."),
|
|
120
|
+
spec(name: "INSIKA_RELAY_TOKEN", secret: true, description: "Bearer the relay consumer sends us. Unset -> the relay channel is not mounted."),
|
|
118
121
|
spec(name: "INSIKA_RELAY_DELIVER_URL", type: :url, description: "Consumer callback the relay POSTs each reply to."),
|
|
119
122
|
spec(name: "INSIKA_RELAY_DELIVER_TOKEN", secret: true, description: "Bearer the relay sends TO the consumer's callback (optional)."),
|
|
120
|
-
spec(name: "INSIKA_WIDGET_ORIGINS", type: :csv, description: "Exact-match origins allowed to embed the web widget. Unset -> the widget channel is not mounted
|
|
123
|
+
spec(name: "INSIKA_WIDGET_ORIGINS", type: :csv, description: "Exact-match origins allowed to embed the web widget. Unset -> the widget channel is not mounted."),
|
|
121
124
|
spec(name: "INSIKA_WIDGET_AGENTS", type: :csv, description: "Agent ids a widget visitor may address. Unset -> the widget channel is not mounted."),
|
|
122
125
|
spec(name: "OPENCLAW_GATEWAY_TOKEN", secret: true, description: "Bearer for /v1 + /a2a (falls back to ADMIN_TOKEN)."),
|
|
123
126
|
spec(name: "OPENCLAW_AGENTS_DIR", type: :path, description: "Directory of OpenClaw-style agent packs."),
|
data/lib/insika/errors.rb
CHANGED
|
@@ -30,7 +30,63 @@ module Insika
|
|
|
30
30
|
end
|
|
31
31
|
end
|
|
32
32
|
|
|
33
|
-
|
|
33
|
+
# A provider/transport failure, wrapped by ProviderErrorClassifier with an
|
|
34
|
+
# ACTION classification (B9). The fields ride in the task's error record and
|
|
35
|
+
# the :task_failed event so the client envelope can tell fatal from
|
|
36
|
+
# retryable and quote the provider's own retry_after (A8). A bare
|
|
37
|
+
# `ProviderError.new("boom")` has an empty classification and adds nothing
|
|
38
|
+
# to the contract.
|
|
39
|
+
class ProviderError < Error
|
|
40
|
+
attr_reader :kind, :retryable, :retry_after
|
|
41
|
+
|
|
42
|
+
def initialize(message = nil, kind: nil, retryable: nil, retry_after: nil)
|
|
43
|
+
@kind = kind
|
|
44
|
+
@retryable = retryable
|
|
45
|
+
@retry_after = retry_after
|
|
46
|
+
super(message)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# the additive envelope fields, compacted — nil retry_after stays absent.
|
|
50
|
+
def classification
|
|
51
|
+
{ kind: kind, retryable: retryable, retry_after: retry_after }.compact
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# The hard budget refused the turn (WS2): the (tenant, agent) spend in the
|
|
56
|
+
# window already met its cap BEFORE the turn ran. A typed, retryable failure —
|
|
57
|
+
# the envelope reads `budget_exceeded` + `retry_after` (seconds until the
|
|
58
|
+
# window rolls) — never a silent drop. `window` is :daily | :monthly.
|
|
59
|
+
class BudgetExceeded < Error
|
|
60
|
+
attr_reader :window, :retry_after
|
|
61
|
+
|
|
62
|
+
def initialize(message = nil, window: nil, retry_after: nil)
|
|
63
|
+
@window = window
|
|
64
|
+
@retry_after = retry_after
|
|
65
|
+
super(message || "budget exceeded (#{window})")
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def classification
|
|
69
|
+
{ kind: :budget_exceeded, retryable: true, retry_after: retry_after }.compact
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# The circuit breaker refused the turn WITHOUT touching the provider (WS3):
|
|
74
|
+
# the (tenant, provider/model) saw `after` failures within `within` seconds.
|
|
75
|
+
# Typed + retryable — the envelope reads `circuit_open` + `retry_after`
|
|
76
|
+
# (seconds until the cooldown lets a half-open trial through).
|
|
77
|
+
class CircuitOpenError < Error
|
|
78
|
+
attr_reader :ref, :retry_after
|
|
79
|
+
|
|
80
|
+
def initialize(message = nil, ref: nil, retry_after: nil)
|
|
81
|
+
@ref = ref
|
|
82
|
+
@retry_after = retry_after
|
|
83
|
+
super(message || "circuit open for #{ref}")
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def classification
|
|
87
|
+
{ kind: :circuit_open, retryable: true, retry_after: retry_after }.compact
|
|
88
|
+
end
|
|
89
|
+
end
|
|
34
90
|
class StoreError < Error; end # persistence backend failed -> task :failed
|
|
35
91
|
class CancelledError < Error; end # cooperative cancellation -> task :cancelled
|
|
36
92
|
|
|
@@ -75,7 +131,7 @@ module Insika
|
|
|
75
131
|
end
|
|
76
132
|
end
|
|
77
133
|
|
|
78
|
-
# Subagent graph integrity
|
|
134
|
+
# Subagent graph integrity. Raised at DEFINITION-time
|
|
79
135
|
# (CreateAgent/UpdateAgent/boot) by SubagentGraph.validate! — a subagents
|
|
80
136
|
# allowlist that forms a cycle or exceeds the depth cap is a configuration
|
|
81
137
|
# error, never a runtime surprise. A ValidationError so the authoring Command
|
|
@@ -101,7 +157,7 @@ module Insika
|
|
|
101
157
|
end
|
|
102
158
|
end
|
|
103
159
|
|
|
104
|
-
# Workflow I/O contract violation
|
|
160
|
+
# Workflow I/O contract violation. A workflow may declare an
|
|
105
161
|
# `input_schema` / `output_schema`; a value that does not conform is rejected.
|
|
106
162
|
# INPUT is validated SYNCHRONOUSLY (TriggerWorkflow) so it is a ValidationError
|
|
107
163
|
# -> HTTP 422, no run created. OUTPUT is validated inside the fiber after the
|
|
@@ -119,14 +175,14 @@ module Insika
|
|
|
119
175
|
end
|
|
120
176
|
end
|
|
121
177
|
|
|
122
|
-
# A channel could not hand a reply to its recipient
|
|
178
|
+
# A channel could not hand a reply to its recipient. NOT a turn
|
|
123
179
|
# failure: the turn already completed and its answer is durable in the session —
|
|
124
180
|
# what failed is the delivery, which lives in the OutboxStore with its own status
|
|
125
181
|
# and its own bounded retry. Raised by a channel's `deliver` so the dispatcher can
|
|
126
182
|
# tell "the recipient refused" from "the engine has a bug".
|
|
127
183
|
class DeliveryError < Error; end
|
|
128
184
|
|
|
129
|
-
# Strict configuration violation (
|
|
185
|
+
# Strict configuration violation (— OpenClaw's config discipline:
|
|
130
186
|
# "recusa boot com chave desconhecida, no silent config compat"). Raised by
|
|
131
187
|
# EnvSchema.enforce! at boot ONLY when strictness is on (INSIKA_CONFIG_STRICT) —
|
|
132
188
|
# by default a bad key WARNS and the engine still boots (last-known-good: a rotated
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
3
|
+
# the PII/secret patterns live in the RUNTIME (single source of
|
|
4
4
|
# truth) — the eval consumes them rather than keeping a divergent copy. Since the
|
|
5
5
|
# module moved under `lib/`, `Safety::Detectors` is loaded by `insika.rb` before this
|
|
6
6
|
# file; the explicit climb out of `evals/` that used to be here is gone.
|
|
@@ -33,12 +33,12 @@ module Insika
|
|
|
33
33
|
# evaluated", never a silent pass. A case passes only if the deterministic checks
|
|
34
34
|
# pass AND (there's no judge verdict OR it passed).
|
|
35
35
|
#
|
|
36
|
-
# `skipped` (a reason, nil = it ran) is the THIRD outcome
|
|
36
|
+
# `skipped` (a reason, nil = it ran) is the THIRD outcome: the
|
|
37
37
|
# deployment lacks something the case declared it needs, so there was nothing to
|
|
38
38
|
# assert. It is never a pass and never a failure — a suite of 40 cases where 12
|
|
39
39
|
# are skipped says something true, where 40 cases with 12 failures on capability
|
|
40
40
|
# grounds says nothing and gets ignored.
|
|
41
|
-
# `pairwise` (a Pairwise::Verdict
|
|
41
|
+
# `pairwise` (a Pairwise::Verdict) is attached when the case
|
|
42
42
|
# carries a `reference:` and a panel is configured. It DELIBERATELY does not enter
|
|
43
43
|
# `pass?`: "worse than the incumbent" is a judgement about a replacement decision,
|
|
44
44
|
# not a regression in the suite, and letting it fail a case would put an opinion
|
|
@@ -53,25 +53,25 @@ module Insika
|
|
|
53
53
|
def judge_pending? = !skipped? && !rubric.to_s.strip.empty? && judge.nil?
|
|
54
54
|
end
|
|
55
55
|
|
|
56
|
-
# Deterministic
|
|
56
|
+
# Deterministic evaluation — cheap, zero-token, zero-flakiness. It's the
|
|
57
57
|
# layer that catches the gross regressions (a tool stopped being called, a secret
|
|
58
|
-
# leaked, the turn errored). Subjective scoring is the LLM-judge in
|
|
58
|
+
# leaked, the turn errored). Subjective scoring is the LLM-judge in.
|
|
59
59
|
module Assertions
|
|
60
|
-
# Named negative detectors for `must_not` now live in the runtime
|
|
60
|
+
# Named negative detectors for `must_not` now live in the runtime. Kept as
|
|
61
61
|
# an alias so any external reference to Evals::Assertions::PII_DETECTORS still
|
|
62
62
|
# resolves; the values ARE the runtime's, never a fork.
|
|
63
63
|
PII_DETECTORS = Insika::Safety::Detectors::PII
|
|
64
64
|
|
|
65
|
-
# HOW MUCH THE AGENT SHOULD ASK BEFORE ACTING
|
|
65
|
+
# HOW MUCH THE AGENT SHOULD ASK BEFORE ACTING. Declared per
|
|
66
66
|
# case because it is a per-STORE decision, not a universal rule: sometimes the
|
|
67
67
|
# agent should establish the objective before searching ("energia, treino ou
|
|
68
|
-
# sono?" —
|
|
68
|
+
# sono?" — a good store agent does this well), and sometimes asking again is the
|
|
69
69
|
# failure and it should just search. A global assertion would be wrong half
|
|
70
70
|
# the time; the judge is TOLD the policy (Judge#build_prompt) and this layer
|
|
71
71
|
# checks the half that needs no reader.
|
|
72
72
|
#
|
|
73
|
-
# Each rule is stated as the CUSTOMER-VISIBLE fact it checks.
|
|
74
|
-
# this as "questions before the first tool call", written before
|
|
73
|
+
# Each rule is stated as the CUSTOMER-VISIBLE fact it checks. phrased
|
|
74
|
+
# this as "questions before the first tool call", written before: text a
|
|
75
75
|
# model emits before calling a tool never reaches the customer now (it rides
|
|
76
76
|
# `:intermediate`), and the eval is a client of `/v1/responses`, so what it can
|
|
77
77
|
# observe per turn is the published answer plus the tools that turn called.
|
|
@@ -87,7 +87,7 @@ module Insika
|
|
|
87
87
|
|
|
88
88
|
module_function
|
|
89
89
|
|
|
90
|
-
# WHAT THIS DEPLOYMENT LACKS for the case to be worth running
|
|
90
|
+
# WHAT THIS DEPLOYMENT LACKS for the case to be worth running.
|
|
91
91
|
# -> [reason]; empty = run it.
|
|
92
92
|
#
|
|
93
93
|
# `available` is the deployment's answer for this agent:
|
|
@@ -237,7 +237,7 @@ module Insika
|
|
|
237
237
|
|
|
238
238
|
# Runs a named detector over the text. "pii_leak" = union of all PII detectors;
|
|
239
239
|
# otherwise a single named pattern. Delegates to the runtime's single source
|
|
240
|
-
#
|
|
240
|
+
# which itself fails loud on an unknown name (a typo'd assertion must not
|
|
241
241
|
# silently pass).
|
|
242
242
|
def detect(name, text)
|
|
243
243
|
Insika::Safety::Detectors.detect(name, text)
|
|
@@ -4,7 +4,7 @@ require "json"
|
|
|
4
4
|
|
|
5
5
|
module Insika
|
|
6
6
|
module Evals
|
|
7
|
-
#
|
|
7
|
+
# gating. A baseline is the accepted state of the golden
|
|
8
8
|
# set — `{ cases: { id => { pass, score } } }`. A gated run compares against it and
|
|
9
9
|
# blocks only on a REGRESSION, so known-failing cases don't wedge the gate while a
|
|
10
10
|
# real drop (a passing case that now fails, or a judge score that fell past the
|
|
@@ -16,7 +16,7 @@ module Insika
|
|
|
16
16
|
|
|
17
17
|
# [CaseResult] -> baseline hash. `at` is stamped by the caller (kept out of here
|
|
18
18
|
# so the module stays deterministic/testable).
|
|
19
|
-
# A SKIPPED case is left out entirely
|
|
19
|
+
# A SKIPPED case is left out entirely: writing it as `pass:
|
|
20
20
|
# false` would accept "this deployment cannot run it" as the accepted state,
|
|
21
21
|
# and the case would never block anywhere again.
|
|
22
22
|
def snapshot(results, at:)
|
|
@@ -38,7 +38,7 @@ module Insika
|
|
|
38
38
|
# in BOTH are compared: a new case (no baseline entry) never blocks the gate (it
|
|
39
39
|
# shows in the report as ❌ but is not a "regression"); document this in README.
|
|
40
40
|
# • pass→fail : baseline pass, now failing (hard regression).
|
|
41
|
-
# • pass→skipped: baseline pass, now unrunnable HERE.
|
|
41
|
+
# • pass→skipped: baseline pass, now unrunnable HERE. says the
|
|
42
42
|
# gate never blocks ON a skip, and it does not: a case that was
|
|
43
43
|
# already skipped or unknown stays silent. But a case that used
|
|
44
44
|
# to run on this deployment and no longer can means the agent
|