insika 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +199 -5
- data/README.md +8 -2
- data/bin/insika +231 -13
- data/docs/AGENTS.md +505 -6
- data/docs/API.md +56 -0
- data/docs/CHANNELS.md +100 -10
- data/docs/CONTEXT.md +147 -19
- data/docs/DEPLOY.md +34 -11
- data/docs/EMBEDDING.md +11 -7
- data/docs/EVALS.md +20 -1
- data/docs/FACTS.md +135 -0
- data/docs/HARVEST.md +117 -0
- data/docs/LOADTEST.md +17 -10
- data/docs/OBSERVABILITY.md +65 -2
- data/docs/REFINEMENT.md +9 -9
- data/docs/RELEASING.md +34 -7
- data/docs/RUNNING-LOCAL.md +4 -4
- data/docs/SECURITY.md +85 -11
- data/docs/SKILLS.md +189 -3
- data/docs/SOAK.md +127 -0
- data/docs/TOOLS.md +70 -2
- data/docs/WHY.md +1 -1
- data/docs/WORKFLOWS.md +2 -2
- data/docs/domain.md +115 -0
- data/docs/index.md +2 -2
- data/docs/onboarding/start.md +1 -1
- data/lib/insika/agent_profile.rb +228 -26
- data/lib/insika/alert_dispatcher.rb +139 -0
- data/lib/insika/balloon_splitter.rb +102 -0
- data/lib/insika/baseline_store.rb +2 -2
- data/lib/insika/budget_ledger.rb +166 -0
- data/lib/insika/cache_series_store.rb +49 -0
- data/lib/insika/channel_delivery.rb +132 -24
- data/lib/insika/channel_registry.rb +1 -1
- data/lib/insika/channels/relay.rb +80 -6
- data/lib/insika/channels/web/widget.js +2 -2
- data/lib/insika/channels/web.rb +9 -9
- data/lib/insika/channels/webhook.rb +58 -0
- data/lib/insika/chat_builder.rb +145 -13
- data/lib/insika/checkpoint_store.rb +16 -0
- data/lib/insika/circuit_state.rb +114 -0
- data/lib/insika/coercion.rb +8 -0
- data/lib/insika/commands/agent_payload.rb +6 -4
- data/lib/insika/commands/cancel_followup.rb +49 -0
- data/lib/insika/commands/create_agent.rb +2 -2
- data/lib/insika/commands/create_session.rb +1 -1
- data/lib/insika/commands/delete_llm_provider.rb +1 -1
- data/lib/insika/commands/delete_skill.rb +43 -0
- data/lib/insika/commands/delete_tenant_data.rb +95 -0
- data/lib/insika/commands/export_customer_memory.rb +48 -0
- data/lib/insika/commands/forget_customer.rb +117 -0
- data/lib/insika/commands/freeze_funnel_baseline.rb +113 -0
- data/lib/insika/commands/gate_harvest.rb +138 -0
- data/lib/insika/commands/gate_refinement.rb +12 -12
- data/lib/insika/commands/import_mcp_tools.rb +1 -1
- data/lib/insika/commands/import_tools.rb +4 -4
- data/lib/insika/commands/issue_tenant_token.rb +41 -0
- data/lib/insika/commands/judge_shadow_pairs.rb +124 -0
- data/lib/insika/commands/memory_forget_fact.rb +20 -4
- data/lib/insika/commands/memory_put_fact.rb +23 -4
- data/lib/insika/commands/promote_harvest.rb +130 -0
- data/lib/insika/commands/record_outcome.rb +46 -0
- data/lib/insika/commands/record_shadow_reply.rb +68 -0
- data/lib/insika/commands/reject_harvest.rb +38 -0
- data/lib/insika/commands/resolve_proposal.rb +108 -0
- data/lib/insika/commands/resolve_refinement.rb +1 -1
- data/lib/insika/commands/revoke_contact.rb +49 -0
- data/lib/insika/commands/revoke_token.rb +39 -0
- data/lib/insika/commands/rollback_harvest.rb +86 -0
- data/lib/insika/commands/rotate_tenant_token.rb +43 -0
- data/lib/insika/commands/run_distillation.rb +186 -0
- data/lib/insika/commands/run_harvest.rb +393 -0
- data/lib/insika/commands/run_refinement.rb +5 -5
- data/lib/insika/commands/send_message.rb +112 -15
- data/lib/insika/commands/session_purge.rb +67 -0
- data/lib/insika/commands/set_agent_tools.rb +1 -1
- data/lib/insika/commands/set_skill_agents.rb +60 -19
- data/lib/insika/commands/trigger_workflow.rb +1 -1
- data/lib/insika/commands/update_agent.rb +1 -1
- data/lib/insika/commands/write_data_tool.rb +1 -1
- data/lib/insika/commands/write_golden.rb +1 -1
- data/lib/insika/commands/write_skill.rb +19 -9
- data/lib/insika/config_store.rb +8 -4
- data/lib/insika/contact_store.rb +183 -0
- data/lib/insika/context/builder.rb +23 -5
- data/lib/insika/context/fragment.rb +31 -3
- data/lib/insika/context/priority.rb +6 -2
- data/lib/insika/context/provider.rb +17 -3
- data/lib/insika/context/providers/briefing.rb +96 -0
- data/lib/insika/context/providers/memory.rb +16 -7
- data/lib/insika/context/providers/prompt.rb +30 -2
- data/lib/insika/context/providers/request.rb +1 -1
- data/lib/insika/context/providers/session.rb +17 -2
- data/lib/insika/context/providers/skill.rb +7 -1
- data/lib/insika/context/providers/skill_trigger.rb +128 -0
- data/lib/insika/context/providers/tool_search.rb +2 -0
- data/lib/insika/context_trace_store.rb +128 -0
- data/lib/insika/delegation_store.rb +2 -2
- data/lib/insika/distill.rb +224 -0
- data/lib/insika/distill_engine.rb +169 -0
- data/lib/insika/doctor.rb +962 -7
- data/lib/insika/dsl/runtime.rb +20 -11
- data/lib/insika/dsl/server_boot.rb +74 -4
- data/lib/insika/dsl/system.rb +1 -1
- data/lib/insika/dsl.rb +152 -15
- data/lib/insika/edge_limiter.rb +167 -8
- data/lib/insika/egress_guard.rb +3 -3
- data/lib/insika/env_schema.rb +22 -12
- data/lib/insika/errors.rb +72 -5
- data/lib/insika/evals/assertions.rb +15 -14
- data/lib/insika/evals/baseline.rb +3 -3
- data/lib/insika/evals/golden.rb +8 -8
- data/lib/insika/evals/judge.rb +7 -7
- data/lib/insika/evals/pairwise.rb +21 -9
- data/lib/insika/evals/report.rb +2 -2
- data/lib/insika/evals/runner.rb +6 -6
- data/lib/insika/evals/transport.rb +2 -2
- data/lib/insika/event_stream.rb +23 -5
- data/lib/insika/evidence.rb +183 -0
- data/lib/insika/executor.rb +1092 -160
- data/lib/insika/followup_engine.rb +207 -0
- data/lib/insika/followup_policy.rb +221 -0
- data/lib/insika/followup_store.rb +306 -0
- data/lib/insika/frontmatter.rb +1 -1
- data/lib/insika/funnel_declaration.rb +106 -0
- data/lib/insika/funnel_fold.rb +179 -0
- data/lib/insika/funnel_store.rb +163 -0
- data/lib/insika/golden_store.rb +3 -3
- data/lib/insika/grounding/matcher.rb +69 -0
- data/lib/insika/grounding.rb +44 -0
- data/lib/insika/harvest/conversion_gate.rb +159 -0
- data/lib/insika/harvest/criterion.rb +98 -0
- data/lib/insika/harvest/gate.rb +194 -0
- data/lib/insika/harvest/negative_list.rb +199 -0
- data/lib/insika/harvest.rb +241 -0
- data/lib/insika/harvest_engine.rb +193 -0
- data/lib/insika/harvest_store.rb +548 -0
- data/lib/insika/http_client.rb +3 -3
- data/lib/insika/inbound_log.rb +1 -1
- data/lib/insika/llm_configurator.rb +3 -3
- data/lib/insika/loop_detector.rb +143 -0
- data/lib/insika/mcp_http_client.rb +4 -4
- data/lib/insika/mcp_tool_ingestor.rb +6 -6
- data/lib/insika/media.rb +298 -0
- data/lib/insika/memory_audit_store.rb +85 -0
- data/lib/insika/memory_store.rb +264 -23
- data/lib/insika/message_origin.rb +8 -3
- data/lib/insika/model_resolver.rb +1 -1
- data/lib/insika/model_selection.rb +5 -4
- data/lib/insika/model_visible.rb +87 -0
- data/lib/insika/model_visible_trace_store.rb +66 -0
- data/lib/insika/onboarding.rb +8 -3
- data/lib/insika/outbox_store.rb +44 -6
- data/lib/insika/outcome_store.rb +147 -0
- data/lib/insika/overlay_tool_registry.rb +3 -4
- data/lib/insika/pack.rb +3 -3
- data/lib/insika/pack_importer.rb +17 -15
- data/lib/insika/packaging.rb +163 -0
- data/lib/insika/parity/criterion.rb +79 -0
- data/lib/insika/parity/verdict.rb +318 -0
- data/lib/insika/pending_action_store.rb +1 -1
- data/lib/insika/plugin/loader.rb +2 -2
- data/lib/insika/policy/policy.rb +1 -1
- data/lib/insika/prefix_fingerprint.rb +58 -0
- data/lib/insika/profile_source.rb +34 -7
- data/lib/insika/proposal_store.rb +271 -0
- data/lib/insika/provider_error_classifier.rb +160 -0
- data/lib/insika/queue_policy.rb +6 -3
- data/lib/insika/recovery.rb +47 -6
- data/lib/insika/refinement/candidate.rb +4 -4
- data/lib/insika/refinement/evidence_collector.rb +6 -6
- data/lib/insika/refinement/gate.rb +7 -7
- data/lib/insika/refinement/panel.rb +7 -7
- data/lib/insika/refinement/proposer.rb +10 -10
- data/lib/insika/refinement_store.rb +12 -12
- data/lib/insika/reliability.rb +211 -0
- data/lib/insika/retention.rb +281 -0
- data/lib/insika/routing.rb +101 -0
- data/lib/insika/safety/config.rb +46 -6
- data/lib/insika/safety/corpus.rb +255 -0
- data/lib/insika/safety/detectors.rb +34 -115
- data/lib/insika/safety/factory.rb +18 -5
- data/lib/insika/safety/grounding_enforcer.rb +59 -0
- data/lib/insika/safety/grounding_validator.rb +49 -0
- data/lib/insika/safety/input_guardrail.rb +20 -5
- data/lib/insika/safety/moderator.rb +19 -11
- data/lib/insika/safety/output_filter.rb +10 -6
- data/lib/insika/safety/output_validator.rb +13 -7
- data/lib/insika/safety/safe_responses.rb +1 -1
- data/lib/insika/sandbox/boundary.rb +2 -2
- data/lib/insika/sandbox.rb +1 -1
- data/lib/insika/schema_guard.rb +35 -0
- data/lib/insika/server/app.rb +366 -54
- data/lib/insika/server/boot.rb +4 -4
- data/lib/insika/server/rack_app.rb +31 -7
- data/lib/insika/server/responses.rb +58 -9
- data/lib/insika/server/tenant_auth.rb +61 -0
- data/lib/insika/session_actor.rb +11 -7
- data/lib/insika/session_store.rb +66 -3
- data/lib/insika/settings_store.rb +15 -5
- data/lib/insika/shadow_pair_store.rb +258 -0
- data/lib/insika/shutdown.rb +4 -4
- data/lib/insika/skill_catalog.rb +131 -20
- data/lib/insika/skill_store.rb +70 -22
- data/lib/insika/soak/envelope.rb +140 -0
- data/lib/insika/soak/report.rb +392 -0
- data/lib/insika/soak/runner.rb +554 -0
- data/lib/insika/steer_injector.rb +1 -1
- data/lib/insika/store.rb +11 -2
- data/lib/insika/stores/memory.rb +6 -0
- data/lib/insika/stores/sqlite.rb +8 -0
- data/lib/insika/studio/app.rb +1058 -75
- data/lib/insika/studio/assets/dist/application.css +1 -1
- data/lib/insika/studio/assets/dist/application.js +27 -26
- data/lib/insika/studio/assets/dist/favicon.svg +6 -0
- data/lib/insika/studio/forms.rb +274 -22
- data/lib/insika/studio/nav_icons.rb +7 -2
- data/lib/insika/studio/views/_message.erb +2 -2
- data/lib/insika/studio/views/agent_detail.erb +629 -86
- data/lib/insika/studio/views/agents.erb +11 -7
- data/lib/insika/studio/views/approvals.erb +4 -1
- data/lib/insika/studio/views/chats.erb +4 -1
- data/lib/insika/studio/views/customer.erb +94 -0
- data/lib/insika/studio/views/customers.erb +32 -0
- data/lib/insika/studio/views/evals.erb +4 -1
- data/lib/insika/studio/views/facts.erb +133 -0
- data/lib/insika/studio/views/followups.erb +125 -0
- data/lib/insika/studio/views/funnel.erb +106 -0
- data/lib/insika/studio/views/harvest.erb +234 -0
- data/lib/insika/studio/views/home.erb +2 -1
- data/lib/insika/studio/views/layout.erb +1 -0
- data/lib/insika/studio/views/parity.erb +147 -0
- data/lib/insika/studio/views/playground.erb +7 -1
- data/lib/insika/studio/views/refinement.erb +4 -4
- data/lib/insika/studio/views/session.erb +133 -3
- data/lib/insika/studio/views/settings.erb +9 -12
- data/lib/insika/studio/views/skills.erb +66 -12
- data/lib/insika/studio/views/system_files.erb +1 -1
- data/lib/insika/studio/views/task.erb +13 -0
- data/lib/insika/studio/views/tasks.erb +4 -1
- data/lib/insika/studio/views/tools.erb +0 -1
- data/lib/insika/subagent_graph.rb +3 -3
- data/lib/insika/task_actor.rb +3 -3
- data/lib/insika/task_store.rb +22 -2
- data/lib/insika/telemetry/pricing.rb +3 -3
- data/lib/insika/telemetry/recorder.rb +1 -1
- data/lib/insika/telemetry.rb +2 -2
- data/lib/insika/testing/store_contract.rb +54 -33
- data/lib/insika/tick.rb +146 -0
- data/lib/insika/token_store.rb +168 -0
- data/lib/insika/tool_assembly.rb +5 -5
- data/lib/insika/tool_definition.rb +25 -15
- data/lib/insika/tool_envelope.rb +70 -1
- data/lib/insika/tool_manifest.rb +11 -7
- data/lib/insika/tool_output_compressor.rb +100 -0
- data/lib/insika/tool_store.rb +1 -1
- data/lib/insika/tool_trace_store.rb +1 -1
- data/lib/insika/tools/concurrency.rb +2 -2
- data/lib/insika/tools/data_defined_tool.rb +14 -5
- data/lib/insika/tools/generate_image.rb +44 -0
- data/lib/insika/tools/load_skill.rb +61 -3
- data/lib/insika/tools/schedule_followup.rb +164 -0
- data/lib/insika/tools/stuck_signal.rb +44 -0
- data/lib/insika/tools/subagent.rb +4 -4
- data/lib/insika/tools/subagents.rb +1 -1
- data/lib/insika/tools/tts.rb +47 -0
- data/lib/insika/tools/update_briefing.rb +126 -0
- data/lib/insika/turn_output.rb +2 -2
- data/lib/insika/turn_state.rb +54 -13
- data/lib/insika/turn_timing.rb +24 -4
- data/lib/insika/usage_ledger.rb +1 -1
- data/lib/insika/version.rb +1 -1
- data/lib/insika/vitals.rb +84 -0
- data/lib/insika/wiring/graph.rb +372 -34
- data/lib/insika/workflow.rb +1 -1
- data/lib/insika/workflow_registry.rb +1 -1
- data/lib/insika.rb +122 -16
- metadata +95 -2
- data/lib/insika/server/admin_auth.rb +0 -29
data/lib/insika/edge_limiter.rb
CHANGED
|
@@ -4,7 +4,7 @@ require_relative "middleware"
|
|
|
4
4
|
require_relative "coercion"
|
|
5
5
|
|
|
6
6
|
module Insika
|
|
7
|
-
# The production edge
|
|
7
|
+
# The production edge: THE named place where volume/cost
|
|
8
8
|
# abuse is cut. A Middleware with two independent limits, both OPT-IN
|
|
9
9
|
# (nil/0 = off — a bare wiring behaves exactly as before):
|
|
10
10
|
#
|
|
@@ -13,16 +13,29 @@ module Insika
|
|
|
13
13
|
# · agent token ceiling — total tokens per agent per window. Checked on entry
|
|
14
14
|
# against the accumulated ledger; the turn's own usage is recorded AFTER the
|
|
15
15
|
# terminal returns (the Middleware wraps stages 5-9, so state.usage is set).
|
|
16
|
+
# · calendar budget — WS2: `AgentProfile#budget` caps the spend per
|
|
17
|
+
# (tenant, agent) over CALENDAR windows (daily/monthly), on the
|
|
18
|
+
# BudgetLedger. Hard (default): crossing the cap raises the typed
|
|
19
|
+
# Insika::BudgetExceeded (the envelope quotes budget_exceeded +
|
|
20
|
+
# retry_after); soft: crossing warns instead — ONE budget_warning event
|
|
21
|
+
# per window plus a note injected into the context. Crossing `alert_at`
|
|
22
|
+
# (default 0.8 of the cap) warns the same way, before the wall. The turn's
|
|
23
|
+
# billed spend (input+output+cached+cache_creation — the A4 rule) lands on
|
|
24
|
+
# the windows after the terminal.
|
|
16
25
|
#
|
|
17
26
|
# Config resolution, per turn (configuration over convention):
|
|
18
27
|
# profile.limits[:chat_rate_limit / :agent_token_ceiling] — per-agent override
|
|
19
28
|
# settings["edge"] — platform default
|
|
20
29
|
# A per-agent 0 explicitly disables a platform default for that agent.
|
|
21
30
|
#
|
|
22
|
-
# On breach it uses the graceful-halt contract
|
|
31
|
+
# On breach it uses the graceful-halt contract: halt_response
|
|
23
32
|
# (the safe reply) + guardrail_block (audit -> :guardrail_blocked) and does NOT
|
|
24
33
|
# call `nxt` — the turn completes with ZERO LLM calls. It sits BEFORE the
|
|
25
34
|
# InputGuardrail in the stack so a flood can't spend the LLM moderator either.
|
|
35
|
+
# The BUDGET breach is the ONE deliberate exception: it is a typed failure
|
|
36
|
+
# (BudgetExceeded), not a customer-facing reply — the operator wants the
|
|
37
|
+
# envelope to say "budget" and quote when the window rolls, not to hand the
|
|
38
|
+
# customer a cost message.
|
|
26
39
|
class EdgeLimiter < Middleware
|
|
27
40
|
CHAT_KIND = "chat"
|
|
28
41
|
TOKENS_KIND = "tokens"
|
|
@@ -35,9 +48,13 @@ module Insika
|
|
|
35
48
|
DEFAULT_RESPONSE = "Estou recebendo muitas mensagens agora. Aguarde um " \
|
|
36
49
|
"momento e tente novamente, por favor."
|
|
37
50
|
|
|
38
|
-
def initialize(ledger:, settings_store: nil)
|
|
51
|
+
def initialize(ledger:, settings_store: nil, budget_ledger: nil, event_stream: nil)
|
|
39
52
|
@ledger = ledger
|
|
40
53
|
@settings = settings_store
|
|
54
|
+
# WS2: the calendar-window ledger. nil = budget off (parity — the bare
|
|
55
|
+
# wiring is byte-identical to before).
|
|
56
|
+
@budget_ledger = budget_ledger
|
|
57
|
+
@event_stream = event_stream
|
|
41
58
|
end
|
|
42
59
|
|
|
43
60
|
def call(state, &nxt)
|
|
@@ -48,8 +65,14 @@ module Insika
|
|
|
48
65
|
# the rate-limit reply exactly when the window is saturated. Entry checks
|
|
49
66
|
# are skipped; the turn's usage still lands on the ledger below.
|
|
50
67
|
resumed = state.resumed
|
|
68
|
+
# a SCHEDULED turn (the FollowupEngine's kick) skips the
|
|
69
|
+
# ENTRY checks exactly like a resume — a follow-up that trips the token
|
|
70
|
+
# ceiling must not receive the rate-limit REPLY (the customer agreed to
|
|
71
|
+
# this message; the volume control is the follow-up policy, not the flood
|
|
72
|
+
# rail). The turn still runs and its usage still lands on the ledger.
|
|
73
|
+
scheduled = scheduled_turn?(state)
|
|
51
74
|
|
|
52
|
-
if !resumed && (limit = positive(limits.key?(:chat_rate_limit) ? limits[:chat_rate_limit] : edge["chat_rate_limit"]))
|
|
75
|
+
if !resumed && !scheduled && (limit = positive(limits.key?(:chat_rate_limit) ? limits[:chat_rate_limit] : edge["chat_rate_limit"]))
|
|
53
76
|
breach = check_chat_rate(state, limit, edge)
|
|
54
77
|
return block(state, edge, **breach) if breach
|
|
55
78
|
end
|
|
@@ -58,7 +81,7 @@ module Insika
|
|
|
58
81
|
# `"chat_rate_limit": null`) reads as OFF for that agent, not "inherit".
|
|
59
82
|
if (ceiling = positive(limits.key?(:agent_token_ceiling) ? limits[:agent_token_ceiling] : edge["agent_token_ceiling"]))
|
|
60
83
|
token_window = positive(edge["agent_token_window"]) || DEFAULT_TOKEN_WINDOW
|
|
61
|
-
unless resumed
|
|
84
|
+
unless resumed || scheduled
|
|
62
85
|
spent = @ledger.count(TOKENS_KIND, state.profile.id.to_s, window: token_window)
|
|
63
86
|
if spent >= ceiling
|
|
64
87
|
return block(state, edge, category: :token_ceiling,
|
|
@@ -69,13 +92,40 @@ module Insika
|
|
|
69
92
|
record_after = token_window
|
|
70
93
|
end
|
|
71
94
|
|
|
72
|
-
|
|
73
|
-
|
|
95
|
+
# WS2: calendar budgets. Entry — a HARD budget at/over the cap raises the
|
|
96
|
+
# typed error (never a customer-facing reply); the alert_at warning and
|
|
97
|
+
# the SOFT over-cap both warn once per window + inject a context note.
|
|
98
|
+
# A resumed turn (crash/pause replay) was already admitted: it is never
|
|
99
|
+
# refused twice — its spend still lands on the ledger below. A scheduled
|
|
100
|
+
# turn rides the same rule: the follow-up policy is the
|
|
101
|
+
# volume control, not the budget wall.
|
|
102
|
+
budget_on = budget_configured?(state)
|
|
103
|
+
budget_enforce(state) unless resumed || scheduled
|
|
104
|
+
|
|
105
|
+
result = begin
|
|
106
|
+
nxt.call(state)
|
|
107
|
+
ensure
|
|
108
|
+
# A turn that FAILED after burning tokens still SPENT them: record the
|
|
109
|
+
# usage the state captured before the error propagates. The ask's usage
|
|
110
|
+
# lands on state.usage before any later stage (guardrail block, tool
|
|
111
|
+
# error, workflow schema) can fail the turn — a failed turn must count
|
|
112
|
+
# against the budget like a completed one (WS2).
|
|
113
|
+
record_usage(state, record_after) if record_after
|
|
114
|
+
record_budget_usage(state) if budget_on
|
|
115
|
+
end
|
|
74
116
|
result
|
|
75
117
|
end
|
|
76
118
|
|
|
77
119
|
private
|
|
78
120
|
|
|
121
|
+
# is this turn the FollowupEngine's synthetic kick? The
|
|
122
|
+
# command type is stamped by the engine only — a consumer cannot send it
|
|
123
|
+
# (the SendMessage edge refuses the spelling, C8).
|
|
124
|
+
def scheduled_turn?(state)
|
|
125
|
+
command = state.respond_to?(:task) && state.task&.command
|
|
126
|
+
command.is_a?(Hash) && command["type"].to_s == "scheduled_followup"
|
|
127
|
+
end
|
|
128
|
+
|
|
79
129
|
# One KV get per turn (same order of cost as the guardrail's config read);
|
|
80
130
|
# no SettingsStore in the wiring -> per-agent limits only.
|
|
81
131
|
def platform_edge
|
|
@@ -101,7 +151,7 @@ module Insika
|
|
|
101
151
|
# prefix (`Executor#usage_of` reports `cached_tokens`/`cache_creation_tokens`
|
|
102
152
|
# alongside it) — on a cached identity that prefix is ~95% of what the
|
|
103
153
|
# provider actually processed, so a ceiling reading only `total_tokens` is
|
|
104
|
-
# blind
|
|
154
|
+
# blind. Same billed-spend rule as `Evals::Runner#billed_tokens`.
|
|
105
155
|
# nil usage (workflow turn / provider without counts) records nothing.
|
|
106
156
|
def record_usage(state, window)
|
|
107
157
|
usage = state.usage || {}
|
|
@@ -126,5 +176,114 @@ module Insika
|
|
|
126
176
|
v = value.to_i
|
|
127
177
|
v.positive? ? v : nil
|
|
128
178
|
end
|
|
179
|
+
|
|
180
|
+
# --- WS2 calendar budgets ------------------------------------------
|
|
181
|
+
|
|
182
|
+
# -> truthy when a budget is configured AND the ledger is wired.
|
|
183
|
+
def budget_configured?(state)
|
|
184
|
+
budget = state.profile.respond_to?(:budget) ? state.profile.budget : nil
|
|
185
|
+
!budget.nil? && !@budget_ledger.nil?
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
# -> truthy (the budget hash) when budget checks ran. Raises BudgetExceeded
|
|
189
|
+
# on a HARD cap breach.
|
|
190
|
+
def budget_enforce(state, now: Time.now)
|
|
191
|
+
budget = state.profile.respond_to?(:budget) ? state.profile.budget : nil
|
|
192
|
+
return nil if budget.nil? || @budget_ledger.nil?
|
|
193
|
+
|
|
194
|
+
tenant = budget_tenant(state)
|
|
195
|
+
agent = state.profile.id.to_s
|
|
196
|
+
budget_windows(budget).each do |w|
|
|
197
|
+
spent = @budget_ledger.current(tenant: tenant, agent: agent, now: now)[w[:window]]
|
|
198
|
+
if spent >= w[:cap]
|
|
199
|
+
unless w[:soft]
|
|
200
|
+
raise Insika::BudgetExceeded.new(
|
|
201
|
+
window: w[:window],
|
|
202
|
+
retry_after: @budget_ledger.reset_in(w[:window], now: now)
|
|
203
|
+
)
|
|
204
|
+
end
|
|
205
|
+
warn_budget(state, tenant, agent, w, spent, now, level: "cap")
|
|
206
|
+
elsif spent >= w[:alert_at]
|
|
207
|
+
warn_budget(state, tenant, agent, w, spent, now, level: "alert_at")
|
|
208
|
+
end
|
|
209
|
+
end
|
|
210
|
+
budget
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
# The (tenant, agent) scope: the COMMAND's tenant (nil -> the BudgetLedger's
|
|
214
|
+
# "platform" cell) — never state.tenant, which falls back to the session id
|
|
215
|
+
# (a per-chat bucket is not a budget).
|
|
216
|
+
def budget_tenant(state)
|
|
217
|
+
command = state.respond_to?(:task) && state.task&.command
|
|
218
|
+
return nil unless command.is_a?(Hash)
|
|
219
|
+
|
|
220
|
+
meta = command["meta"] || command[:meta] || {}
|
|
221
|
+
meta["tenant"] || meta[:tenant]
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
# -> [{ window:, cap:, soft:, alert_at: }] — one entry per configured window
|
|
225
|
+
# (a 0/absent cap is off). absent `soft` = FALSE (hard): a limit that does
|
|
226
|
+
# not limit is decoration; the alert_at warning is the soft half.
|
|
227
|
+
def budget_windows(budget)
|
|
228
|
+
alert_at = budget["alert_at"].to_f
|
|
229
|
+
alert_at = 0.8 if alert_at <= 0 || alert_at >= 1
|
|
230
|
+
soft = budget["soft"] == true
|
|
231
|
+
%i[daily monthly].filter_map do |window|
|
|
232
|
+
cap = budget[window.to_s].to_i
|
|
233
|
+
cap.positive? ? { window: window, cap: cap, soft: soft,
|
|
234
|
+
alert_at: (cap * alert_at).floor } : nil
|
|
235
|
+
end
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
# The warning: a note in the context (the model sees it, the customer's
|
|
239
|
+
# transcript does not) + the budget_warning event — each LEVEL once per
|
|
240
|
+
# (window) cell: the `alert_at` crossing and the real soft-cap crossing are
|
|
241
|
+
# separate markers, so the cap event is never swallowed by the 80% one that
|
|
242
|
+
# fired earlier (WS2).
|
|
243
|
+
def warn_budget(state, tenant, agent, w, spent, now, level:)
|
|
244
|
+
inject_budget_note(state,
|
|
245
|
+
"[budget: agent '#{agent}' is at #{spent}/#{w[:cap]} tokens this " \
|
|
246
|
+
"#{w[:window]} window — keep this turn cheap]")
|
|
247
|
+
return if @budget_ledger.mark_alert(tenant: tenant, agent: agent, window: w[:window],
|
|
248
|
+
level: level, now: now)
|
|
249
|
+
|
|
250
|
+
# `tenant` on the META too, not only in the payload: the tenant-scoped
|
|
251
|
+
# /v1/events subscription filters on meta[:tenant] and is fail-closed, so
|
|
252
|
+
# a warning about the tenant's OWN budget never reached the tenant.
|
|
253
|
+
meta = { task_id: state.task&.id, session_id: state.task&.session_id,
|
|
254
|
+
at: Time.now.utc.iso8601 }
|
|
255
|
+
meta[:tenant] = tenant unless tenant.nil?
|
|
256
|
+
@event_stream&.emit(Insika::Event.new(
|
|
257
|
+
type: :budget_warning,
|
|
258
|
+
data: { agent: agent, tenant: tenant, window: w[:window],
|
|
259
|
+
spent: spent, cap: w[:cap], level: level },
|
|
260
|
+
meta: meta
|
|
261
|
+
))
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
# Appends the note to the assembled system prompt: the real Data package is
|
|
265
|
+
# immutable (with), the specs' minimal Struct is mutable — both duck-typed.
|
|
266
|
+
def inject_budget_note(state, note)
|
|
267
|
+
ctx = state.context
|
|
268
|
+
return if ctx.nil?
|
|
269
|
+
|
|
270
|
+
if ctx.respond_to?(:with)
|
|
271
|
+
state.context = ctx.with(system: "#{ctx.system}\n\n#{note}")
|
|
272
|
+
elsif ctx.respond_to?(:system=)
|
|
273
|
+
ctx.system = "#{ctx.system}\n\n#{note}"
|
|
274
|
+
end
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
# The turn's REAL billed spend (input + output + cached + cache_creation —
|
|
278
|
+
# the A4 rule) on the calendar windows.
|
|
279
|
+
def record_budget_usage(state, now: Time.now)
|
|
280
|
+
usage = state.usage || {}
|
|
281
|
+
tokens = usage[:total_tokens].to_i + usage[:cached_tokens].to_i +
|
|
282
|
+
usage[:cache_creation_tokens].to_i
|
|
283
|
+
return if tokens.zero?
|
|
284
|
+
|
|
285
|
+
@budget_ledger.add(tenant: budget_tenant(state), agent: state.profile.id.to_s,
|
|
286
|
+
by: tokens, now: now)
|
|
287
|
+
end
|
|
129
288
|
end
|
|
130
289
|
end
|
data/lib/insika/egress_guard.rb
CHANGED
|
@@ -8,14 +8,14 @@ module Insika
|
|
|
8
8
|
# EGRESS guard for data-tools (SSRF). A data-tool makes a server-side HTTP
|
|
9
9
|
# request with a URL coming from UI-editable config — without a guard, it's an
|
|
10
10
|
# SSRF vector (hitting cloud metadata, internal services, localhost). Rules
|
|
11
|
-
# (spec
|
|
11
|
+
# (spec):
|
|
12
12
|
# - https only by default (http requires explicit opt-in);
|
|
13
13
|
# - host required;
|
|
14
14
|
# - optional host allowlist (when present, only it passes);
|
|
15
15
|
# - resolves the host and BLOCKS if ANY address falls into a private/
|
|
16
16
|
# loopback/link-local/metadata network (defense against DNS rebinding);
|
|
17
17
|
# - `allow_private:` (opt-in) ALLOWS the private target — to reach a trusted
|
|
18
|
-
# INTERNAL API (
|
|
18
|
+
# INTERNAL API (the consumer's /api/internal/* comes in via an
|
|
19
19
|
# allowlist). Dangerous without `host_allowlist`: PIN it to a known host.
|
|
20
20
|
# Default false = strict guard.
|
|
21
21
|
#
|
|
@@ -48,7 +48,7 @@ module Insika
|
|
|
48
48
|
addrs = resolve(host)
|
|
49
49
|
return "host did not resolve" if addrs.empty?
|
|
50
50
|
# allow_private skips the private-network block (trusted internal API,
|
|
51
|
-
#
|
|
51
|
+
# Without it, a private/loopback/metadata target is always blocked.
|
|
52
52
|
return "private-network destination blocked" if !allow_private && addrs.any? { |ip| blocked?(ip) }
|
|
53
53
|
|
|
54
54
|
nil
|
data/lib/insika/env_schema.rb
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Insika
|
|
4
|
-
# STRICT config, environment layer
|
|
4
|
+
# STRICT config, environment layer. OpenClaw's config discipline
|
|
5
5
|
# — "recusa boot com chave desconhecida, no silent config compat" — applied to the
|
|
6
6
|
# env vars the engine reads at boot. A declarative registry of the keys the engine
|
|
7
7
|
# OWNS (config over convention: the schema IS data), used two ways:
|
|
@@ -12,7 +12,7 @@ module Insika
|
|
|
12
12
|
# silence), and a DEPRECATED legacy key still set under the old HARNESS_ prefix.
|
|
13
13
|
# Unknown-key detection is scoped to the OWNED prefixes only, so the platform's
|
|
14
14
|
# own vars (Railway's RAILWAY_*, PORT, PATH, the litestream sidecar's
|
|
15
|
-
# LITESTREAM_*, the deployment's DEEPSEEK_*/
|
|
15
|
+
# LITESTREAM_*, the deployment's DEEPSEEK_*/CONSUMER_*) are never flagged.
|
|
16
16
|
# · `enforce!(strict:)` — the boot gate. WARNS on every finding by default and
|
|
17
17
|
# lets the engine come up (last-known-good — a rotated key or a typo must never
|
|
18
18
|
# take the whole service down, same reasoning as the resilient DEEPSEEK boot);
|
|
@@ -92,7 +92,7 @@ module Insika
|
|
|
92
92
|
Spec.new(name: name, type: type, secret: secret, required: required, enum: enum, description: description)
|
|
93
93
|
end
|
|
94
94
|
|
|
95
|
-
# The engine's own keys. Deployment/app keys (DEEPSEEK_*,
|
|
95
|
+
# The engine's own keys. Deployment/app keys (DEEPSEEK_*, CONSUMER_*, …) are NOT
|
|
96
96
|
# here — a root passes them as `extra:`.
|
|
97
97
|
DEFAULT = [
|
|
98
98
|
spec(name: "INSIKA_DB", type: :path, description: "SQLite path; durable config+state. Unset -> ephemeral memory."),
|
|
@@ -102,22 +102,32 @@ module Insika
|
|
|
102
102
|
spec(name: "INSIKA_ENV", description: "Environment name shown in the Studio (falls back to RACK_ENV)."),
|
|
103
103
|
spec(name: "INSIKA_A2A_AGENT", description: "Agent id to expose over inbound A2A (opt-in)."),
|
|
104
104
|
spec(name: "INSIKA_A2A_REMOTES", type: :csv, description: "Comma-separated remote A2A endpoints."),
|
|
105
|
-
spec(name: "INSIKA_EGRESS_ALLOW_HTTP", type: :boolean, description: "Allow plain http egress from data-tools (default: https only)."),
|
|
105
|
+
spec(name: "INSIKA_EGRESS_ALLOW_HTTP", type: :boolean, description: "Allow plain http egress from data-tools, channel callbacks and media fetches (default: https only)."),
|
|
106
106
|
spec(name: "INSIKA_EGRESS_ALLOW_PRIVATE", type: :boolean, description: "Allow egress to private/loopback ranges (SSRF guard off)."),
|
|
107
|
-
spec(name: "INSIKA_EGRESS_HOSTS", type: :csv, description: "Comma-separated host allowlist for data-tool egress."),
|
|
107
|
+
spec(name: "INSIKA_EGRESS_HOSTS", type: :csv, description: "Comma-separated host allowlist for data-tool egress (media fetches are NOT pinned by it)."),
|
|
108
108
|
spec(name: "INSIKA_OTEL", type: :boolean, description: "Turn on OpenTelemetry export (opt-in)."),
|
|
109
109
|
spec(name: "INSIKA_MODEL_PRICING", description: "JSON rates table (USD per million tokens) for the estimated-cost attribute; unset -> no cost reported."),
|
|
110
|
-
spec(name: "INSIKA_TURN_TIMING", type: :boolean, description: "Emit per-turn TTFB breakdown in responses (opt-in
|
|
110
|
+
spec(name: "INSIKA_TURN_TIMING", type: :boolean, description: "Emit per-turn TTFB breakdown in responses (opt-in)."),
|
|
111
111
|
spec(name: "INSIKA_SUBAGENT_DEPTH_CAP", type: :integer, description: "Max delegation depth in the subagent graph (default 5)."),
|
|
112
112
|
spec(name: "INSIKA_SUBAGENT_FANOUT_CAP", type: :integer, description: "Max parallel children in spawn_subagents (default 8)."),
|
|
113
|
-
spec(name: "INSIKA_CONFIG_STRICT", type: :boolean, description: "Refuse boot on any config finding instead of warning
|
|
114
|
-
spec(name: "INSIKA_BOOT_ID", description: "Boot generation id shared by all workers of one container start; the recovery task sweep runs once per id
|
|
115
|
-
spec(name: "INSIKA_DRAIN_TIMEOUT", type: :integer, description: "Seconds a stopping worker waits for in-flight turns before abandoning them to the next boot's recovery (default 20
|
|
116
|
-
spec(name: "
|
|
117
|
-
spec(name: "
|
|
113
|
+
spec(name: "INSIKA_CONFIG_STRICT", type: :boolean, description: "Refuse boot on any config finding instead of warning."),
|
|
114
|
+
spec(name: "INSIKA_BOOT_ID", description: "Boot generation id shared by all workers of one container start; the recovery task sweep runs once per id. Unset -> every boot sweeps."),
|
|
115
|
+
spec(name: "INSIKA_DRAIN_TIMEOUT", type: :integer, description: "Seconds a stopping worker waits for in-flight turns before abandoning them to the next boot's recovery (default 20)."),
|
|
116
|
+
spec(name: "INSIKA_TICK_INTERVAL", type: :integer, description: "Seconds between tick passes (outbox drain + stale recovery sweep). Default 60; 0 disables."),
|
|
117
|
+
spec(name: "INSIKA_TICK_STALE_AFTER", type: :integer, description: "Seconds a :queued/:running task must sit untouched before the tick sweeps it (default 900). Must exceed the largest turn_timeout of the deployment."),
|
|
118
|
+
spec(name: "INSIKA_STT_MODEL", description: "Model used to transcribe audio message parts (WS9). Unset -> RubyLLM's default transcription model."),
|
|
119
|
+
spec(name: "INSIKA_STT_LANGUAGE", description: "Language hint for the transcription of audio message parts (WS9)."),
|
|
120
|
+
spec(name: "INSIKA_TENANCY", enum: %w[single_tenant multi_tenant], description: "single_tenant (default: one operator credential) or multi_tenant (per-tenant + operator tokens resolved from the store)."),
|
|
121
|
+
spec(name: "INSIKA_ONBOARDING", type: :boolean, description: "Expose the public onboarding surface (/start.md, /models.json, /docs) in production (opt-in)."),
|
|
122
|
+
spec(name: "INSIKA_RELAY_TOKEN", secret: true, description: "Bearer the relay consumer sends us. Unset -> the relay channel is not mounted."),
|
|
118
123
|
spec(name: "INSIKA_RELAY_DELIVER_URL", type: :url, description: "Consumer callback the relay POSTs each reply to."),
|
|
119
124
|
spec(name: "INSIKA_RELAY_DELIVER_TOKEN", secret: true, description: "Bearer the relay sends TO the consumer's callback (optional)."),
|
|
120
|
-
spec(name: "
|
|
125
|
+
spec(name: "INSIKA_RELAY_SHADOW", type: :boolean, description: "Shadow mode: the relay records replies instead of delivering them."),
|
|
126
|
+
spec(name: "INSIKA_RELAY_DELIVERY", type: :enum, enum: %w[at_end progressive], description: "How the relay flushes the outbox: at_end (one POST) or progressive (one POST per balloon). Unset -> at_end."),
|
|
127
|
+
spec(name: "INSIKA_PARITY_CRITERION", type: :path, description: "The frozen parity criterion file (required in shadow mode)."),
|
|
128
|
+
spec(name: "INSIKA_HARVEST_CRITERION", type: :path, description: "The frozen harvest conversion criterion file (strict-loaded before any promotion)."),
|
|
129
|
+
spec(name: "INSIKA_HARVEST_NEGATIVE", type: :path, description: "The negative-list seed file the harvest CLI imports into agent profiles."),
|
|
130
|
+
spec(name: "INSIKA_WIDGET_ORIGINS", type: :csv, description: "Exact-match origins allowed to embed the web widget. Unset -> the widget channel is not mounted."),
|
|
121
131
|
spec(name: "INSIKA_WIDGET_AGENTS", type: :csv, description: "Agent ids a widget visitor may address. Unset -> the widget channel is not mounted."),
|
|
122
132
|
spec(name: "OPENCLAW_GATEWAY_TOKEN", secret: true, description: "Bearer for /v1 + /a2a (falls back to ADMIN_TOKEN)."),
|
|
123
133
|
spec(name: "OPENCLAW_AGENTS_DIR", type: :path, description: "Directory of OpenClaw-style agent packs."),
|
data/lib/insika/errors.rb
CHANGED
|
@@ -30,7 +30,63 @@ module Insika
|
|
|
30
30
|
end
|
|
31
31
|
end
|
|
32
32
|
|
|
33
|
-
|
|
33
|
+
# A provider/transport failure, wrapped by ProviderErrorClassifier with an
|
|
34
|
+
# ACTION classification (B9). The fields ride in the task's error record and
|
|
35
|
+
# the :task_failed event so the client envelope can tell fatal from
|
|
36
|
+
# retryable and quote the provider's own retry_after (A8). A bare
|
|
37
|
+
# `ProviderError.new("boom")` has an empty classification and adds nothing
|
|
38
|
+
# to the contract.
|
|
39
|
+
class ProviderError < Error
|
|
40
|
+
attr_reader :kind, :retryable, :retry_after
|
|
41
|
+
|
|
42
|
+
def initialize(message = nil, kind: nil, retryable: nil, retry_after: nil)
|
|
43
|
+
@kind = kind
|
|
44
|
+
@retryable = retryable
|
|
45
|
+
@retry_after = retry_after
|
|
46
|
+
super(message)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# the additive envelope fields, compacted — nil retry_after stays absent.
|
|
50
|
+
def classification
|
|
51
|
+
{ kind: kind, retryable: retryable, retry_after: retry_after }.compact
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# The hard budget refused the turn (WS2): the (tenant, agent) spend in the
|
|
56
|
+
# window already met its cap BEFORE the turn ran. A typed, retryable failure —
|
|
57
|
+
# the envelope reads `budget_exceeded` + `retry_after` (seconds until the
|
|
58
|
+
# window rolls) — never a silent drop. `window` is :daily | :monthly.
|
|
59
|
+
class BudgetExceeded < Error
|
|
60
|
+
attr_reader :window, :retry_after
|
|
61
|
+
|
|
62
|
+
def initialize(message = nil, window: nil, retry_after: nil)
|
|
63
|
+
@window = window
|
|
64
|
+
@retry_after = retry_after
|
|
65
|
+
super(message || "budget exceeded (#{window})")
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def classification
|
|
69
|
+
{ kind: :budget_exceeded, retryable: true, retry_after: retry_after }.compact
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# The circuit breaker refused the turn WITHOUT touching the provider (WS3):
|
|
74
|
+
# the (tenant, provider/model) saw `after` failures within `within` seconds.
|
|
75
|
+
# Typed + retryable — the envelope reads `circuit_open` + `retry_after`
|
|
76
|
+
# (seconds until the cooldown lets a half-open trial through).
|
|
77
|
+
class CircuitOpenError < Error
|
|
78
|
+
attr_reader :ref, :retry_after
|
|
79
|
+
|
|
80
|
+
def initialize(message = nil, ref: nil, retry_after: nil)
|
|
81
|
+
@ref = ref
|
|
82
|
+
@retry_after = retry_after
|
|
83
|
+
super(message || "circuit open for #{ref}")
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def classification
|
|
87
|
+
{ kind: :circuit_open, retryable: true, retry_after: retry_after }.compact
|
|
88
|
+
end
|
|
89
|
+
end
|
|
34
90
|
class StoreError < Error; end # persistence backend failed -> task :failed
|
|
35
91
|
class CancelledError < Error; end # cooperative cancellation -> task :cancelled
|
|
36
92
|
|
|
@@ -75,7 +131,7 @@ module Insika
|
|
|
75
131
|
end
|
|
76
132
|
end
|
|
77
133
|
|
|
78
|
-
# Subagent graph integrity
|
|
134
|
+
# Subagent graph integrity. Raised at DEFINITION-time
|
|
79
135
|
# (CreateAgent/UpdateAgent/boot) by SubagentGraph.validate! — a subagents
|
|
80
136
|
# allowlist that forms a cycle or exceeds the depth cap is a configuration
|
|
81
137
|
# error, never a runtime surprise. A ValidationError so the authoring Command
|
|
@@ -101,7 +157,7 @@ module Insika
|
|
|
101
157
|
end
|
|
102
158
|
end
|
|
103
159
|
|
|
104
|
-
# Workflow I/O contract violation
|
|
160
|
+
# Workflow I/O contract violation. A workflow may declare an
|
|
105
161
|
# `input_schema` / `output_schema`; a value that does not conform is rejected.
|
|
106
162
|
# INPUT is validated SYNCHRONOUSLY (TriggerWorkflow) so it is a ValidationError
|
|
107
163
|
# -> HTTP 422, no run created. OUTPUT is validated inside the fiber after the
|
|
@@ -119,14 +175,25 @@ module Insika
|
|
|
119
175
|
end
|
|
120
176
|
end
|
|
121
177
|
|
|
122
|
-
# A channel could not hand a reply to its recipient
|
|
178
|
+
# A channel could not hand a reply to its recipient. NOT a turn
|
|
123
179
|
# failure: the turn already completed and its answer is durable in the session —
|
|
124
180
|
# what failed is the delivery, which lives in the OutboxStore with its own status
|
|
125
181
|
# and its own bounded retry. Raised by a channel's `deliver` so the dispatcher can
|
|
126
182
|
# tell "the recipient refused" from "the engine has a bug".
|
|
127
183
|
class DeliveryError < Error; end
|
|
128
184
|
|
|
129
|
-
#
|
|
185
|
+
# WS4 routing failed: a route's delegate agent is not configured, or its turn
|
|
186
|
+
# failed. An operator/config error — the envelope names the :routing stage
|
|
187
|
+
# instead of swallowing it as :unknown.
|
|
188
|
+
class RoutingError < Error; end
|
|
189
|
+
|
|
190
|
+
# WS9 media failed: an audio part could not be fetched or transcribed, an
|
|
191
|
+
# image attachment could not be built, or a media URL was egress-blocked. A
|
|
192
|
+
# customer's voice message that never entered the turn must not be silently
|
|
193
|
+
# dropped — the :media stage names it.
|
|
194
|
+
class MediaError < Error; end
|
|
195
|
+
|
|
196
|
+
# Strict configuration violation (— OpenClaw's config discipline:
|
|
130
197
|
# "recusa boot com chave desconhecida, no silent config compat"). Raised by
|
|
131
198
|
# EnvSchema.enforce! at boot ONLY when strictness is on (INSIKA_CONFIG_STRICT) —
|
|
132
199
|
# by default a bad key WARNS and the engine still boots (last-known-good: a rotated
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
3
|
+
# the PII/secret patterns live in the RUNTIME (single source of
|
|
4
4
|
# truth) — the eval consumes them rather than keeping a divergent copy. Since the
|
|
5
5
|
# module moved under `lib/`, `Safety::Detectors` is loaded by `insika.rb` before this
|
|
6
6
|
# file; the explicit climb out of `evals/` that used to be here is gone.
|
|
@@ -33,12 +33,12 @@ module Insika
|
|
|
33
33
|
# evaluated", never a silent pass. A case passes only if the deterministic checks
|
|
34
34
|
# pass AND (there's no judge verdict OR it passed).
|
|
35
35
|
#
|
|
36
|
-
# `skipped` (a reason, nil = it ran) is the THIRD outcome
|
|
36
|
+
# `skipped` (a reason, nil = it ran) is the THIRD outcome: the
|
|
37
37
|
# deployment lacks something the case declared it needs, so there was nothing to
|
|
38
38
|
# assert. It is never a pass and never a failure — a suite of 40 cases where 12
|
|
39
39
|
# are skipped says something true, where 40 cases with 12 failures on capability
|
|
40
40
|
# grounds says nothing and gets ignored.
|
|
41
|
-
# `pairwise` (a Pairwise::Verdict
|
|
41
|
+
# `pairwise` (a Pairwise::Verdict) is attached when the case
|
|
42
42
|
# carries a `reference:` and a panel is configured. It DELIBERATELY does not enter
|
|
43
43
|
# `pass?`: "worse than the incumbent" is a judgement about a replacement decision,
|
|
44
44
|
# not a regression in the suite, and letting it fail a case would put an opinion
|
|
@@ -53,25 +53,26 @@ module Insika
|
|
|
53
53
|
def judge_pending? = !skipped? && !rubric.to_s.strip.empty? && judge.nil?
|
|
54
54
|
end
|
|
55
55
|
|
|
56
|
-
# Deterministic
|
|
56
|
+
# Deterministic evaluation — cheap, zero-token, zero-flakiness. It's the
|
|
57
57
|
# layer that catches the gross regressions (a tool stopped being called, a secret
|
|
58
|
-
# leaked, the turn errored). Subjective scoring is the LLM-judge in
|
|
58
|
+
# leaked, the turn errored). Subjective scoring is the LLM-judge in.
|
|
59
59
|
module Assertions
|
|
60
|
-
# Named negative detectors for `must_not` now live in the runtime
|
|
60
|
+
# Named negative detectors for `must_not` now live in the runtime. Kept as
|
|
61
61
|
# an alias so any external reference to Evals::Assertions::PII_DETECTORS still
|
|
62
|
-
# resolves; the values ARE the runtime's, never a fork.
|
|
63
|
-
|
|
62
|
+
# resolves; the values ARE the runtime's, never a fork. the
|
|
63
|
+
# pattern data moved to the corpus, still under the same Safety umbrella.
|
|
64
|
+
PII_DETECTORS = Insika::Safety::Corpus::PII
|
|
64
65
|
|
|
65
|
-
# HOW MUCH THE AGENT SHOULD ASK BEFORE ACTING
|
|
66
|
+
# HOW MUCH THE AGENT SHOULD ASK BEFORE ACTING. Declared per
|
|
66
67
|
# case because it is a per-STORE decision, not a universal rule: sometimes the
|
|
67
68
|
# agent should establish the objective before searching ("energia, treino ou
|
|
68
|
-
# sono?" —
|
|
69
|
+
# sono?" — a good store agent does this well), and sometimes asking again is the
|
|
69
70
|
# failure and it should just search. A global assertion would be wrong half
|
|
70
71
|
# the time; the judge is TOLD the policy (Judge#build_prompt) and this layer
|
|
71
72
|
# checks the half that needs no reader.
|
|
72
73
|
#
|
|
73
|
-
# Each rule is stated as the CUSTOMER-VISIBLE fact it checks.
|
|
74
|
-
# this as "questions before the first tool call", written before
|
|
74
|
+
# Each rule is stated as the CUSTOMER-VISIBLE fact it checks. phrased
|
|
75
|
+
# this as "questions before the first tool call", written before: text a
|
|
75
76
|
# model emits before calling a tool never reaches the customer now (it rides
|
|
76
77
|
# `:intermediate`), and the eval is a client of `/v1/responses`, so what it can
|
|
77
78
|
# observe per turn is the published answer plus the tools that turn called.
|
|
@@ -87,7 +88,7 @@ module Insika
|
|
|
87
88
|
|
|
88
89
|
module_function
|
|
89
90
|
|
|
90
|
-
# WHAT THIS DEPLOYMENT LACKS for the case to be worth running
|
|
91
|
+
# WHAT THIS DEPLOYMENT LACKS for the case to be worth running.
|
|
91
92
|
# -> [reason]; empty = run it.
|
|
92
93
|
#
|
|
93
94
|
# `available` is the deployment's answer for this agent:
|
|
@@ -237,7 +238,7 @@ module Insika
|
|
|
237
238
|
|
|
238
239
|
# Runs a named detector over the text. "pii_leak" = union of all PII detectors;
|
|
239
240
|
# otherwise a single named pattern. Delegates to the runtime's single source
|
|
240
|
-
#
|
|
241
|
+
# which itself fails loud on an unknown name (a typo'd assertion must not
|
|
241
242
|
# silently pass).
|
|
242
243
|
def detect(name, text)
|
|
243
244
|
Insika::Safety::Detectors.detect(name, text)
|
|
@@ -4,7 +4,7 @@ require "json"
|
|
|
4
4
|
|
|
5
5
|
module Insika
|
|
6
6
|
module Evals
|
|
7
|
-
#
|
|
7
|
+
# gating. A baseline is the accepted state of the golden
|
|
8
8
|
# set — `{ cases: { id => { pass, score } } }`. A gated run compares against it and
|
|
9
9
|
# blocks only on a REGRESSION, so known-failing cases don't wedge the gate while a
|
|
10
10
|
# real drop (a passing case that now fails, or a judge score that fell past the
|
|
@@ -16,7 +16,7 @@ module Insika
|
|
|
16
16
|
|
|
17
17
|
# [CaseResult] -> baseline hash. `at` is stamped by the caller (kept out of here
|
|
18
18
|
# so the module stays deterministic/testable).
|
|
19
|
-
# A SKIPPED case is left out entirely
|
|
19
|
+
# A SKIPPED case is left out entirely: writing it as `pass:
|
|
20
20
|
# false` would accept "this deployment cannot run it" as the accepted state,
|
|
21
21
|
# and the case would never block anywhere again.
|
|
22
22
|
def snapshot(results, at:)
|
|
@@ -38,7 +38,7 @@ module Insika
|
|
|
38
38
|
# in BOTH are compared: a new case (no baseline entry) never blocks the gate (it
|
|
39
39
|
# shows in the report as ❌ but is not a "regression"); document this in README.
|
|
40
40
|
# • pass→fail : baseline pass, now failing (hard regression).
|
|
41
|
-
# • pass→skipped: baseline pass, now unrunnable HERE.
|
|
41
|
+
# • pass→skipped: baseline pass, now unrunnable HERE. says the
|
|
42
42
|
# gate never blocks ON a skip, and it does not: a case that was
|
|
43
43
|
# already skipped or unknown stays silent. But a case that used
|
|
44
44
|
# to run on this deployment and no longer can means the agent
|
data/lib/insika/evals/golden.rb
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
require "yaml"
|
|
4
4
|
|
|
5
|
-
# Evals — the quality harness
|
|
6
|
-
# call it (the refinement gate of
|
|
5
|
+
# Evals — the quality harness. It lives in `lib/` so the engine itself can
|
|
6
|
+
# call it (the refinement gate of needs to score a candidate agent, and a
|
|
7
7
|
# second copy of the judge would be the worst possible outcome), but it stays a
|
|
8
8
|
# CLIENT: it reaches a running deployment over HTTP through `HttpTransport` and never
|
|
9
9
|
# reads a store directly. `evals/run.rb` is a thin CLI over this module.
|
|
@@ -28,17 +28,17 @@ module Insika
|
|
|
28
28
|
# Names of the negative assertions to run (e.g. "pii_leak", "tool_error").
|
|
29
29
|
def must_not = Array(expect["must_not"]).map(&:to_s)
|
|
30
30
|
|
|
31
|
-
# How much the agent should ask before acting
|
|
31
|
+
# How much the agent should ask before acting. nil = the store
|
|
32
32
|
# has no opinion and only the rubric decides.
|
|
33
33
|
def policy = GoldenLoader.presence(expect["policy"])
|
|
34
34
|
|
|
35
|
-
# What the DEPLOYMENT must have for this case to mean anything
|
|
35
|
+
# What the DEPLOYMENT must have for this case to mean anything.
|
|
36
36
|
# Empty = runs everywhere.
|
|
37
37
|
def required_tools = Array(requires["tools"]).map(&:to_s)
|
|
38
38
|
def required_capabilities = Array(requires["capabilities"]).map(&:to_s)
|
|
39
39
|
def requirements? = !(required_tools + required_capabilities).empty?
|
|
40
40
|
|
|
41
|
-
# THE INCUMBENT'S CONVERSATION for the same opening
|
|
41
|
+
# THE INCUMBENT'S CONVERSATION for the same opening — the other
|
|
42
42
|
# half of a pairwise comparison. Data in the case, not a store read: the eval is
|
|
43
43
|
# a client, and a pair that lives in one reviewable file cannot go stale against
|
|
44
44
|
# a database nobody looked at.
|
|
@@ -47,14 +47,14 @@ module Insika
|
|
|
47
47
|
def reference? = !reference_messages.empty?
|
|
48
48
|
|
|
49
49
|
# Did a PERSON type part of the reference half? After a handoff the operator's
|
|
50
|
-
# words are stored as `role: assistant
|
|
50
|
+
# words are stored as `role: assistant`, and comparing a model to a human
|
|
51
51
|
# and calling it a win is a lie in both directions — so the pair is LABELLED and
|
|
52
52
|
# the report never prints the outcome without it.
|
|
53
53
|
def human_assisted?
|
|
54
54
|
reference_messages.any? { |m| MessageOrigin.origin_of(m) == MessageOrigin::OPERATOR }
|
|
55
55
|
end
|
|
56
56
|
|
|
57
|
-
# LLM-judge rubric + threshold (consumed in
|
|
57
|
+
# LLM-judge rubric + threshold (consumed in — deferred here).
|
|
58
58
|
def rubric = expect["rubric"]
|
|
59
59
|
def min_score = expect["min_score"]
|
|
60
60
|
end
|
|
@@ -126,7 +126,7 @@ module Insika
|
|
|
126
126
|
raise InvalidGolden, "#{where} needs a 'role' of user or assistant" unless %w[user assistant].include?(role)
|
|
127
127
|
|
|
128
128
|
text = presence(raw["text"]) || (raise InvalidGolden, "#{where} needs a non-empty 'text'")
|
|
129
|
-
# The SAME closed vocabulary the engine stamps
|
|
129
|
+
# The SAME closed vocabulary the engine stamps. A typo'd marker would
|
|
130
130
|
# read as "absent" downstream, which is how a human turn gets scored as the
|
|
131
131
|
# incumbent's model.
|
|
132
132
|
origin = begin
|