insika 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +69 -3
- data/README.md +1 -1
- data/bin/insika +22 -7
- data/docs/AGENTS.md +129 -5
- data/docs/CHANNELS.md +1 -1
- data/docs/CONTEXT.md +22 -5
- data/docs/DEPLOY.md +30 -10
- data/docs/EMBEDDING.md +11 -7
- data/docs/EVALS.md +1 -1
- data/docs/LOADTEST.md +3 -2
- data/docs/OBSERVABILITY.md +11 -2
- data/docs/REFINEMENT.md +6 -6
- data/docs/RELEASING.md +7 -7
- data/docs/RUNNING-LOCAL.md +1 -1
- data/docs/SECURITY.md +24 -11
- data/docs/SKILLS.md +189 -3
- data/docs/WHY.md +1 -1
- data/docs/WORKFLOWS.md +2 -2
- data/docs/index.md +1 -1
- data/docs/onboarding/start.md +1 -1
- data/lib/insika/agent_profile.rb +89 -22
- data/lib/insika/alert_dispatcher.rb +139 -0
- data/lib/insika/baseline_store.rb +2 -2
- data/lib/insika/budget_ledger.rb +135 -0
- data/lib/insika/channel_delivery.rb +14 -11
- data/lib/insika/channel_registry.rb +1 -1
- data/lib/insika/channels/relay.rb +3 -3
- data/lib/insika/channels/web/widget.js +2 -2
- data/lib/insika/channels/web.rb +7 -7
- data/lib/insika/channels/webhook.rb +58 -0
- data/lib/insika/chat_builder.rb +62 -13
- data/lib/insika/circuit_state.rb +114 -0
- data/lib/insika/coercion.rb +8 -0
- data/lib/insika/commands/agent_payload.rb +5 -3
- data/lib/insika/commands/create_agent.rb +2 -2
- data/lib/insika/commands/create_session.rb +1 -1
- data/lib/insika/commands/delete_llm_provider.rb +1 -1
- data/lib/insika/commands/delete_skill.rb +43 -0
- data/lib/insika/commands/gate_refinement.rb +12 -12
- data/lib/insika/commands/import_mcp_tools.rb +1 -1
- data/lib/insika/commands/import_tools.rb +4 -4
- data/lib/insika/commands/issue_tenant_token.rb +41 -0
- data/lib/insika/commands/resolve_refinement.rb +1 -1
- data/lib/insika/commands/revoke_token.rb +39 -0
- data/lib/insika/commands/rotate_tenant_token.rb +43 -0
- data/lib/insika/commands/run_refinement.rb +5 -5
- data/lib/insika/commands/send_message.rb +9 -9
- data/lib/insika/commands/set_agent_tools.rb +1 -1
- data/lib/insika/commands/set_skill_agents.rb +60 -19
- data/lib/insika/commands/trigger_workflow.rb +1 -1
- data/lib/insika/commands/update_agent.rb +1 -1
- data/lib/insika/commands/write_data_tool.rb +1 -1
- data/lib/insika/commands/write_golden.rb +1 -1
- data/lib/insika/commands/write_skill.rb +19 -9
- data/lib/insika/config_store.rb +8 -4
- data/lib/insika/context/builder.rb +2 -2
- data/lib/insika/context/fragment.rb +27 -3
- data/lib/insika/context/priority.rb +3 -2
- data/lib/insika/context/providers/memory.rb +1 -1
- data/lib/insika/context/providers/request.rb +1 -1
- data/lib/insika/context/providers/session.rb +17 -2
- data/lib/insika/context/providers/skill.rb +5 -1
- data/lib/insika/context/providers/skill_trigger.rb +128 -0
- data/lib/insika/context_trace_store.rb +92 -0
- data/lib/insika/delegation_store.rb +2 -2
- data/lib/insika/doctor.rb +250 -5
- data/lib/insika/dsl/runtime.rb +12 -9
- data/lib/insika/dsl/server_boot.rb +4 -3
- data/lib/insika/dsl/system.rb +1 -1
- data/lib/insika/dsl.rb +72 -15
- data/lib/insika/edge_limiter.rb +144 -6
- data/lib/insika/egress_guard.rb +3 -3
- data/lib/insika/env_schema.rb +13 -10
- data/lib/insika/errors.rb +61 -5
- data/lib/insika/evals/assertions.rb +12 -12
- data/lib/insika/evals/baseline.rb +3 -3
- data/lib/insika/evals/golden.rb +8 -8
- data/lib/insika/evals/judge.rb +7 -7
- data/lib/insika/evals/pairwise.rb +3 -3
- data/lib/insika/evals/report.rb +2 -2
- data/lib/insika/evals/runner.rb +6 -6
- data/lib/insika/evals/transport.rb +2 -2
- data/lib/insika/event_stream.rb +23 -5
- data/lib/insika/executor.rb +423 -108
- data/lib/insika/frontmatter.rb +1 -1
- data/lib/insika/golden_store.rb +2 -2
- data/lib/insika/http_client.rb +3 -3
- data/lib/insika/inbound_log.rb +1 -1
- data/lib/insika/llm_configurator.rb +3 -3
- data/lib/insika/loop_detector.rb +143 -0
- data/lib/insika/mcp_http_client.rb +4 -4
- data/lib/insika/mcp_tool_ingestor.rb +6 -6
- data/lib/insika/message_origin.rb +2 -2
- data/lib/insika/model_resolver.rb +1 -1
- data/lib/insika/model_selection.rb +5 -4
- data/lib/insika/onboarding.rb +2 -2
- data/lib/insika/outbox_store.rb +2 -2
- data/lib/insika/overlay_tool_registry.rb +3 -4
- data/lib/insika/pack.rb +3 -3
- data/lib/insika/pack_importer.rb +17 -15
- data/lib/insika/pending_action_store.rb +1 -1
- data/lib/insika/plugin/loader.rb +2 -2
- data/lib/insika/policy/policy.rb +1 -1
- data/lib/insika/profile_source.rb +12 -6
- data/lib/insika/provider_error_classifier.rb +160 -0
- data/lib/insika/queue_policy.rb +2 -2
- data/lib/insika/recovery.rb +47 -6
- data/lib/insika/refinement/candidate.rb +4 -4
- data/lib/insika/refinement/evidence_collector.rb +6 -6
- data/lib/insika/refinement/gate.rb +7 -7
- data/lib/insika/refinement/panel.rb +7 -7
- data/lib/insika/refinement/proposer.rb +9 -9
- data/lib/insika/refinement_store.rb +12 -12
- data/lib/insika/reliability.rb +185 -0
- data/lib/insika/safety/config.rb +2 -2
- data/lib/insika/safety/detectors.rb +5 -5
- data/lib/insika/safety/factory.rb +3 -3
- data/lib/insika/safety/input_guardrail.rb +19 -4
- data/lib/insika/safety/moderator.rb +19 -11
- data/lib/insika/safety/output_filter.rb +2 -2
- data/lib/insika/safety/output_validator.rb +2 -2
- data/lib/insika/safety/safe_responses.rb +1 -1
- data/lib/insika/sandbox/boundary.rb +2 -2
- data/lib/insika/sandbox.rb +1 -1
- data/lib/insika/server/app.rb +223 -51
- data/lib/insika/server/boot.rb +4 -4
- data/lib/insika/server/rack_app.rb +15 -7
- data/lib/insika/server/responses.rb +18 -8
- data/lib/insika/server/tenant_auth.rb +61 -0
- data/lib/insika/session_actor.rb +3 -3
- data/lib/insika/session_store.rb +1 -1
- data/lib/insika/settings_store.rb +5 -5
- data/lib/insika/shutdown.rb +4 -4
- data/lib/insika/skill_catalog.rb +127 -20
- data/lib/insika/skill_store.rb +70 -22
- data/lib/insika/steer_injector.rb +1 -1
- data/lib/insika/store.rb +1 -1
- data/lib/insika/studio/app.rb +183 -61
- data/lib/insika/studio/assets/dist/application.js +25 -24
- data/lib/insika/studio/forms.rb +13 -18
- data/lib/insika/studio/nav_icons.rb +1 -1
- data/lib/insika/studio/views/_message.erb +2 -2
- data/lib/insika/studio/views/agent_detail.erb +2 -2
- data/lib/insika/studio/views/agents.erb +1 -1
- data/lib/insika/studio/views/refinement.erb +4 -4
- data/lib/insika/studio/views/session.erb +78 -3
- data/lib/insika/studio/views/settings.erb +7 -12
- data/lib/insika/studio/views/skills.erb +67 -12
- data/lib/insika/subagent_graph.rb +3 -3
- data/lib/insika/task_actor.rb +3 -3
- data/lib/insika/task_store.rb +1 -1
- data/lib/insika/telemetry/pricing.rb +3 -3
- data/lib/insika/telemetry/recorder.rb +1 -1
- data/lib/insika/telemetry.rb +2 -2
- data/lib/insika/testing/store_contract.rb +27 -27
- data/lib/insika/tick.rb +122 -0
- data/lib/insika/token_store.rb +168 -0
- data/lib/insika/tool_assembly.rb +5 -5
- data/lib/insika/tool_definition.rb +8 -8
- data/lib/insika/tool_envelope.rb +1 -1
- data/lib/insika/tool_manifest.rb +6 -6
- data/lib/insika/tool_output_compressor.rb +100 -0
- data/lib/insika/tool_store.rb +1 -1
- data/lib/insika/tool_trace_store.rb +1 -1
- data/lib/insika/tools/concurrency.rb +2 -2
- data/lib/insika/tools/data_defined_tool.rb +4 -5
- data/lib/insika/tools/load_skill.rb +61 -3
- data/lib/insika/tools/stuck_signal.rb +44 -0
- data/lib/insika/tools/subagent.rb +4 -4
- data/lib/insika/tools/subagents.rb +1 -1
- data/lib/insika/turn_output.rb +2 -2
- data/lib/insika/turn_state.rb +17 -13
- data/lib/insika/turn_timing.rb +2 -2
- data/lib/insika/usage_ledger.rb +1 -1
- data/lib/insika/version.rb +1 -1
- data/lib/insika/wiring/graph.rb +77 -26
- data/lib/insika/workflow.rb +1 -1
- data/lib/insika/workflow_registry.rb +1 -1
- data/lib/insika.rb +32 -15
- metadata +19 -2
- data/lib/insika/server/admin_auth.rb +0 -29
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Insika
|
|
4
|
+
# Classifies provider/transport failures by ACTION (B9). Four kinds —
|
|
5
|
+
# the hermes/openclaw structural rule: NON-retryable checked first, so an
|
|
6
|
+
# error we do not recognize defaults to :fatal (a retry would hammer a
|
|
7
|
+
# poisoned credential; a fatal is retried only after the operator fixes the
|
|
8
|
+
# cause):
|
|
9
|
+
#
|
|
10
|
+
# :fatal 401/402/403/400 (auth, billing, permanent quota,
|
|
11
|
+
# bad request, context too long) — retrying does not help.
|
|
12
|
+
# :retryable 5xx/529/socket/timeout — the same call may succeed
|
|
13
|
+
# moments later.
|
|
14
|
+
# :rate_limited_short a 429 that says "back off briefly" (RPM-scale).
|
|
15
|
+
# :rate_limited_long a 429 with a long retry-after — quota-scale.
|
|
16
|
+
#
|
|
17
|
+
# The classification is STRING-based (class names, no constant references):
|
|
18
|
+
# the core loads without ruby_llm, and the smoke-shim's fake RubyLLM is a
|
|
19
|
+
# drop-in. `retry_after` is read from the provider's own Retry-After header
|
|
20
|
+
# when the error carries a response, else a per-kind default.
|
|
21
|
+
class ProviderErrorClassifier
|
|
22
|
+
Classification = Data.define(:kind, :retryable, :retry_after) do
|
|
23
|
+
# the additive envelope fields — compacted so a retry_after-less fatal
|
|
24
|
+
# never invents one.
|
|
25
|
+
def to_h
|
|
26
|
+
{ kind: kind, retryable: retryable, retry_after: retry_after }.compact
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
KINDS = %i[fatal retryable rate_limited_short rate_limited_long].freeze
|
|
31
|
+
|
|
32
|
+
# Above this a 429 means quota, not RPM (this is what tells the two apart —
|
|
33
|
+
# a short 429 wants a quick retry; a long one is a billing event).
|
|
34
|
+
SHORT_RETRY_LIMIT = 60 # seconds
|
|
35
|
+
|
|
36
|
+
DEFAULTS = {
|
|
37
|
+
retryable: 5,
|
|
38
|
+
rate_limited_short: 10,
|
|
39
|
+
rate_limited_long: 300
|
|
40
|
+
}.freeze
|
|
41
|
+
|
|
42
|
+
# RubyLLM's taxonomy (error.rb), matched by class name so the core stays
|
|
43
|
+
# ruby_llm-free at load time.
|
|
44
|
+
FATAL_ERROR_NAMES = %w[
|
|
45
|
+
RubyLLM::ContextLengthExceededError RubyLLM::BadRequestError
|
|
46
|
+
RubyLLM::UnauthorizedError RubyLLM::PaymentRequiredError RubyLLM::ForbiddenError
|
|
47
|
+
].freeze
|
|
48
|
+
RATE_LIMITED_ERROR_NAME = "RubyLLM::RateLimitError"
|
|
49
|
+
RETRYABLE_ERROR_NAMES = %w[
|
|
50
|
+
RubyLLM::ServerError RubyLLM::ServiceUnavailableError RubyLLM::OverloadedError
|
|
51
|
+
].freeze
|
|
52
|
+
RUBY_LLM_ERROR_NAMES = (
|
|
53
|
+
FATAL_ERROR_NAMES + [RATE_LIMITED_ERROR_NAME] +
|
|
54
|
+
RETRYABLE_ERROR_NAMES + ["RubyLLM::Error"]
|
|
55
|
+
).freeze
|
|
56
|
+
|
|
57
|
+
# Transport failures while talking to the provider: connection
|
|
58
|
+
# refused/reset, DNS, TLS, timeouts (Faraday wraps its own names; the
|
|
59
|
+
# stdlib ones surface from raw sockets).
|
|
60
|
+
TRANSPORT_NAME_PATTERNS = [
|
|
61
|
+
/\AFaraday::/,
|
|
62
|
+
/\ASocketError\z/,
|
|
63
|
+
/\AIOError\z/,
|
|
64
|
+
/\AErrno::/,
|
|
65
|
+
/\ANet::(Read|Open)Timeout\z/,
|
|
66
|
+
/\ATimeout::Error\z/,
|
|
67
|
+
/\AOpenSSL::SSL::SSLError\z/
|
|
68
|
+
].freeze
|
|
69
|
+
|
|
70
|
+
class << self
|
|
71
|
+
# -> Classification
|
|
72
|
+
def classify(error)
|
|
73
|
+
names = class_names(error)
|
|
74
|
+
|
|
75
|
+
# Non-retryable first (the structural rule): a known fatal is NEVER
|
|
76
|
+
# retried, and an unknown error defaults to fatal — never to retry.
|
|
77
|
+
return fatal if (names & FATAL_ERROR_NAMES).any?
|
|
78
|
+
|
|
79
|
+
return rate_limited(error) if names.include?(RATE_LIMITED_ERROR_NAME)
|
|
80
|
+
|
|
81
|
+
return retryable if (names & RETRYABLE_ERROR_NAMES).any?
|
|
82
|
+
return retryable if transport?(names)
|
|
83
|
+
|
|
84
|
+
# A generic RubyLLM::Error (or a raw HTTP error) still carries the
|
|
85
|
+
# status: 429 and 5xx are retryable regardless of the wrapping class.
|
|
86
|
+
case http_status(error)
|
|
87
|
+
when 429 then rate_limited(error)
|
|
88
|
+
when 500..599 then retryable
|
|
89
|
+
else fatal
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# True when the error came from the provider call itself (RubyLLM
|
|
94
|
+
# family or transport) — the executor routes these to the :ruby_llm
|
|
95
|
+
# stage with a wrapped classification instead of :unknown.
|
|
96
|
+
def provider_error?(error)
|
|
97
|
+
names = class_names(error)
|
|
98
|
+
(names & RUBY_LLM_ERROR_NAMES).any? || transport?(names)
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# The typed ProviderError the executor stores and emits.
|
|
102
|
+
def wrap(error)
|
|
103
|
+
c = classify(error)
|
|
104
|
+
Insika::ProviderError.new(
|
|
105
|
+
error.message || error.class.name,
|
|
106
|
+
kind: c.kind, retryable: c.retryable, retry_after: c.retry_after
|
|
107
|
+
)
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
private
|
|
111
|
+
|
|
112
|
+
def fatal
|
|
113
|
+
Classification.new(kind: :fatal, retryable: false, retry_after: nil)
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def retryable
|
|
117
|
+
Classification.new(kind: :retryable, retryable: true,
|
|
118
|
+
retry_after: DEFAULTS[:retryable])
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def rate_limited(error)
|
|
122
|
+
ra = retry_after_header(error)
|
|
123
|
+
if ra && ra > SHORT_RETRY_LIMIT
|
|
124
|
+
Classification.new(kind: :rate_limited_long, retryable: true, retry_after: ra)
|
|
125
|
+
else
|
|
126
|
+
Classification.new(kind: :rate_limited_short, retryable: true,
|
|
127
|
+
retry_after: ra || DEFAULTS[:rate_limited_short])
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def class_names(error)
|
|
132
|
+
([error.class.name] + Array(error.class.ancestors).map(&:name)).compact
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def transport?(names)
|
|
136
|
+
names.any? { |n| TRANSPORT_NAME_PATTERNS.any? { |p| p.match?(n) } }
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
# The provider's own Retry-After (seconds), when the error carries a
|
|
140
|
+
# response. All access guarded — a bare double must not raise.
|
|
141
|
+
def retry_after_header(error)
|
|
142
|
+
headers = response_headers(error)
|
|
143
|
+
value = headers && (headers["retry-after"] || headers["Retry-After"])
|
|
144
|
+
value = value.to_s.strip
|
|
145
|
+
value.match?(/\A\d+\z/) ? value.to_i : nil
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def http_status(error)
|
|
149
|
+
response = error.respond_to?(:response) ? error.response : nil
|
|
150
|
+
status = response && response.respond_to?(:status) ? response.status : nil
|
|
151
|
+
status&.to_i
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
def response_headers(error)
|
|
155
|
+
response = error.respond_to?(:response) ? error.response : nil
|
|
156
|
+
response && response.respond_to?(:headers) ? response.headers : nil
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
end
|
|
160
|
+
end
|
data/lib/insika/queue_policy.rb
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
require_relative "coercion"
|
|
4
4
|
|
|
5
5
|
module Insika
|
|
6
|
-
#
|
|
6
|
+
# what happens to an inbound message for a session that is ALREADY
|
|
7
7
|
# busy. Today the engine has exactly one answer, "it waits in line"; this names
|
|
8
8
|
# that answer `followup` and adds three others:
|
|
9
9
|
#
|
|
@@ -32,7 +32,7 @@ module Insika
|
|
|
32
32
|
# `turn_timeout` because they are bounds on the same thing — how much work one
|
|
33
33
|
# turn is allowed to absorb.
|
|
34
34
|
class QueuePolicy
|
|
35
|
-
# All four of
|
|
35
|
+
# All four of are delivered, so there is no "specified but unshipped"
|
|
36
36
|
# tier any more — a mode outside this set is a typo, and it is refused rather than
|
|
37
37
|
# approximated. Treating an unknown mode as `followup` would look exactly like a
|
|
38
38
|
# mode that never fires.
|
data/lib/insika/recovery.rb
CHANGED
|
@@ -17,7 +17,7 @@ module Insika
|
|
|
17
17
|
class Recovery
|
|
18
18
|
SWEEP_SCOPE = "recovery"
|
|
19
19
|
|
|
20
|
-
# The per-boot-generation sweep claim
|
|
20
|
+
# The per-boot-generation sweep claim. N workers share one
|
|
21
21
|
# store, and the sweep's "orphaned :running" test is per-process: a worker
|
|
22
22
|
# booting while a sibling holds a live turn would see it as an orphan and
|
|
23
23
|
# re-run it. So the TASK sweep runs once per boot generation — the first
|
|
@@ -52,27 +52,59 @@ module Insika
|
|
|
52
52
|
# -> { resumed: [ids], failed: [ids] }
|
|
53
53
|
# The initial sweep runs OUTSIDE the per-task rescue: a StoreError here
|
|
54
54
|
# aborts the boot.
|
|
55
|
-
|
|
55
|
+
#
|
|
56
|
+
# stale_after (seconds): the periodic tick's semantics instead of
|
|
57
|
+
# boot's. Only :queued/:running tasks untouched for longer than that are
|
|
58
|
+
# candidates — :waiting/:paused are idle by nature (a human wait), so
|
|
59
|
+
# staleness cannot tell a live one from a dead one, and they stay boot
|
|
60
|
+
# recovery's. The threshold exists because a live :running turn is bounded
|
|
61
|
+
# by turn_timeout: anything older cannot be alive.
|
|
62
|
+
def run(stale_after: nil)
|
|
56
63
|
resumed = []
|
|
57
64
|
failed = []
|
|
58
65
|
# Ordered by created_at: tasks from the SAME session are reprocessed
|
|
59
66
|
# in their original order. Global time ordering is harmless for standalone tasks.
|
|
60
|
-
# 1) interrupted (crash mid-flight) -> resume from the checkpoint.
|
|
61
|
-
|
|
67
|
+
# 1) interrupted (crash mid-flight) -> resume from the checkpoint. Tick mode
|
|
68
|
+
# narrows this to :running — the only mid-flight state staleness can judge.
|
|
69
|
+
running = stale_after ? @task_store.with_status(:running) : @task_store.running_or_interrupted
|
|
70
|
+
sweep(running, stale_after).each { |task| process(task, resumed, failed, tick: !stale_after.nil?) }
|
|
62
71
|
# 2) queued but never started (turn in the SessionActor queue at crash time)
|
|
63
72
|
# -> re-run from scratch (the same resume_task handles :queued). Without
|
|
64
73
|
# this, a :queued turn in the volatile queue would be lost on kill -9.
|
|
65
|
-
@task_store.queued
|
|
74
|
+
sweep(@task_store.queued, stale_after).each { |task| process(task, resumed, failed, tick: !stale_after.nil?) }
|
|
66
75
|
log(:info, "recovery finished: #{resumed.size} resumed, #{failed.size} failed")
|
|
67
76
|
{ resumed: resumed, failed: failed }
|
|
68
77
|
end
|
|
69
78
|
|
|
70
79
|
private
|
|
71
80
|
|
|
81
|
+
# Boot mode (stale_after nil) takes the candidate list as-is; tick mode keeps
|
|
82
|
+
# only tasks untouched past the threshold — the liveness gate of.
|
|
83
|
+
def sweep(tasks, stale_after)
|
|
84
|
+
tasks = tasks.sort_by(&:created_at)
|
|
85
|
+
return tasks unless stale_after
|
|
86
|
+
|
|
87
|
+
cutoff = Time.now.utc - stale_after
|
|
88
|
+
tasks.select { |task| stale?(task, cutoff) }
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def stale?(task, cutoff)
|
|
92
|
+
Time.iso8601(task.updated_at.to_s) < cutoff
|
|
93
|
+
rescue ArgumentError
|
|
94
|
+
true # an unreadable timestamp is not proof of life — treat as stale
|
|
95
|
+
end
|
|
96
|
+
|
|
72
97
|
# Failing to resume ONE task does not bring down the boot: a non-store
|
|
73
98
|
# dispatch/latest error -> mark :failed and continue. StoreError -> propagate
|
|
74
99
|
# (aborts the boot).
|
|
75
|
-
|
|
100
|
+
#
|
|
101
|
+
# tick: true flips ONE rescue: a ValidationError from the dispatch
|
|
102
|
+
# is ResumeTask's local liveness check ("task is running") — someone alive
|
|
103
|
+
# owns it, the normal case on a timer, so the task is SKIPPED for the next
|
|
104
|
+
# tick. At boot a ValidationError means corruption (nothing may be alive),
|
|
105
|
+
# so it still fails the task there. This is the trap defused in-process:
|
|
106
|
+
# the tick must never murder a live turn with its own recovery.
|
|
107
|
+
def process(task, resumed, failed, tick: false)
|
|
76
108
|
# :queued never started (no checkpoint) but IS recoverable — ResumeTask
|
|
77
109
|
# re-runs from the Command. An interrupted task requires a checkpoint;
|
|
78
110
|
# without one, it is unrecoverable.
|
|
@@ -88,6 +120,15 @@ module Insika
|
|
|
88
120
|
end
|
|
89
121
|
rescue Insika::StoreError
|
|
90
122
|
raise
|
|
123
|
+
rescue Insika::ValidationError => e
|
|
124
|
+
# Boot keeps the old contract (corruption -> :failed); the tick skips.
|
|
125
|
+
if tick
|
|
126
|
+
log(:info, "skipped (alive elsewhere): #{task.id} — #{e.message}")
|
|
127
|
+
else
|
|
128
|
+
fail_task(task.id, class_name: e.class.name, message: e.message, stage: "recovery")
|
|
129
|
+
failed << task.id unless failed.include?(task.id)
|
|
130
|
+
log(:warn, "failed to resume #{task.id}: #{e.class}: #{e.message}")
|
|
131
|
+
end
|
|
91
132
|
rescue StandardError => e
|
|
92
133
|
fail_task(task.id, class_name: e.class.name, message: e.message, stage: "recovery")
|
|
93
134
|
failed << task.id unless failed.include?(task.id)
|
|
@@ -4,7 +4,7 @@ require "securerandom"
|
|
|
4
4
|
|
|
5
5
|
module Insika
|
|
6
6
|
module Refinement
|
|
7
|
-
# ONE proposed change to an agent's instruction files
|
|
7
|
+
# ONE proposed change to an agent's instruction files. Data, not
|
|
8
8
|
# a diff of free text, and that is the load-bearing decision of the whole phase:
|
|
9
9
|
#
|
|
10
10
|
# · an ANCHORED edit is reviewable — the operator reads three lines, not a
|
|
@@ -40,7 +40,7 @@ module Insika
|
|
|
40
40
|
|
|
41
41
|
# A dropped edit and why. Kept ON the candidate rather than logged: "the model
|
|
42
42
|
# proposed four things and one was stale" is exactly what an operator reviewing
|
|
43
|
-
# the loop's usefulness needs, and
|
|
43
|
+
# the loop's usefulness needs, and asks them to judge precisely that.
|
|
44
44
|
Dropped = Data.define(:file, :op, :reason) do
|
|
45
45
|
def to_h = { "file" => file, "op" => op, "reason" => reason }
|
|
46
46
|
end
|
|
@@ -63,7 +63,7 @@ module Insika
|
|
|
63
63
|
"edits" => edits.map(&:to_h), "dropped" => dropped.map(&:to_h) }
|
|
64
64
|
end
|
|
65
65
|
|
|
66
|
-
# The bounds, all config
|
|
66
|
+
# The bounds, all config. Defaults are deliberately small: what makes a
|
|
67
67
|
# diff reviewable is that it is short, and what keeps the gate's signal readable
|
|
68
68
|
# is that a run changed few things.
|
|
69
69
|
DEFAULT_LIMITS = { "max_edits" => 3, "max_bytes" => 1200, "max_total_growth" => 0.15 }.freeze
|
|
@@ -75,7 +75,7 @@ module Insika
|
|
|
75
75
|
|
|
76
76
|
# raw: { "proposer" =>, "rationale" =>, "edits" => [ … ] } (string keys)
|
|
77
77
|
# allowlist: the agent's `refinement.files`. EMPTY MEANS NOTHING IS WRITABLE —
|
|
78
|
-
# report-only
|
|
78
|
+
# report-only, so every edit drops. Not "no restriction":
|
|
79
79
|
# an unset allowlist that meant "anything" would turn a missing
|
|
80
80
|
# config into the most permissive setting there is.
|
|
81
81
|
# contents: name => current content, for staleness and growth.
|
|
@@ -5,7 +5,7 @@ require "time"
|
|
|
5
5
|
|
|
6
6
|
module Insika
|
|
7
7
|
module Refinement
|
|
8
|
-
#
|
|
8
|
+
# Reads a window of an agent's real traffic and emits RANKED
|
|
9
9
|
# FINDINGS — "here is what broke, how often, and in which conversations". No
|
|
10
10
|
# model runs here and nothing is written to the agent: this is the evidence half
|
|
11
11
|
# of the loop, and it is deliberately useful on its own.
|
|
@@ -17,7 +17,7 @@ module Insika
|
|
|
17
17
|
# ToolTraceStore — per-session tool calls with ok/args/result (already masked
|
|
18
18
|
# and clipped by the store itself)
|
|
19
19
|
#
|
|
20
|
-
# Two signals of
|
|
20
|
+
# Two signals of are NOT computed here, and that is a finding about
|
|
21
21
|
# the engine rather than about an agent: guardrail decisions and edge-limit hits
|
|
22
22
|
# are emitted as EVENTS and never persisted, so the only durable footprint they
|
|
23
23
|
# leave is the canned safe reply in the transcript — which is exactly what the
|
|
@@ -42,7 +42,7 @@ module Insika
|
|
|
42
42
|
# result delivered as a new turn — is persisted with `role: user` like any
|
|
43
43
|
# other, because it is what the model saw. Counting those as the customer
|
|
44
44
|
# repeating themselves turned the first production run into 219 false positives,
|
|
45
|
-
# every one of them the engine reading its own `<
|
|
45
|
+
# every one of them the engine reading its own `<store_cep_required>` back.
|
|
46
46
|
#
|
|
47
47
|
# `MessageOrigin` is the structural answer and is preferred whenever a message
|
|
48
48
|
# carries it. This regex stays for everything written before that field existed
|
|
@@ -188,12 +188,12 @@ module Insika
|
|
|
188
188
|
# of an instruction the agent is not following. Heuristic on purpose (token overlap,
|
|
189
189
|
# no model call); the snippet is PII-redacted.
|
|
190
190
|
#
|
|
191
|
-
# "After the agent answered" is the load-bearing half, and
|
|
191
|
+
# "After the agent answered" is the load-bearing half, and is what forced
|
|
192
192
|
# it to be said out loud. Two customer messages in a row is now ORDINARY: `collect`
|
|
193
193
|
# merges the fragments a person types into one turn and `steer` appends one into a
|
|
194
194
|
# run in flight, so a turn legitimately holds two of them. Someone still typing is
|
|
195
195
|
# not someone repeating themselves — and a steered message cannot be told apart in
|
|
196
|
-
# the transcript, because it correctly declares no origin
|
|
196
|
+
# the transcript, because it correctly declares no origin. The structure is the
|
|
197
197
|
# only honest signal: a reply has to sit between the two.
|
|
198
198
|
def repetition_findings(session_ids)
|
|
199
199
|
hits = session_ids.flat_map do |sid|
|
|
@@ -320,7 +320,7 @@ module Insika
|
|
|
320
320
|
|
|
321
321
|
def words(text) = text.to_s.downcase.scan(/[[:alnum:]]+/).uniq
|
|
322
322
|
|
|
323
|
-
# Every reply the deployment may emit INSTEAD of a real answer: the
|
|
323
|
+
# Every reply the deployment may emit INSTEAD of a real answer: the
|
|
324
324
|
# defaults, the agent's own overrides, and the edge limiter's reply (per-agent
|
|
325
325
|
# first, then the platform setting).
|
|
326
326
|
def canned_replies(profile)
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
module Insika
|
|
4
4
|
module Refinement
|
|
5
|
-
# Scores a candidate by RUNNING it
|
|
6
|
-
# the edit looks good — that measures nothing, and
|
|
5
|
+
# Scores a candidate by RUNNING it. Not by asking a model whether
|
|
6
|
+
# the edit looks good — that measures nothing, and says so in one line.
|
|
7
7
|
#
|
|
8
8
|
# 1. clone the agent into a throwaway id (`<agent>-cand-<run8>`)
|
|
9
9
|
# 2. copy its instruction files, apply the candidate's edits to the COPY
|
|
@@ -27,7 +27,7 @@ module Insika
|
|
|
27
27
|
# it — nil when no turn carried usage. `cached` is how much of that came from
|
|
28
28
|
# the prompt cache, kept separate because it is what explains one candidate
|
|
29
29
|
# costing 8× another over the same cases. `tokens` is what the panel's budget
|
|
30
|
-
#
|
|
30
|
+
# spends and what the operator reads on the run: a gate is the expensive
|
|
31
31
|
# half of refinement and a loop whose cost is invisible is one nobody can decide
|
|
32
32
|
# to keep.
|
|
33
33
|
Report = Data.define(:candidate_id, :passed, :reason, :cases, :passed_cases,
|
|
@@ -48,9 +48,9 @@ module Insika
|
|
|
48
48
|
# credential the operator can rotate.
|
|
49
49
|
# capabilities_factory: -> an `Evals::HttpCapabilities` for the clone, or nil.
|
|
50
50
|
# Without it a case whose `requires` the agent cannot satisfy RUNS and fails
|
|
51
|
-
# (
|
|
51
|
+
# (says it must skip) — and then the gate and `evals/run.rb`, the
|
|
52
52
|
# two callers of the one evaluator, disagree about what the corpus even
|
|
53
|
-
# measures.
|
|
53
|
+
# measures. exists to prevent exactly that.
|
|
54
54
|
def initialize(profiles:, agent_files:, goldens:, baselines:, transport_factory:,
|
|
55
55
|
capabilities_factory: nil, judge_factory: nil, tolerance: DEFAULT_TOLERANCE)
|
|
56
56
|
@profiles = profiles
|
|
@@ -68,7 +68,7 @@ module Insika
|
|
|
68
68
|
# not gate is more useful than an exception in a log.
|
|
69
69
|
def score(agent_id:, candidate:, run_id:, tolerance: nil)
|
|
70
70
|
cases = @goldens.for_agent(agent_id)
|
|
71
|
-
return refusal(candidate, "the agent has no golden cases — nothing to gate against
|
|
71
|
+
return refusal(candidate, "the agent has no golden cases — nothing to gate against") if cases.empty?
|
|
72
72
|
|
|
73
73
|
baseline = @baselines.get(agent_id)
|
|
74
74
|
# Without an accepted state, `Baseline.compare` compares nothing and reports
|
|
@@ -109,7 +109,7 @@ module Insika
|
|
|
109
109
|
# Measured, not reasoned: gating the real pilot agent with `settings["evals"]`
|
|
110
110
|
# unset reported **6/6, no regression** against a baseline the same corpus had
|
|
111
111
|
# just scored **2/6** — `produto-sem-cep` was judged 0.0 and "passed". Both
|
|
112
|
-
# candidates on the panel cleared. That is
|
|
112
|
+
# candidates on the panel cleared. That is's failure exactly: the CLI and
|
|
113
113
|
# the gate, the two callers of the one evaluator, disagreeing about what the
|
|
114
114
|
# corpus measures.
|
|
115
115
|
judge = @judge_factory&.call
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
module Insika
|
|
4
4
|
module Refinement
|
|
5
|
-
# What a refinement run is allowed to SPEND
|
|
5
|
+
# What a refinement run is allowed to SPEND.
|
|
6
6
|
#
|
|
7
7
|
# A panel of 3 proposers over a 7-case golden set is 3 model calls plus 21
|
|
8
8
|
# replayed conversations, each a real turn with real tools. That is the honest
|
|
@@ -52,15 +52,15 @@ module Insika
|
|
|
52
52
|
"unmetered" => @unmetered }
|
|
53
53
|
end
|
|
54
54
|
|
|
55
|
-
# The proposer PANEL
|
|
55
|
+
# The proposer PANEL: N models write N independent
|
|
56
56
|
# candidates, the gate scores each one by replaying the golden set, and the best
|
|
57
57
|
# SURVIVOR becomes the proposal a human is asked about.
|
|
58
58
|
#
|
|
59
59
|
# Independent, not consensus-seeking. Two models agreeing on wording is weak
|
|
60
|
-
# evidence and a golden case passing is strong evidence
|
|
60
|
+
# evidence and a golden case passing is strong evidence, so convergence only
|
|
61
61
|
# ever breaks a tie between candidates the gate already ranked equal.
|
|
62
62
|
#
|
|
63
|
-
# A panel of one is
|
|
63
|
+
# A panel of one is unchanged, which is why there is no second code path:
|
|
64
64
|
# `refinement.proposer` (a single ref) resolves to a one-element panel.
|
|
65
65
|
class Panel
|
|
66
66
|
# One member of the panel: the candidate, WHO wrote it (more than one model when
|
|
@@ -80,7 +80,7 @@ module Insika
|
|
|
80
80
|
# gate: a Refinement::Gate (anything answering #score).
|
|
81
81
|
# proposers: [Refinement::Proposer], already resolved by ProposerFactory.panel.
|
|
82
82
|
# budget: a Budget. The default is unlimited — a deployment that configured
|
|
83
|
-
# none gets
|
|
83
|
+
# none gets behaviour, which had no ceiling either.
|
|
84
84
|
def initialize(gate:, proposers: [], budget: Budget.new, fan_out: nil)
|
|
85
85
|
@gate = gate
|
|
86
86
|
@proposers = Array(proposers)
|
|
@@ -112,7 +112,7 @@ module Insika
|
|
|
112
112
|
# -> the best SURVIVOR, or nil when none passed.
|
|
113
113
|
#
|
|
114
114
|
# Highest graded score first; ties broken by the fewest edits (a smaller diff is
|
|
115
|
-
# a smaller bet), then by how many proposers converged on it
|
|
115
|
+
# a smaller bet), then by how many proposers converged on it. `min_by`
|
|
116
116
|
# over a negated tuple keeps the comparison in one place and stays stable, so
|
|
117
117
|
# two genuinely indistinguishable candidates resolve to the first proposer the
|
|
118
118
|
# operator listed rather than to whichever fiber finished first.
|
|
@@ -131,7 +131,7 @@ module Insika
|
|
|
131
131
|
|
|
132
132
|
private
|
|
133
133
|
|
|
134
|
-
# All proposers at once, bounded by the
|
|
134
|
+
# All proposers at once, bounded by the fan-out cap (says the
|
|
135
135
|
# panel reuses it). Each is one blocking HTTP call to a provider, so the
|
|
136
136
|
# wall-clock is the slowest model rather than their sum.
|
|
137
137
|
#
|
|
@@ -4,7 +4,7 @@ require "json"
|
|
|
4
4
|
|
|
5
5
|
module Insika
|
|
6
6
|
module Refinement
|
|
7
|
-
# Writes a CANDIDATE from a run's findings
|
|
7
|
+
# Writes a CANDIDATE from a run's findings — the
|
|
8
8
|
# one place in refinement where a model is asked for anything.
|
|
9
9
|
#
|
|
10
10
|
# It is deliberately the WEAKEST link and it is built that way: everything this
|
|
@@ -180,8 +180,8 @@ module Insika
|
|
|
180
180
|
|
|
181
181
|
# Resolves WHICH model(s) write the candidate, and builds the ask.
|
|
182
182
|
#
|
|
183
|
-
# refinement.proposers on the agent (
|
|
184
|
-
# -> refinement.proposer ("deepseek/deepseek-
|
|
183
|
+
# refinement.proposers on the agent (a PANEL)
|
|
184
|
+
# -> refinement.proposer ("deepseek/deepseek-v4-flash" | "deepseek-v4-flash")
|
|
185
185
|
# -> the platform utility_model
|
|
186
186
|
# -> nothing, and the caller refuses. There is no default model here on purpose:
|
|
187
187
|
# guessing one spends an operator's provider budget without being asked.
|
|
@@ -189,17 +189,17 @@ module Insika
|
|
|
189
189
|
module_function
|
|
190
190
|
|
|
191
191
|
# config: the agent's `refinement` hash. -> Proposer | nil (the FIRST of the
|
|
192
|
-
# panel —
|
|
192
|
+
# panel — single-proposer entry point, kept because a deployment that
|
|
193
193
|
# never configured a panel is a panel of one).
|
|
194
194
|
def build(config, utility_model: nil, ask_factory: nil, llm: nil)
|
|
195
195
|
panel(config, utility_model: utility_model, ask_factory: ask_factory, llm: llm).first
|
|
196
196
|
end
|
|
197
197
|
|
|
198
198
|
# -> [Proposer], in configured order, DEDUPED by model ref and capped at the
|
|
199
|
-
#
|
|
199
|
+
# fan-out (says the panel reuses it). Two entries naming the same
|
|
200
200
|
# model are one proposer: asking the same model twice at temperature 0 measures
|
|
201
|
-
# its variance, which is exactly what
|
|
202
|
-
# `llm
|
|
201
|
+
# its variance, which is exactly what rejected for the judges.
|
|
202
|
+
# `llm`: the graph's own RubyLLM context; nil = the global
|
|
203
203
|
# constant. Today only the deployment root builds a panel, and a deployment
|
|
204
204
|
# is one graph per process — the seam exists so an embedded graph that ever
|
|
205
205
|
# gains the refinement commands proposes on its own credentials.
|
|
@@ -213,7 +213,7 @@ module Insika
|
|
|
213
213
|
end
|
|
214
214
|
end
|
|
215
215
|
|
|
216
|
-
# `proposers` accepts either syntax — a bare ref ("deepseek/deepseek-
|
|
216
|
+
# `proposers` accepts either syntax — a bare ref ("deepseek/deepseek-v4-flash") or
|
|
217
217
|
# the RFC's `{ "model" =>, "provider"? => }` — because the two already coexist in
|
|
218
218
|
# this config (`proposer` is a bare ref, `judges` are hashes) and refusing one of
|
|
219
219
|
# them would only teach operators which page they were reading.
|
|
@@ -248,7 +248,7 @@ module Insika
|
|
|
248
248
|
# is actually configured.
|
|
249
249
|
#
|
|
250
250
|
# Returns the MESSAGE, not `.content`: the token counts ride on it and the
|
|
251
|
-
# budget
|
|
251
|
+
# budget is what spends them. `Proposer` reads either shape.
|
|
252
252
|
def ruby_llm_ask(model, provider, llm: nil)
|
|
253
253
|
require "ruby_llm"
|
|
254
254
|
llm ||= RubyLLM
|
|
@@ -4,7 +4,7 @@ require "securerandom"
|
|
|
4
4
|
require "time"
|
|
5
5
|
|
|
6
6
|
module Insika
|
|
7
|
-
# REFINEMENT DOMAIN store
|
|
7
|
+
# REFINEMENT DOMAIN store. One record per refinement RUN:
|
|
8
8
|
# the window that was read, the ranked findings the EvidenceCollector produced,
|
|
9
9
|
# and the run's outcome. RUNTIME data (it is derived from sessions/tasks/traces),
|
|
10
10
|
# so it takes the raw `store:` like SessionStore/TaskStore — not the ConfigStore.
|
|
@@ -15,8 +15,8 @@ module Insika
|
|
|
15
15
|
# Store contract orders lexicographically) and `latest_for` is its last element.
|
|
16
16
|
# An agent id containing ":" would break that split, so it is rejected on write.
|
|
17
17
|
#
|
|
18
|
-
#
|
|
19
|
-
# status was terminal.
|
|
18
|
+
# wrote no edits anywhere — a Run was a REPORT and every non-collecting
|
|
19
|
+
# status was terminal. adds the rest of the lifecycle on
|
|
20
20
|
# the SAME record, additively: a gated candidate and the operator's decision.
|
|
21
21
|
#
|
|
22
22
|
# collecting ─▶ completed ─▶ gating ─▶ awaiting_approval ─▶ applied
|
|
@@ -24,13 +24,13 @@ module Insika
|
|
|
24
24
|
# ╰─▶ failed or the operator did)
|
|
25
25
|
#
|
|
26
26
|
# **The approval lives here, not in `PendingActionStore`** — a deliberate deviation
|
|
27
|
-
# from
|
|
27
|
+
# from That store is coupled to a suspended TURN: `ApproveAction` resolves
|
|
28
28
|
# the record and then calls `executor.approve(task_id)` to wake a fiber. A
|
|
29
29
|
# refinement proposal has no turn and no fiber, so reusing it would mean inventing
|
|
30
30
|
# a task id, a fake `tool` name, and a wake-up that must never do anything — and
|
|
31
31
|
# the operator's approvals inbox would fill with rows that are not tool calls. The
|
|
32
|
-
# property
|
|
33
|
-
# has had it since
|
|
32
|
+
# property actually wanted is durability across a `kill -9`, and this store
|
|
33
|
+
# has had it since.
|
|
34
34
|
class RefinementStore
|
|
35
35
|
include Coercion
|
|
36
36
|
|
|
@@ -42,7 +42,7 @@ module Insika
|
|
|
42
42
|
|
|
43
43
|
# OPEN means "this run will change without anyone asking": work is in flight
|
|
44
44
|
# (`collecting`, `gating`) or a human owes it an answer (`awaiting_approval`).
|
|
45
|
-
# Everything else is terminal — INCLUDING `completed`, which
|
|
45
|
+
# Everything else is terminal — INCLUDING `completed`, which already
|
|
46
46
|
# treated that way and which stays true: a report is finished, and gating one is
|
|
47
47
|
# a new deliberate action, not a continuation. (Widening `terminal?` here is what
|
|
48
48
|
# the Studio's "latest report" lookup reads, so getting it wrong hides the report.)
|
|
@@ -50,7 +50,7 @@ module Insika
|
|
|
50
50
|
|
|
51
51
|
# `candidate`/`gate` are the WINNER — the one proposal a human is asked about, and
|
|
52
52
|
# the only thing `ResolveRefinement` ever applies. `candidates` is the whole panel
|
|
53
|
-
#
|
|
53
|
+
# every candidate that was built, who wrote it, and how
|
|
54
54
|
# it scored, including the ones that lost and the ones the budget never gated. A
|
|
55
55
|
# phase-C run recorded one candidate and no panel; it still reads back correctly,
|
|
56
56
|
# because a panel of one is the same shape.
|
|
@@ -118,7 +118,7 @@ module Insika
|
|
|
118
118
|
end
|
|
119
119
|
end
|
|
120
120
|
|
|
121
|
-
#
|
|
121
|
+
# the proposal's lifecycle -------------------------------
|
|
122
122
|
|
|
123
123
|
# Attaches the candidate(s) under gate and moves the run to :gating. Only a
|
|
124
124
|
# `completed` run can be gated: a report with no findings has nothing to propose
|
|
@@ -145,15 +145,15 @@ module Insika
|
|
|
145
145
|
end
|
|
146
146
|
|
|
147
147
|
# Records the gate's verdict. A PASS parks the run at :awaiting_approval — a
|
|
148
|
-
# human still has to say yes, which is the product and not a formality
|
|
148
|
+
# human still has to say yes, which is the product and not a formality. A
|
|
149
149
|
# FAIL is terminal as :rejected, with the gate report as the stated reason: the
|
|
150
150
|
# same finding must re-surface with new evidence before anything is proposed
|
|
151
|
-
# again, so there is no silent retry loop
|
|
151
|
+
# again, so there is no silent retry loop.
|
|
152
152
|
#
|
|
153
153
|
# `report` is the WINNER's (or, when nothing survived, the most informative
|
|
154
154
|
# refusal). `panel` is every scored entry — the store attaches it as-is and picks
|
|
155
155
|
# the winning candidate out of it by id. Which candidate WON is the caller's
|
|
156
|
-
# ranking decision
|
|
156
|
+
# ranking decision; this store does not rank, it records. -> Run.
|
|
157
157
|
def gated(id, report:, panel: nil, cost: nil)
|
|
158
158
|
update(id) do |record|
|
|
159
159
|
unless record["status"] == "gating"
|