insika 0.2.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +325 -17
- data/README.md +52 -11
- data/bin/insika +895 -8
- data/bin/insika-router +87 -0
- data/docs/AGENTS.md +229 -163
- data/docs/API.md +56 -0
- data/docs/ARCHITECTURE.md +3 -2
- data/docs/ARTIFACTS.md +95 -0
- data/docs/BENCHMARK.md +2 -2
- data/docs/CHANNELS.md +111 -21
- data/docs/CONTEXT.md +132 -19
- data/docs/DEMO.md +80 -0
- data/docs/DEPLOY.md +75 -4
- data/docs/EMBEDDING.md +1 -1
- data/docs/EVALS.md +146 -2
- data/docs/FACTS.md +135 -0
- data/docs/HARVEST.md +116 -0
- data/docs/KNOWLEDGE.md +290 -0
- data/docs/LOADTEST.md +16 -10
- data/docs/MEDIA.md +128 -0
- data/docs/OBSERVABILITY.md +65 -6
- data/docs/OUTCOMES.md +137 -0
- data/docs/PLUGINS.md +51 -6
- data/docs/POLICY.md +216 -0
- data/docs/REFINEMENT.md +17 -12
- data/docs/RELEASING.md +27 -0
- data/docs/ROUTER.md +213 -0
- data/docs/RUNNING-LOCAL.md +6 -6
- data/docs/SCHEDULING.md +121 -0
- data/docs/SECURITY.md +79 -2
- data/docs/SKILLS.md +13 -4
- data/docs/SOAK.md +127 -0
- data/docs/TEMPLATES.md +134 -0
- data/docs/TOOLS.md +220 -27
- data/docs/WHY.md +1 -1
- data/docs/WORKFLOWS.md +2 -2
- data/docs/_includes/head_custom.html +5 -0
- data/docs/_includes/title.html +13 -0
- data/docs/_sass/color_schemes/insika.scss +32 -0
- data/docs/_sass/custom/custom.scss +199 -0
- data/docs/_sass/custom/setup.scss +26 -0
- data/docs/assets/img/favicon.svg +7 -0
- data/docs/assets/img/insika-mark.svg +7 -0
- data/docs/core-concepts.md +21 -0
- data/docs/domain.md +115 -0
- data/docs/improve.md +20 -0
- data/docs/index.md +9 -6
- data/docs/integrate.md +20 -0
- data/docs/operate.md +13 -6
- data/docs/prompts/ADD-TOOL.md +118 -0
- data/docs/prompts/DIAGNOSE-TURN.md +65 -0
- data/docs/prompts/GO-LIVE.md +138 -0
- data/docs/prompts/RUN-EXAMPLES.md +70 -0
- data/docs/reference.md +19 -0
- data/docs/ship.md +10 -2
- data/docs/start-here.md +18 -0
- data/lib/insika/agent_profile.rb +202 -10
- data/lib/insika/artifact_signing.rb +82 -0
- data/lib/insika/artifact_store.rb +160 -0
- data/lib/insika/balloon_splitter.rb +102 -0
- data/lib/insika/budget_ledger.rb +34 -3
- data/lib/insika/cache_series_store.rb +49 -0
- data/lib/insika/channel_delivery.rb +119 -14
- data/lib/insika/channels/relay.rb +77 -3
- data/lib/insika/channels/web.rb +2 -2
- data/lib/insika/chat_builder.rb +105 -2
- data/lib/insika/checkpoint_store.rb +16 -0
- data/lib/insika/commands/agent_payload.rb +3 -3
- data/lib/insika/commands/backfill_knowledge.rb +145 -0
- data/lib/insika/commands/cancel_followup.rb +49 -0
- data/lib/insika/commands/delete_artifact.rb +35 -0
- data/lib/insika/commands/delete_concept.rb +34 -0
- data/lib/insika/commands/delete_mcp.rb +6 -2
- data/lib/insika/commands/delete_tenant_data.rb +107 -0
- data/lib/insika/commands/export_customer_memory.rb +48 -0
- data/lib/insika/commands/forget_customer.rb +117 -0
- data/lib/insika/commands/freeze_funnel_baseline.rb +113 -0
- data/lib/insika/commands/gate_harvest.rb +138 -0
- data/lib/insika/commands/gate_refinement.rb +1 -1
- data/lib/insika/commands/judge_shadow_pairs.rb +124 -0
- data/lib/insika/commands/memory_forget_fact.rb +20 -4
- data/lib/insika/commands/memory_put_fact.rb +23 -4
- data/lib/insika/commands/promote_harvest.rb +130 -0
- data/lib/insika/commands/record_outcome.rb +46 -0
- data/lib/insika/commands/record_shadow_reply.rb +68 -0
- data/lib/insika/commands/refresh_mcp_tools.rb +47 -0
- data/lib/insika/commands/reject_harvest.rb +38 -0
- data/lib/insika/commands/resolve_proposal.rb +108 -0
- data/lib/insika/commands/restore_concept.rb +34 -0
- data/lib/insika/commands/revoke_contact.rb +49 -0
- data/lib/insika/commands/rollback_harvest.rb +86 -0
- data/lib/insika/commands/run_distillation.rb +186 -0
- data/lib/insika/commands/run_harvest.rb +393 -0
- data/lib/insika/commands/seed_demo_data.rb +31 -0
- data/lib/insika/commands/send_message.rb +103 -6
- data/lib/insika/commands/session_purge.rb +67 -0
- data/lib/insika/commands/upsert_mcp.rb +6 -3
- data/lib/insika/commands/write_concept.rb +57 -0
- data/lib/insika/contact_store.rb +183 -0
- data/lib/insika/context/builder.rb +21 -3
- data/lib/insika/context/fragment.rb +7 -3
- data/lib/insika/context/priority.rb +5 -0
- data/lib/insika/context/provider.rb +17 -3
- data/lib/insika/context/providers/briefing.rb +96 -0
- data/lib/insika/context/providers/knowledge.rb +108 -0
- data/lib/insika/context/providers/memory.rb +16 -7
- data/lib/insika/context/providers/prompt.rb +57 -23
- data/lib/insika/context/providers/skill.rb +2 -0
- data/lib/insika/context/providers/tool_search.rb +2 -0
- data/lib/insika/context_trace_store.rb +38 -2
- data/lib/insika/cron.rb +189 -0
- data/lib/insika/demo/agent_attrs.rb +43 -0
- data/lib/insika/demo/golden_cases.rb +81 -0
- data/lib/insika/demo/seeder.rb +336 -0
- data/lib/insika/distill.rb +224 -0
- data/lib/insika/distill_engine.rb +169 -0
- data/lib/insika/doctor.rb +889 -11
- data/lib/insika/dsl/definition.rb +3 -2
- data/lib/insika/dsl/runtime.rb +69 -82
- data/lib/insika/dsl/server_boot.rb +92 -1
- data/lib/insika/dsl/system.rb +10 -2
- data/lib/insika/dsl.rb +186 -5
- data/lib/insika/edge_limiter.rb +28 -7
- data/lib/insika/env_schema.rb +25 -3
- data/lib/insika/errors.rb +11 -0
- data/lib/insika/evals/assertions.rb +3 -2
- data/lib/insika/evals/golden.rb +41 -4
- data/lib/insika/evals/judge.rb +47 -2
- data/lib/insika/evals/pairwise.rb +29 -6
- data/lib/insika/evals/persona.rb +98 -0
- data/lib/insika/evals/runner.rb +9 -0
- data/lib/insika/evals/simulator.rb +225 -0
- data/lib/insika/evals/transport.rb +83 -1
- data/lib/insika/event_stream.rb +10 -0
- data/lib/insika/evidence.rb +183 -0
- data/lib/insika/executor.rb +869 -76
- data/lib/insika/followup_engine.rb +207 -0
- data/lib/insika/followup_policy.rb +198 -0
- data/lib/insika/followup_store.rb +306 -0
- data/lib/insika/funnel_declaration.rb +106 -0
- data/lib/insika/funnel_fold.rb +179 -0
- data/lib/insika/funnel_store.rb +163 -0
- data/lib/insika/golden_store.rb +17 -2
- data/lib/insika/grounding/matcher.rb +69 -0
- data/lib/insika/grounding.rb +44 -0
- data/lib/insika/harvest/conversion_gate.rb +159 -0
- data/lib/insika/harvest/criterion.rb +98 -0
- data/lib/insika/harvest/gate.rb +194 -0
- data/lib/insika/harvest/negative_list.rb +199 -0
- data/lib/insika/harvest.rb +241 -0
- data/lib/insika/harvest_engine.rb +193 -0
- data/lib/insika/harvest_store.rb +548 -0
- data/lib/insika/knowledge.rb +680 -0
- data/lib/insika/knowledge_store.rb +140 -0
- data/lib/insika/mcp_client.rb +94 -0
- data/lib/insika/mcp_json.rb +74 -0
- data/lib/insika/mcp_live_tool.rb +43 -0
- data/lib/insika/mcp_store.rb +98 -26
- data/lib/insika/mcp_tool_ingestor.rb +30 -8
- data/lib/insika/mcp_tool_registry.rb +100 -0
- data/lib/insika/media.rb +382 -0
- data/lib/insika/memory_audit_store.rb +85 -0
- data/lib/insika/memory_store.rb +264 -23
- data/lib/insika/message_origin.rb +7 -2
- data/lib/insika/middleware.rb +9 -0
- data/lib/insika/model_visible.rb +87 -0
- data/lib/insika/model_visible_trace_store.rb +66 -0
- data/lib/insika/onboarding.rb +22 -1
- data/lib/insika/outbox_store.rb +42 -4
- data/lib/insika/outcome_store.rb +147 -0
- data/lib/insika/overlay_tool_registry.rb +37 -17
- data/lib/insika/packaging.rb +163 -0
- data/lib/insika/parity/criterion.rb +79 -0
- data/lib/insika/parity/verdict.rb +318 -0
- data/lib/insika/prefix_fingerprint.rb +58 -0
- data/lib/insika/profile_source.rb +29 -1
- data/lib/insika/prompt_catalog.rb +10 -0
- data/lib/insika/proposal_store.rb +271 -0
- data/lib/insika/queue_policy.rb +4 -1
- data/lib/insika/refinement/proposer.rb +1 -1
- data/lib/insika/reliability.rb +32 -6
- data/lib/insika/retention.rb +316 -0
- data/lib/insika/router/app.rb +157 -0
- data/lib/insika/router/backend_pool.rb +98 -0
- data/lib/insika/router/hash_ring.rb +55 -0
- data/lib/insika/router/proxy_body.rb +34 -0
- data/lib/insika/router/session_key.rb +54 -0
- data/lib/insika/router.rb +18 -0
- data/lib/insika/routing.rb +101 -0
- data/lib/insika/safety/config.rb +44 -4
- data/lib/insika/safety/corpus.rb +255 -0
- data/lib/insika/safety/detectors.rb +32 -113
- data/lib/insika/safety/factory.rb +15 -2
- data/lib/insika/safety/grounding_enforcer.rb +59 -0
- data/lib/insika/safety/grounding_validator.rb +49 -0
- data/lib/insika/safety/input_guardrail.rb +1 -1
- data/lib/insika/safety/moderator.rb +1 -1
- data/lib/insika/safety/output_filter.rb +9 -5
- data/lib/insika/safety/output_validator.rb +11 -5
- data/lib/insika/schedule.rb +177 -0
- data/lib/insika/schedule_engine.rb +314 -0
- data/lib/insika/schedule_store.rb +208 -0
- data/lib/insika/schema_guard.rb +35 -0
- data/lib/insika/server/app.rb +247 -17
- data/lib/insika/server/rack_app.rb +21 -1
- data/lib/insika/server/responses.rb +40 -1
- data/lib/insika/session_actor.rb +8 -4
- data/lib/insika/session_store.rb +65 -2
- data/lib/insika/settings_store.rb +10 -0
- data/lib/insika/shadow_pair_store.rb +258 -0
- data/lib/insika/skill_catalog.rb +16 -0
- data/lib/insika/soak/envelope.rb +140 -0
- data/lib/insika/soak/report.rb +392 -0
- data/lib/insika/soak/runner.rb +554 -0
- data/lib/insika/steer_injector.rb +21 -10
- data/lib/insika/store.rb +10 -1
- data/lib/insika/stores/memory.rb +6 -0
- data/lib/insika/stores/sqlite.rb +8 -0
- data/lib/insika/studio/app.rb +1437 -54
- data/lib/insika/studio/assets/dist/application.css +1 -1
- data/lib/insika/studio/assets/dist/application.js +23 -23
- data/lib/insika/studio/assets/dist/favicon.svg +6 -0
- data/lib/insika/studio/forms.rb +308 -10
- data/lib/insika/studio/nav_icons.rb +19 -1
- data/lib/insika/studio/views/_agent_tab_cache.erb +25 -0
- data/lib/insika/studio/views/_agent_tab_config.erb +514 -0
- data/lib/insika/studio/views/_agent_tab_history.erb +24 -0
- data/lib/insika/studio/views/_agent_tab_loops.erb +54 -0
- data/lib/insika/studio/views/_agent_tab_memory.erb +51 -0
- data/lib/insika/studio/views/_agent_tab_outcomes.erb +31 -0
- data/lib/insika/studio/views/_agent_tab_prompts.erb +108 -0
- data/lib/insika/studio/views/_agent_tab_skills.erb +38 -0
- data/lib/insika/studio/views/_agents_master.erb +44 -0
- data/lib/insika/studio/views/_message.erb +49 -32
- data/lib/insika/studio/views/agent_detail.erb +62 -278
- data/lib/insika/studio/views/agents.erb +71 -54
- data/lib/insika/studio/views/approvals.erb +4 -1
- data/lib/insika/studio/views/artifact.erb +23 -0
- data/lib/insika/studio/views/artifacts.erb +59 -0
- data/lib/insika/studio/views/chats.erb +4 -1
- data/lib/insika/studio/views/customer.erb +94 -0
- data/lib/insika/studio/views/customers.erb +32 -0
- data/lib/insika/studio/views/evals.erb +6 -3
- data/lib/insika/studio/views/facts.erb +133 -0
- data/lib/insika/studio/views/followups.erb +125 -0
- data/lib/insika/studio/views/funnel.erb +106 -0
- data/lib/insika/studio/views/harvest.erb +234 -0
- data/lib/insika/studio/views/home.erb +106 -66
- data/lib/insika/studio/views/knowledge.erb +123 -0
- data/lib/insika/studio/views/layout.erb +15 -11
- data/lib/insika/studio/views/mcp.erb +174 -80
- data/lib/insika/studio/views/parity.erb +147 -0
- data/lib/insika/studio/views/playground.erb +7 -1
- data/lib/insika/studio/views/session.erb +233 -124
- data/lib/insika/studio/views/settings.erb +41 -1
- data/lib/insika/studio/views/skills.erb +1 -2
- data/lib/insika/studio/views/system_files.erb +1 -1
- data/lib/insika/studio/views/task.erb +13 -0
- data/lib/insika/studio/views/tasks.erb +4 -1
- data/lib/insika/studio/views/tools.erb +24 -10
- data/lib/insika/task_store.rb +21 -1
- data/lib/insika/templates/browser-agent/README.md +36 -0
- data/lib/insika/templates/browser-agent/agent.rb +49 -0
- data/lib/insika/templates/daily-digest/README.md +38 -0
- data/lib/insika/templates/daily-digest/agent.rb +77 -0
- data/lib/insika/templates/repo-explorer/README.md +36 -0
- data/lib/insika/templates/repo-explorer/agent.rb +45 -0
- data/lib/insika/templates/research-analyst/README.md +26 -0
- data/lib/insika/templates/research-analyst/agent.rb +58 -0
- data/lib/insika/templates/review-panel/README.md +20 -0
- data/lib/insika/templates/review-panel/agent.rb +50 -0
- data/lib/insika/templates/travel-planner/README.md +35 -0
- data/lib/insika/templates/travel-planner/agent.rb +87 -0
- data/lib/insika/templates.rb +112 -0
- data/lib/insika/testing/store_contract.rb +27 -6
- data/lib/insika/tick.rb +47 -11
- data/lib/insika/timezone.rb +45 -0
- data/lib/insika/tool_definition.rb +17 -7
- data/lib/insika/tool_envelope.rb +69 -0
- data/lib/insika/tool_manifest.rb +5 -1
- data/lib/insika/tools/data_defined_tool.rb +10 -0
- data/lib/insika/tools/generate_image.rb +89 -0
- data/lib/insika/tools/load_knowledge.rb +74 -0
- data/lib/insika/tools/run_persona_eval.rb +328 -0
- data/lib/insika/tools/save_artifact.rb +95 -0
- data/lib/insika/tools/schedule_followup.rb +164 -0
- data/lib/insika/tools/tts.rb +47 -0
- data/lib/insika/tools/update_briefing.rb +126 -0
- data/lib/insika/turn_output.rb +1 -1
- data/lib/insika/turn_state.rb +49 -1
- data/lib/insika/turn_timing.rb +22 -2
- data/lib/insika/version.rb +1 -1
- data/lib/insika/vitals.rb +84 -0
- data/lib/insika/wiring/graph.rb +472 -13
- data/lib/insika/wiring/graph_chat.rb +102 -0
- data/lib/insika.rb +149 -3
- metadata +181 -5
- data/docs/build.md +0 -14
- data/docs/understand.md +0 -10
data/bin/insika
CHANGED
|
@@ -4,8 +4,9 @@
|
|
|
4
4
|
# `insika` CLI — strict config + doctor --fix. The operator's
|
|
5
5
|
# DX front door to the strict-config discipline:
|
|
6
6
|
#
|
|
7
|
-
# insika doctor [--fix] [--json]
|
|
8
|
-
# the safe autofixes;
|
|
7
|
+
# insika doctor [--fix] [--json] [--domain] diagnose the deployment config; --fix applies
|
|
8
|
+
# the safe autofixes; --domain appends the domain inventory;
|
|
9
|
+
# exits non-zero on any error
|
|
9
10
|
# insika env [--json] list the known config keys + current (masked) values
|
|
10
11
|
# insika version print the engine version
|
|
11
12
|
# insika help this text
|
|
@@ -15,6 +16,7 @@
|
|
|
15
16
|
# DEEPSEEK, no seeding. So it is safe to run against a live deployment's volume.
|
|
16
17
|
|
|
17
18
|
require "optparse"
|
|
19
|
+
require "fileutils"
|
|
18
20
|
require_relative "../lib/insika"
|
|
19
21
|
|
|
20
22
|
module Insika
|
|
@@ -27,9 +29,19 @@ module Insika
|
|
|
27
29
|
when "doctor" then doctor(argv)
|
|
28
30
|
when "env" then env(argv)
|
|
29
31
|
when "refine" then refine(argv)
|
|
32
|
+
when "harvest" then harvest(argv)
|
|
33
|
+
when "harvest:negative" then harvest_negative_import(argv)
|
|
34
|
+
when "harvest:criterion" then harvest_criterion_check(argv)
|
|
35
|
+
when "knowledge:backfill" then knowledge_backfill(argv)
|
|
36
|
+
when "knowledge:export" then knowledge_export(argv)
|
|
37
|
+
when "soak" then exit(Insika::Soak::Runner.main(argv, stdout: $stdout, stderr: $stderr))
|
|
30
38
|
when "evals:import" then evals_import(argv)
|
|
31
39
|
when "evals:export" then evals_export(argv)
|
|
32
40
|
when "evals:baseline" then evals_baseline(argv)
|
|
41
|
+
when "evals:simulate" then evals_simulate(argv)
|
|
42
|
+
when "demo:seed" then demo_seed(argv)
|
|
43
|
+
when "mcp" then mcp(argv)
|
|
44
|
+
when "new" then new_from_template(argv)
|
|
33
45
|
when "version", "--version", "-v" then puts "insika #{Insika::VERSION}"
|
|
34
46
|
when "help", "--help", "-h" then puts usage
|
|
35
47
|
else
|
|
@@ -45,28 +57,66 @@ module Insika
|
|
|
45
57
|
Usage: insika <command> [options]
|
|
46
58
|
|
|
47
59
|
Commands:
|
|
48
|
-
doctor [--fix] [--json]
|
|
60
|
+
doctor [--fix] [--json] [--domain] diagnose the deployment configuration
|
|
49
61
|
env [--json] list known config keys + current (masked) values
|
|
50
62
|
refine --agent ID [...] report what broke in an agent's real traffic
|
|
63
|
+
harvest --agent ID [...] mine one window of finished traffic
|
|
64
|
+
harvest:negative import --agent ID --file F
|
|
65
|
+
seed the profile's negative list from a rules file
|
|
66
|
+
harvest:criterion check --file F
|
|
67
|
+
strict-load the frozen conversion criterion
|
|
68
|
+
knowledge:backfill --agent ID [--since ISO8601] [--json]
|
|
69
|
+
the recovery path for post-turn extraction: replays
|
|
70
|
+
the agent's stored sessions through the same
|
|
71
|
+
extractor a live turn uses
|
|
72
|
+
knowledge:export --agent ID [--out DIR] [--tenant T] [--format md|graphml]
|
|
73
|
+
writes one <name>.md per learned concept under DIR —
|
|
74
|
+
a dump, not a converter (the storage format IS the
|
|
75
|
+
export format), for okf-gem/graphify tooling; --format
|
|
76
|
+
graphml writes one knowledge.graphml graph file instead
|
|
77
|
+
soak --run | --verify FILE | --preflight the 72h soak
|
|
51
78
|
evals:import [--dir D] load the golden corpus into the store (editable in Studio)
|
|
52
79
|
evals:export [--dir D] write the stored cases back out as YAML
|
|
53
80
|
evals:baseline <sub> the accepted state per agent: show | import | export
|
|
81
|
+
evals:simulate --persona F --target <agent|url>
|
|
82
|
+
generate a simulated conversation against an
|
|
83
|
+
agent: a persona model talks to the
|
|
84
|
+
target until max_turns or a stop marker; the
|
|
85
|
+
transcript is scored by the judge panel.
|
|
86
|
+
Requires --staging or --eval-profile (a simulated
|
|
87
|
+
run must not write for real); --eval-profile is
|
|
88
|
+
verified against the target's derived
|
|
89
|
+
side-effect tools (--eval-tools).
|
|
90
|
+
demo:seed [--force] populate the "demo-store" agent with enough data to see
|
|
91
|
+
every Studio loop at once (funnel+baseline, follow-ups,
|
|
92
|
+
refinement runs, approvals, distillation, evals+baseline)
|
|
93
|
+
mcp list the configured MCP instances
|
|
94
|
+
mcp add --name N [...] add/edit an instance (see mcp add --help)
|
|
95
|
+
mcp remove NAME delete an instance
|
|
96
|
+
mcp import FILE.json upsert every entry of a `mcpServers` JSON document
|
|
97
|
+
mcp test NAME connect live, list its tools, print them (or the error)
|
|
98
|
+
mcp refresh NAME same as test, plus caches the result for display
|
|
99
|
+
new --list the template gallery: trail + one-line description
|
|
100
|
+
new <template> [dir] copy a template (default dir: ./<template>) and print the run line
|
|
54
101
|
version print the engine version
|
|
55
102
|
help show this help
|
|
56
103
|
|
|
57
104
|
Environment:
|
|
58
105
|
INSIKA_DB SQLite path (durable). Unset -> ephemeral memory.
|
|
59
106
|
INSIKA_CONFIG_STRICT when truthy, the server refuses boot on any finding.
|
|
107
|
+
INSIKA_MCP_STDIO set to 1 to allow stdio MCP instances to start
|
|
108
|
+
(arbitrary command execution by config — off by default)
|
|
60
109
|
TXT
|
|
61
110
|
end
|
|
62
111
|
|
|
63
|
-
# insika doctor [--fix] [--json]
|
|
112
|
+
# insika doctor [--fix] [--json] [--domain]
|
|
64
113
|
def doctor(argv)
|
|
65
|
-
opts = { fix: false, json: false }
|
|
114
|
+
opts = { fix: false, json: false, domain: false }
|
|
66
115
|
OptionParser.new do |o|
|
|
67
|
-
o.banner = "Usage: insika doctor [--fix] [--json]"
|
|
116
|
+
o.banner = "Usage: insika doctor [--fix] [--json] [--domain]"
|
|
68
117
|
o.on("--fix", "apply the safe autofixes, then re-diagnose") { opts[:fix] = true }
|
|
69
118
|
o.on("--json", "emit the report as JSON") { opts[:json] = true }
|
|
119
|
+
o.on("--domain", "append the domain inventory (declared personas, corpora, funnels, evidence)") { opts[:domain] = true }
|
|
70
120
|
end.parse!(argv)
|
|
71
121
|
|
|
72
122
|
doc = build_doctor
|
|
@@ -74,9 +124,12 @@ module Insika
|
|
|
74
124
|
|
|
75
125
|
if opts[:json]
|
|
76
126
|
require "json"
|
|
77
|
-
|
|
127
|
+
envelope = report.to_h
|
|
128
|
+
envelope["domain"] = doc.domain.to_h if opts[:domain]
|
|
129
|
+
puts JSON.pretty_generate(envelope)
|
|
78
130
|
else
|
|
79
131
|
puts report.to_s(color: $stdout.tty?)
|
|
132
|
+
puts doc.domain.to_s if opts[:domain]
|
|
80
133
|
end
|
|
81
134
|
exit(report.ok? ? 0 : 1)
|
|
82
135
|
end
|
|
@@ -258,8 +311,552 @@ def baseline_export(path)
|
|
|
258
311
|
puts "exported #{merged.size} case(s) from #{store.agents.size} agent baseline(s) to #{path}"
|
|
259
312
|
end
|
|
260
313
|
|
|
314
|
+
# insika evals:simulate --persona F --target <agent|url> [--staging|--eval-profile]
|
|
315
|
+
#
|
|
316
|
+
# Generates a SIMULATED conversation: a persona model (the cheap
|
|
317
|
+
# utility_model) plays a customer with the persona as its whole instruction — and
|
|
318
|
+
# one hard rule: it may ONLY assert what the persona's `knows` contains. The
|
|
319
|
+
# target agent answers over the same Transport the replay uses (HttpTransport
|
|
320
|
+
# against a running deployment, or a thin A2A transport when --target is a URL).
|
|
321
|
+
#
|
|
322
|
+
# SAFETY (fixed in the spec): a simulated conversation must not write for real,
|
|
323
|
+
# because `simulated: true` marks the transcript, it does not disarm the tools.
|
|
324
|
+
# So this CLI REQUIRES one of:
|
|
325
|
+
# --staging the operator declares the target is a staging deployment
|
|
326
|
+
# --eval-profile the target runs an eval profile (side-effect tools swapped
|
|
327
|
+
# for dry-runs) — a VERIFIED declaration, not a trust-me flag:
|
|
328
|
+
# the CLI derives the target's side-effect tools from the
|
|
329
|
+
# deployment's own registry (GET /v1/agents/:id) or the local
|
|
330
|
+
# store (INSIKA_DB), and requires --eval-tools to cover every
|
|
331
|
+
# one of them. An A2A target (no reachable registry) requires
|
|
332
|
+
# the explicit --eval-tools list.
|
|
333
|
+
# and refuses to run without one. Every run is marked `simulated: true`.
|
|
334
|
+
def evals_simulate(argv)
|
|
335
|
+
require "yaml"
|
|
336
|
+
require "json"
|
|
337
|
+
require "fileutils"
|
|
338
|
+
require "time"
|
|
339
|
+
opts = { base_url: ENV["INSIKA_URL"] || "http://localhost:9292",
|
|
340
|
+
token: ENV["OPENCLAW_GATEWAY_TOKEN"] || ENV["ADMIN_TOKEN"] || "local-demo",
|
|
341
|
+
timeout: 120, persona_model: ENV["EVAL_PERSONA_MODEL"],
|
|
342
|
+
judge_model: ENV["EVAL_JUDGE_MODEL"], judge_provider: ENV["EVAL_JUDGE_PROVIDER"] }
|
|
343
|
+
OptionParser.new do |o|
|
|
344
|
+
o.banner = "Usage: insika evals:simulate --persona F --target <agent|url> [options]"
|
|
345
|
+
o.on("--persona FILE", "the persona YAML — a case with a `persona:` key (required)") { |v| opts[:persona] = v }
|
|
346
|
+
o.on("--target T", "agent id (over --base-url) or an A2A http(s) URL (required)") { |v| opts[:target] = v }
|
|
347
|
+
o.on("--base-url URL", "deployment base URL (default #{opts[:base_url]})") { |v| opts[:base_url] = v }
|
|
348
|
+
o.on("--token T", "gateway token (default OPENCLAW_GATEWAY_TOKEN/local-demo)") { |v| opts[:token] = v }
|
|
349
|
+
o.on("--timeout N", Integer, "per-turn read timeout, and the A2A poll budget in seconds (default 120)") { |v| opts[:timeout] = v }
|
|
350
|
+
o.on("--staging", "the target is a staging deployment (side effects acceptable)") { opts[:staging] = true }
|
|
351
|
+
o.on("--eval-profile", "the target runs an eval profile (side-effect tools swapped for fakes)") { opts[:eval_profile] = true }
|
|
352
|
+
o.on("--eval-tools LIST", "comma-separated side-effect tools the eval profile swaps (verified against the derived set; required for an A2A target)") { |v| opts[:eval_tools] = v }
|
|
353
|
+
o.on("--persona-model MODEL", "the model playing the customer (default settings utility_model)") { |v| opts[:persona_model] = v }
|
|
354
|
+
o.on("--judge-model MODEL", "score the transcript's rubric with this model") { |v| opts[:judge_model] = v }
|
|
355
|
+
o.on("--judge-provider PROVIDER", "provider for the judge model") { |v| opts[:judge_provider] = v }
|
|
356
|
+
o.on("--pairwise", "compare the simulated transcript against the case's `reference:`") { opts[:pairwise] = true }
|
|
357
|
+
o.on("--conv ID", "conversation id (default: sim-<case id>-<random>, unique per run)") { |v| opts[:conv] = v }
|
|
358
|
+
o.on("--out FILE", "write the JSON report here") { |v| opts[:out] = v }
|
|
359
|
+
end.parse!(argv)
|
|
360
|
+
|
|
361
|
+
abort "insika evals:simulate: --persona is required" if opts[:persona].to_s.empty?
|
|
362
|
+
abort "insika evals:simulate: --target is required (<agent> or an A2A <url>)" if opts[:target].to_s.empty?
|
|
363
|
+
unless opts[:staging] || opts[:eval_profile]
|
|
364
|
+
abort "insika evals:simulate: a simulated run must not write for real — pass --staging " \
|
|
365
|
+
"(the target is a staging deployment) or --eval-profile (side-effect tools are swapped)"
|
|
366
|
+
end
|
|
367
|
+
|
|
368
|
+
golden = load_sim_persona(opts[:persona])
|
|
369
|
+
# --target names the agent actually simulated. An agent id overrides the
|
|
370
|
+
# persona file's `agent:` (the file is a reusable script); a URL means an A2A
|
|
371
|
+
# endpoint, where the agent id is irrelevant (the transport ignores it).
|
|
372
|
+
a2a_target = opts[:target].to_s.match?(%r{\Ahttps?://})
|
|
373
|
+
target_agent = a2a_target ? golden.agent : opts[:target].to_s
|
|
374
|
+
safety = build_sim_safety(opts, golden, target_agent, a2a_target)
|
|
375
|
+
|
|
376
|
+
settings = eval_settings_from_env
|
|
377
|
+
ask = build_persona_ask(opts, settings)
|
|
378
|
+
judge = build_sim_judge(opts, settings)
|
|
379
|
+
|
|
380
|
+
transport = build_sim_transport(opts)
|
|
381
|
+
simulator = Insika::Evals::Simulator.new(transport: transport, ask: ask, safety: safety)
|
|
382
|
+
# Unique per run: two runs sharing a conv share the deployment SESSION, so the
|
|
383
|
+
# second inherits the first's history — including across the A/B arms of a
|
|
384
|
+
# paired comparison, which it silently invalidates.
|
|
385
|
+
require "securerandom"
|
|
386
|
+
conv = opts[:conv] || "sim-#{golden.id}-#{SecureRandom.hex(4)}"
|
|
387
|
+
run = simulator.run(persona: golden.persona, agent: target_agent, conv: conv)
|
|
388
|
+
|
|
389
|
+
verdict = judge && judge.score_conversation(
|
|
390
|
+
rubric: golden.rubric, transcript: run.transcript, policy: golden.policy,
|
|
391
|
+
min_score: golden.min_score || Insika::Evals::Judge::DEFAULT_MIN_SCORE
|
|
392
|
+
)
|
|
393
|
+
|
|
394
|
+
pairwise = if opts[:pairwise] && golden.reference?
|
|
395
|
+
p = build_sim_pairwise(opts, settings)
|
|
396
|
+
if p
|
|
397
|
+
p.compare_texts(
|
|
398
|
+
ours: Insika::Evals::Pairwise.transcript_text(run.transcript),
|
|
399
|
+
theirs: Insika::Evals::Pairwise.reference_transcript(golden.reference_messages),
|
|
400
|
+
vs: golden.human_assisted? ? "human-assisted" : "agent"
|
|
401
|
+
)
|
|
402
|
+
end
|
|
403
|
+
end
|
|
404
|
+
|
|
405
|
+
at = Time.now.utc.iso8601
|
|
406
|
+
report = { "simulated" => true, "at" => at, "id" => golden.id, "agent" => golden.agent,
|
|
407
|
+
"conv" => conv, "stop" => run.stop.to_s, "turns" => run.turns, "error" => run.error,
|
|
408
|
+
"persona" => golden.persona.to_h, "transcript" => run.transcript }
|
|
409
|
+
if verdict
|
|
410
|
+
report["judge"] = { "score" => verdict.score, "pass" => verdict.pass, "reason" => verdict.reason }
|
|
411
|
+
# A split panel must be visible in the report (the whole point of a panel
|
|
412
|
+
# over a lone judge): one entry per judge model, in settings order.
|
|
413
|
+
report["judge"]["judges"] = verdict.judges if verdict.judges
|
|
414
|
+
end
|
|
415
|
+
report["pairwise"] = { "outcome" => pairwise.outcome, "vs" => pairwise.vs, "reason" => pairwise.reason } if pairwise
|
|
416
|
+
puts simulate_markdown(report, verdict, pairwise)
|
|
417
|
+
if opts[:out]
|
|
418
|
+
FileUtils.mkdir_p(File.dirname(File.expand_path(opts[:out])))
|
|
419
|
+
File.write(opts[:out], JSON.pretty_generate(report))
|
|
420
|
+
puts "report: #{opts[:out]}"
|
|
421
|
+
end
|
|
422
|
+
exit(run.stop == :error || (verdict && !verdict.pass) ? 1 : 0)
|
|
423
|
+
rescue Insika::Evals::PersonaLoader::InvalidPersona, Insika::Evals::GoldenLoader::InvalidGolden,
|
|
424
|
+
Insika::Evals::Simulator::UnsafeTarget => e
|
|
425
|
+
warn "insika evals:simulate: #{e.message}"
|
|
426
|
+
exit 2
|
|
427
|
+
end
|
|
428
|
+
|
|
429
|
+
# --persona: a case YAML carrying a `persona:` key (the alternative
|
|
430
|
+
# shape). A case without turns but with a persona is the only thing this CLI
|
|
431
|
+
# drives — a scripted case has nothing to simulate.
|
|
432
|
+
def load_sim_persona(path)
|
|
433
|
+
raw = YAML.safe_load(File.read(path), permitted_classes: [], aliases: false) || {}
|
|
434
|
+
golden = Insika::Evals::GoldenLoader.build(raw, source: path)
|
|
435
|
+
raise Insika::Evals::GoldenLoader::InvalidGolden, "#{path}: needs a `persona:` (a simulated case)" unless golden.simulated?
|
|
436
|
+
|
|
437
|
+
golden
|
|
438
|
+
rescue Errno::ENOENT => e
|
|
439
|
+
raise Insika::Evals::GoldenLoader::InvalidGolden, e.message
|
|
440
|
+
end
|
|
441
|
+
|
|
442
|
+
# The safety gate for a run. --staging is a straight declaration. --eval-profile
|
|
443
|
+
# is a VERIFIED one: the CLI derives the target's side-effect tools (see
|
|
444
|
+
# derive_side_effect_tools) and refuses unless the operator's --eval-tools names
|
|
445
|
+
# every one of them — an eval profile that leaves a side-effect tool unswapped is
|
|
446
|
+
# a lie, and a bare `--eval-profile` on a write-capable agent is exactly the
|
|
447
|
+
# trust-me flag the docs promise is not what happens.
|
|
448
|
+
def build_sim_safety(opts, golden, target_agent, a2a_target)
|
|
449
|
+
return Insika::Evals::Simulator::Safety.staging if opts[:staging]
|
|
450
|
+
|
|
451
|
+
declared = opts[:eval_tools].to_s.split(",").map(&:strip).reject(&:empty?)
|
|
452
|
+
derived = if a2a_target
|
|
453
|
+
nil # a remote A2A agent has no reachable registry — the list must be declared
|
|
454
|
+
else
|
|
455
|
+
derive_side_effect_tools(opts, target_agent)
|
|
456
|
+
end
|
|
457
|
+
|
|
458
|
+
if derived
|
|
459
|
+
uncovered = derived - declared
|
|
460
|
+
unless uncovered.empty?
|
|
461
|
+
abort "insika evals:simulate: --eval-profile, but the target's derived side-effect " \
|
|
462
|
+
"tool(s) (#{uncovered.join(', ')}) are not declared in --eval-tools — an eval " \
|
|
463
|
+
"profile must swap EVERY side-effect tool (or use --staging)"
|
|
464
|
+
end
|
|
465
|
+
elsif declared.empty?
|
|
466
|
+
abort "insika evals:simulate: cannot derive the target's side-effect tools (no " \
|
|
467
|
+
"/v1/agents/:id answer and no local store) — declare the swap list explicitly " \
|
|
468
|
+
"with --eval-tools, or use --staging"
|
|
469
|
+
end
|
|
470
|
+
|
|
471
|
+
Insika::Evals::Simulator::Safety.new(eval_profile: true, side_effect_tools: derived || [],
|
|
472
|
+
swapped_tools: declared)
|
|
473
|
+
end
|
|
474
|
+
|
|
475
|
+
# The target's side-effect tools — DERIVED, never hand-typed into the safety
|
|
476
|
+
# gate. -> [names] | nil when derivation is impossible. Local store first (the
|
|
477
|
+
# definitive answer when INSIKA_DB points at the deployment); otherwise the
|
|
478
|
+
# deployment's own registry over GET /v1/agents/:id (`side_effect_tools` — the
|
|
479
|
+
# server computes it from the same allowlist − deny resolution the capability
|
|
480
|
+
# view uses).
|
|
481
|
+
def derive_side_effect_tools(opts, agent)
|
|
482
|
+
local = local_side_effect_tools(agent)
|
|
483
|
+
return local unless local.nil?
|
|
484
|
+
|
|
485
|
+
capabilities = Insika::Evals::HttpCapabilities.new(base_url: opts[:base_url], token: opts[:token])
|
|
486
|
+
answer = capabilities.for(agent)
|
|
487
|
+
answer && !answer["side_effect_tools"].nil? ? Array(answer["side_effect_tools"]).map(&:to_s) : nil
|
|
488
|
+
end
|
|
489
|
+
|
|
490
|
+
# Derivation from the local store (INSIKA_DB set): the profile's reachable set
|
|
491
|
+
# intersected with the store-backed overlay registry's side-effect marks. nil
|
|
492
|
+
# when there is no store or the agent is not in it.
|
|
493
|
+
def local_side_effect_tools(agent)
|
|
494
|
+
db = Insika::EnvSchema.read("INSIKA_DB", ENV)
|
|
495
|
+
return nil if db.nil? || db.empty?
|
|
496
|
+
|
|
497
|
+
backend = Insika::Wiring::Graph.backend_from_env
|
|
498
|
+
config_store = Insika::ConfigStore.new(store: backend)
|
|
499
|
+
profile = Insika::StoredProfileSource.new(config_store: config_store).fetch(agent)
|
|
500
|
+
return nil if profile.nil?
|
|
501
|
+
|
|
502
|
+
registry = Insika::OverlayToolRegistry.new(
|
|
503
|
+
base: Insika::ToolRegistry.new, tool_store: Insika::ToolStore.new(config_store: config_store),
|
|
504
|
+
http: Insika::HttpClient.new
|
|
505
|
+
)
|
|
506
|
+
Insika::Evals::EvalProfile.side_effect_tools(profile, registry)
|
|
507
|
+
end
|
|
508
|
+
|
|
509
|
+
# One place for the CLI's provider keys. Every synchronous CLI command that
|
|
510
|
+
# builds its own extractor/miner/judge ask (no full app boot, so LlmConfigurator
|
|
511
|
+
# never ran) calls this first — a judge panel is only as multi-model as the keys
|
|
512
|
+
# this block wires: a settings panel pointing at OpenRouter models silently
|
|
513
|
+
# 401'd here while the engine (LlmConfigurator) knew the key all along.
|
|
514
|
+
def configure_cli_llm!
|
|
515
|
+
require "ruby_llm"
|
|
516
|
+
RubyLLM.configure do |c|
|
|
517
|
+
c.deepseek_api_key = ENV["DEEPSEEK_API_KEY"] if ENV["DEEPSEEK_API_KEY"]
|
|
518
|
+
c.openai_api_key = ENV["OPENAI_API_KEY"] if ENV["OPENAI_API_KEY"]
|
|
519
|
+
c.openai_api_base = ENV["OPENAI_API_BASE"] if ENV["OPENAI_API_BASE"]
|
|
520
|
+
c.openrouter_api_key = ENV["OPENROUTER_API_KEY"] if ENV["OPENROUTER_API_KEY"]
|
|
521
|
+
c.gemini_api_key = ENV["GEMINI_API_KEY"] if ENV["GEMINI_API_KEY"]
|
|
522
|
+
end
|
|
523
|
+
end
|
|
524
|
+
|
|
525
|
+
# The persona MODEL — the cheap side that plays the customer. Defaults to the
|
|
526
|
+
# platform `utility_model` (the same setting the judges and the moderator read);
|
|
527
|
+
# --persona-model overrides. Refuses when neither is set: never guess a model to
|
|
528
|
+
# spend money on (the proposer's rule, applied to the cheap side too).
|
|
529
|
+
def build_persona_ask(opts, settings)
|
|
530
|
+
model = opts[:persona_model] || settings["utility_model"]
|
|
531
|
+
abort "insika evals:simulate: no persona model — set the platform utility_model or pass --persona-model" if model.to_s.empty?
|
|
532
|
+
|
|
533
|
+
configure_cli_llm!
|
|
534
|
+
# ruby_llm_ask runs with assume_model_exists, which requires an explicit
|
|
535
|
+
# provider — a bare model id (e.g. a platform utility_model outside the
|
|
536
|
+
# RubyLLM registry) aborted every simulated run before this fallback.
|
|
537
|
+
provider = ENV["EVAL_PERSONA_PROVIDER"] || settings["default_provider"]
|
|
538
|
+
Insika::Evals::JudgePanel.ruby_llm_ask(model, provider)
|
|
539
|
+
end
|
|
540
|
+
|
|
541
|
+
def build_sim_judge(opts, settings)
|
|
542
|
+
models = if opts[:judge_model]
|
|
543
|
+
[{ "model" => opts[:judge_model], "provider" => opts[:judge_provider] }]
|
|
544
|
+
else
|
|
545
|
+
Array(settings["evals"] && settings["evals"]["judges"])
|
|
546
|
+
end
|
|
547
|
+
return nil if models.reject { |m| (m["model"] || m[:model]).to_s.empty? }.empty?
|
|
548
|
+
|
|
549
|
+
configure_cli_llm!
|
|
550
|
+
Insika::Evals::JudgePanel.build(settings["evals"] || {}, overrides: {
|
|
551
|
+
"judge_model" => opts[:judge_model], "judge_provider" => opts[:judge_provider]
|
|
552
|
+
})&.first
|
|
553
|
+
end
|
|
554
|
+
|
|
555
|
+
# The pairwise judge — the SAME configured panel, asked which conversation served
|
|
556
|
+
# the customer better. Same opt-in as a replay's --pairwise (2 calls per judge).
|
|
557
|
+
def build_sim_pairwise(opts, settings)
|
|
558
|
+
models = if opts[:judge_model]
|
|
559
|
+
[{ "model" => opts[:judge_model], "provider" => opts[:judge_provider] }]
|
|
560
|
+
else
|
|
561
|
+
Array(settings["evals"] && settings["evals"]["judges"])
|
|
562
|
+
end
|
|
563
|
+
return nil if models.reject { |m| (m["model"] || m[:model]).to_s.empty? }.empty?
|
|
564
|
+
|
|
565
|
+
configure_cli_llm!
|
|
566
|
+
Insika::Evals::JudgePanel.pairwise(settings["evals"] || {}, overrides: {
|
|
567
|
+
"judge_model" => opts[:judge_model], "judge_provider" => opts[:judge_provider]
|
|
568
|
+
})&.first
|
|
569
|
+
end
|
|
570
|
+
|
|
571
|
+
# --target <agent|url>: an agent id rides HttpTransport over the deployment's
|
|
572
|
+
# /v1/responses (the same surface the replay uses); a full http(s) URL is an A2A
|
|
573
|
+
# endpoint (an agent that only speaks A2A) driven by the thin A2A transport. The
|
|
574
|
+
# A2A client is built with a SYNCHRONOUS Net::HTTP adapter (the CLI has no
|
|
575
|
+
# reactor) — `post_json(url, body) -> Hash` is the client's whole contract.
|
|
576
|
+
#
|
|
577
|
+
# POLLING: the A2A client's default cadence (0.02s, 30 polls = ~0.6s total) is
|
|
578
|
+
# for in-process loopback. A remote agent takes SECONDS to finish a task, so the
|
|
579
|
+
# CLI polls once per second, bounded by --timeout (the same flag the HTTP
|
|
580
|
+
# transport's read timeout honors).
|
|
581
|
+
def build_sim_transport(opts)
|
|
582
|
+
target = opts[:target]
|
|
583
|
+
if target.to_s.match?(%r{\Ahttps?://})
|
|
584
|
+
require_relative "../lib/insika/server/a2a/client"
|
|
585
|
+
require_relative "../lib/insika/server/a2a/message"
|
|
586
|
+
require "net/http"
|
|
587
|
+
require "uri"
|
|
588
|
+
cadence = 1.0
|
|
589
|
+
client = Insika::Server::A2A::Client.new(
|
|
590
|
+
http: SyncPostJson.new,
|
|
591
|
+
poll_max: [opts[:timeout].to_i, 1].max,
|
|
592
|
+
sleeper: ->(_delay) { sleep(cadence) }
|
|
593
|
+
)
|
|
594
|
+
Insika::Evals::A2ATransport.new(client: client, url: target)
|
|
595
|
+
else
|
|
596
|
+
Insika::Evals::HttpTransport.new(base_url: opts[:base_url], token: opts[:token], timeout: opts[:timeout])
|
|
597
|
+
end
|
|
598
|
+
end
|
|
599
|
+
|
|
600
|
+
# A synchronous A2A JSON-RPC HTTP adapter for the CLI (no async reactor). The A2A
|
|
601
|
+
# client calls `post_json(url, body) -> Hash`; this answers it over Net::HTTP. An
|
|
602
|
+
# unreachable host or a non-JSON body becomes a clean `{ "error" => ... }` envelope —
|
|
603
|
+
# the A2A client turns that into a turn error, never a crash.
|
|
604
|
+
class SyncPostJson
|
|
605
|
+
def post_json(url, body)
|
|
606
|
+
uri = URI.parse(url)
|
|
607
|
+
req = Net::HTTP::Post.new(uri)
|
|
608
|
+
req["Content-Type"] = "application/json"
|
|
609
|
+
req["Accept"] = "application/json"
|
|
610
|
+
req.body = body.is_a?(String) ? body : JSON.generate(body)
|
|
611
|
+
res = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https",
|
|
612
|
+
open_timeout: 10, read_timeout: 120) { |http| http.request(req) }
|
|
613
|
+
JSON.parse(res.body.to_s)
|
|
614
|
+
rescue StandardError => e
|
|
615
|
+
{ "error" => { "code" => -32_600, "message" => "A2A call failed: #{e.class}: #{e.message}" } }
|
|
616
|
+
end
|
|
617
|
+
end
|
|
618
|
+
|
|
619
|
+
def eval_settings_from_env
|
|
620
|
+
db = Insika::EnvSchema.read("INSIKA_DB", ENV)
|
|
621
|
+
return {} if db.nil? || db.empty?
|
|
622
|
+
|
|
623
|
+
Insika::SettingsStore.new(config_store: Insika::ConfigStore.new(store: Insika::Stores::SQLite.new(path: db))).get
|
|
624
|
+
rescue StandardError
|
|
625
|
+
{}
|
|
626
|
+
end
|
|
627
|
+
|
|
628
|
+
def simulate_markdown(report, verdict, pairwise = nil)
|
|
629
|
+
lines = ["# Simulated run — #{report['id']} (#{report['agent']})", "",
|
|
630
|
+
"**simulated: true** · stop: #{report['stop']}#{" (turn #{report['turns']})" if report['turns']}" \
|
|
631
|
+
"#{" · error: #{report['error']}" if report['error']}", ""]
|
|
632
|
+
report["transcript"].each do |m|
|
|
633
|
+
role = m[:role].to_s == "user" ? "customer" : "assistant"
|
|
634
|
+
lines << "**#{role}**"
|
|
635
|
+
lines << m[:text].to_s.empty? ? " _#{m[:error] || 'no reply'}_" : " #{m[:text]}"
|
|
636
|
+
lines << " _tools: #{m[:tools].join(', ')}_" if m[:tools] && !m[:tools].empty?
|
|
637
|
+
end
|
|
638
|
+
lines << ""
|
|
639
|
+
if verdict
|
|
640
|
+
lines << "**judge: #{verdict.pass ? '✅' : '❌'} #{verdict.score}** — #{verdict.reason}"
|
|
641
|
+
else
|
|
642
|
+
lines << "_(no judge configured — transcript only)_"
|
|
643
|
+
end
|
|
644
|
+
if pairwise
|
|
645
|
+
mark = { "better" => "🟢", "comparable" => "🟡", "worse" => "🔴" }.fetch(pairwise.outcome, "·")
|
|
646
|
+
lines << "**vs incumbent (#{pairwise.vs}): #{mark} #{pairwise.outcome}** — #{pairwise.reason}"
|
|
647
|
+
end
|
|
648
|
+
"#{lines.join("\n")}\n"
|
|
649
|
+
end
|
|
650
|
+
|
|
651
|
+
# insika mcp list|add|remove|import|test|refresh
|
|
652
|
+
def mcp(argv)
|
|
653
|
+
sub = argv.shift
|
|
654
|
+
case sub
|
|
655
|
+
when "list" then mcp_list
|
|
656
|
+
when "add" then mcp_add(argv)
|
|
657
|
+
when "remove" then mcp_remove(argv)
|
|
658
|
+
when "import" then mcp_import(argv)
|
|
659
|
+
when "test" then mcp_run(argv, verb: "test", persist_label: nil)
|
|
660
|
+
when "refresh" then mcp_run(argv, verb: "refresh", persist_label: "cached")
|
|
661
|
+
else
|
|
662
|
+
warn "insika mcp: unknown subcommand '#{sub}' (list|add|remove|import|test|refresh)"
|
|
663
|
+
exit 2
|
|
664
|
+
end
|
|
665
|
+
rescue Insika::Error => e
|
|
666
|
+
warn "insika mcp: #{e.message}"
|
|
667
|
+
exit 2
|
|
668
|
+
end
|
|
669
|
+
|
|
670
|
+
def mcp_list
|
|
671
|
+
store = mcp_store
|
|
672
|
+
names = store.names
|
|
673
|
+
return puts("no MCP instances configured (insika mcp add / insika mcp import FILE.json)") if names.empty?
|
|
674
|
+
|
|
675
|
+
names.each do |name|
|
|
676
|
+
r = store.get(name)
|
|
677
|
+
status = r["enabled"] ? "enabled" : "disabled"
|
|
678
|
+
target = r["transport"] == "stdio" ? r["command"].to_s : r["url"].to_s
|
|
679
|
+
puts format("%-20s %-8s %-8s %-40s %d tool(s)", name, r["transport"], status, target, Array(r["tools_cache"]).size)
|
|
680
|
+
end
|
|
681
|
+
end
|
|
682
|
+
|
|
683
|
+
def mcp_add(argv)
|
|
684
|
+
opts = { args: [], headers: {}, env: {}, enabled: true }
|
|
685
|
+
OptionParser.new do |o|
|
|
686
|
+
o.banner = "Usage: insika mcp add --name NAME [--transport stdio|http|sse] [options]"
|
|
687
|
+
o.on("--name NAME", "instance name (required)") { |v| opts[:name] = v }
|
|
688
|
+
o.on("--transport T", "stdio|http|sse (default: stdio with --command, else http)") { |v| opts[:transport] = v }
|
|
689
|
+
o.on("--command CMD", "stdio: the executable") { |v| opts[:command] = v }
|
|
690
|
+
o.on("--arg ARG", "stdio: one argv entry (repeatable)") { |v| opts[:args] << v }
|
|
691
|
+
o.on("--url URL", "http/sse: the server URL") { |v| opts[:url] = v }
|
|
692
|
+
o.on("--header 'Name: value'", "http/sse: one header (repeatable)") do |v|
|
|
693
|
+
k, val = v.split(":", 2)
|
|
694
|
+
opts[:headers][k.to_s.strip] = val.to_s.strip
|
|
695
|
+
end
|
|
696
|
+
o.on("--env 'KEY=value'", "stdio: one env var (repeatable)") do |v|
|
|
697
|
+
k, val = v.split("=", 2)
|
|
698
|
+
opts[:env][k.to_s.strip] = val.to_s.strip
|
|
699
|
+
end
|
|
700
|
+
o.on("--description DESC", "free text") { |v| opts[:description] = v }
|
|
701
|
+
o.on("--disabled", "create it disabled") { opts[:enabled] = false }
|
|
702
|
+
end.parse!(argv)
|
|
703
|
+
raise Insika::ValidationError, "--name is required" if opts[:name].nil?
|
|
704
|
+
|
|
705
|
+
opts[:transport] ||= Insika::Coercion.presence(opts[:command]) ? "stdio" : "http"
|
|
706
|
+
record = mcp_store.upsert(opts)
|
|
707
|
+
puts "MCP instance '#{record['name']}' saved (#{record['transport']}, #{record['enabled'] ? 'enabled' : 'disabled'})"
|
|
708
|
+
end
|
|
709
|
+
|
|
710
|
+
def mcp_remove(argv)
|
|
711
|
+
name = argv.shift
|
|
712
|
+
raise Insika::ValidationError, "usage: insika mcp remove NAME" if name.nil?
|
|
713
|
+
|
|
714
|
+
existed = mcp_store.delete(name)
|
|
715
|
+
puts existed ? "MCP instance '#{name}' removed" : "MCP instance '#{name}' did not exist"
|
|
716
|
+
end
|
|
717
|
+
|
|
718
|
+
def mcp_import(argv)
|
|
719
|
+
path = argv.shift
|
|
720
|
+
raise Insika::ValidationError, "usage: insika mcp import FILE.json" if path.nil?
|
|
721
|
+
raise Insika::ValidationError, "no such file: #{path}" unless File.exist?(path)
|
|
722
|
+
|
|
723
|
+
records = Insika::McpJson.import(File.read(path), mcp_store: mcp_store)
|
|
724
|
+
puts "imported #{records.size} MCP instance(s) from #{path}"
|
|
725
|
+
records.each { |r| puts " #{r['name']} (#{r['transport']})" }
|
|
726
|
+
end
|
|
727
|
+
|
|
728
|
+
# Shared by `test`/`refresh` — both connect live and list tools
|
|
729
|
+
# (Insika::McpToolRegistry#refresh always writes tools_cache; `test` differs
|
|
730
|
+
# only in what it prints). Exits 1 (not 2) on a
|
|
731
|
+
# connection/transport failure — the instance is configured correctly, the
|
|
732
|
+
# SERVER is unreachable, which is a finding, not a usage error. Rescues
|
|
733
|
+
# StandardError, not just Insika::Error: the registry's docstring is
|
|
734
|
+
# explicit that a transport failure is the GEM's error class (e.g.
|
|
735
|
+
# ruby_llm-mcp's own AuthenticationRequiredError) and it is this caller's
|
|
736
|
+
# job to turn it into a clean one-liner instead of a raw backtrace.
|
|
737
|
+
def mcp_run(argv, verb:, persist_label:)
|
|
738
|
+
name = argv.shift
|
|
739
|
+
raise Insika::ValidationError, "usage: insika mcp #{verb} NAME" if name.nil?
|
|
740
|
+
|
|
741
|
+
tools = mcp_registry.refresh(name)
|
|
742
|
+
suffix = persist_label ? ", #{persist_label}" : ""
|
|
743
|
+
puts "MCP instance '#{name}': connected, #{tools.size} tool(s)#{suffix}"
|
|
744
|
+
tools.each { |t| puts " #{t['name']} — #{t['description']}" }
|
|
745
|
+
rescue StandardError => e
|
|
746
|
+
warn "MCP instance '#{name}': #{e.message}"
|
|
747
|
+
exit 1
|
|
748
|
+
end
|
|
749
|
+
|
|
750
|
+
# insika new --list | insika new <template> [dir]
|
|
751
|
+
def new_from_template(argv)
|
|
752
|
+
if argv.first == "--list"
|
|
753
|
+
new_list
|
|
754
|
+
else
|
|
755
|
+
new_copy(argv)
|
|
756
|
+
end
|
|
757
|
+
rescue Insika::Error => e
|
|
758
|
+
warn "insika new: #{e.message}"
|
|
759
|
+
exit 2
|
|
760
|
+
end
|
|
761
|
+
|
|
762
|
+
def new_list
|
|
763
|
+
entries = Insika::Templates.all
|
|
764
|
+
return puts("no templates available") if entries.empty?
|
|
765
|
+
|
|
766
|
+
entries.each { |t| puts format("%-18s %-10s %s", t.name, t.trail, t.description) }
|
|
767
|
+
end
|
|
768
|
+
|
|
769
|
+
# Copies the template's directory verbatim (agent.rb + README.md) — no
|
|
770
|
+
# rewriting, the generated script IS the editing surface. Prints the
|
|
771
|
+
# exact run line, including any env the template needs SET (not just
|
|
772
|
+
# available as an override, like MCP_URL's keyless default) — the
|
|
773
|
+
# EgressGuard/stdio-gate failure is silent otherwise, same discipline as
|
|
774
|
+
# the `mcp add`/data-tool egress hints elsewhere in this CLI.
|
|
775
|
+
def new_copy(argv)
|
|
776
|
+
name = argv.shift
|
|
777
|
+
raise Insika::ValidationError, "usage: insika new <template> [dir] (see insika new --list)" if name.nil?
|
|
778
|
+
|
|
779
|
+
entry = Insika::Templates.read(name)
|
|
780
|
+
dest = argv.shift || "./#{name}"
|
|
781
|
+
raise Insika::ValidationError, "already exists: #{dest}" if File.exist?(dest)
|
|
782
|
+
|
|
783
|
+
FileUtils.mkdir_p(dest)
|
|
784
|
+
FileUtils.cp(Insika::Templates.agent_path(name), File.join(dest, "agent.rb"))
|
|
785
|
+
readme = Insika::Templates.readme_path(name)
|
|
786
|
+
FileUtils.cp(readme, File.join(dest, "README.md")) if File.file?(readme)
|
|
787
|
+
|
|
788
|
+
puts "Created #{dest}/ (#{entry.title})"
|
|
789
|
+
puts
|
|
790
|
+
puts "Run it:"
|
|
791
|
+
env_flags = entry.env.map { |e| "#{e}=1" }.join(" ")
|
|
792
|
+
env_flags = "#{env_flags} " unless env_flags.empty?
|
|
793
|
+
puts " #{env_flags}DEEPSEEK_API_KEY=sk-... ruby #{dest}/agent.rb"
|
|
794
|
+
puts "\nRequires: #{entry.requires}" if entry.requires
|
|
795
|
+
end
|
|
796
|
+
|
|
797
|
+
# insika demo:seed [--force]
|
|
798
|
+
#
|
|
799
|
+
# Populates the "demo-store" agent with enough data to see every Studio
|
|
800
|
+
# loop working at once: a funnel with a frozen baseline, follow-ups in
|
|
801
|
+
# all four states, refinement runs across the lifecycle, pending +
|
|
802
|
+
# resolved approvals, distillation proposals/facts, and a golden set
|
|
803
|
+
# with a baseline (Insika::Demo::Seeder — the same object the Studio's
|
|
804
|
+
# "Seed demo data" button dispatches). Safe against a live volume: it
|
|
805
|
+
# only ever touches the demo agent's own records, and is a no-op on a
|
|
806
|
+
# second run unless --force asks for another batch.
|
|
807
|
+
def demo_seed(argv)
|
|
808
|
+
opts = { force: false }
|
|
809
|
+
OptionParser.new do |o|
|
|
810
|
+
o.banner = "Usage: insika demo:seed [--force]"
|
|
811
|
+
o.on("--force", "seed again even if the demo agent already exists") { opts[:force] = true }
|
|
812
|
+
end.parse!(argv)
|
|
813
|
+
|
|
814
|
+
backend = Insika::Wiring::Graph.backend_from_env
|
|
815
|
+
config_store = Insika::ConfigStore.new(store: backend)
|
|
816
|
+
profiles = Insika::StoredProfileSource.new(config_store: config_store)
|
|
817
|
+
seeder = Insika::Demo::Seeder.new(
|
|
818
|
+
profiles: profiles, store: backend,
|
|
819
|
+
session_store: Insika::SessionStore.new(store: backend),
|
|
820
|
+
task_store: Insika::TaskStore.new(store: backend),
|
|
821
|
+
outcome_store: Insika::OutcomeStore.new(store: backend),
|
|
822
|
+
funnel_store: Insika::FunnelStore.new(store: backend),
|
|
823
|
+
followup_store: Insika::FollowupStore.new(store: backend),
|
|
824
|
+
refinement_store: Insika::RefinementStore.new(store: backend),
|
|
825
|
+
pending_action_store: Insika::PendingActionStore.new(store: backend),
|
|
826
|
+
proposal_store: Insika::ProposalStore.new(store: backend),
|
|
827
|
+
memory_store: Insika::MemoryStore.new(store: backend),
|
|
828
|
+
golden_store: Insika::GoldenStore.new(config_store: config_store),
|
|
829
|
+
baseline_store: Insika::BaselineStore.new(config_store: config_store),
|
|
830
|
+
event_stream: Insika::EventStream.new
|
|
831
|
+
)
|
|
832
|
+
result = seeder.seed!(force: opts[:force])
|
|
833
|
+
|
|
834
|
+
unless result[:seeded]
|
|
835
|
+
warn "insika demo:seed: already seeded (#{result[:reason]}) — pass --force to seed again."
|
|
836
|
+
return
|
|
837
|
+
end
|
|
838
|
+
|
|
839
|
+
puts "seeded demo agent '#{result[:agent]}':"
|
|
840
|
+
result[:counts].each { |area, n| puts " #{area}: #{n}" }
|
|
841
|
+
puts "\nOpen /studio/funnel?agent=#{result[:agent]} (also /studio/followups, " \
|
|
842
|
+
"/studio/refinement, /studio/approvals, /studio/harvest, /studio/evals) in " \
|
|
843
|
+
"whichever Studio instance points at this INSIKA_DB."
|
|
844
|
+
rescue Insika::Error => e
|
|
845
|
+
warn "insika demo:seed: #{e.message}"
|
|
846
|
+
exit 2
|
|
847
|
+
end
|
|
848
|
+
|
|
261
849
|
# -- helpers -------------------------------------------------------
|
|
262
850
|
|
|
851
|
+
def mcp_store
|
|
852
|
+
backend = Insika::Wiring::Graph.backend_from_env
|
|
853
|
+
Insika::McpStore.new(config_store: Insika::ConfigStore.new(store: backend))
|
|
854
|
+
end
|
|
855
|
+
|
|
856
|
+
def mcp_registry
|
|
857
|
+
Insika::McpToolRegistry.new(mcp_store: mcp_store)
|
|
858
|
+
end
|
|
859
|
+
|
|
263
860
|
def baseline_store
|
|
264
861
|
backend = Insika::Wiring::Graph.backend_from_env
|
|
265
862
|
Insika::BaselineStore.new(config_store: Insika::ConfigStore.new(store: backend))
|
|
@@ -296,6 +893,250 @@ end
|
|
|
296
893
|
}.compact, transport: :cli))
|
|
297
894
|
end
|
|
298
895
|
|
|
896
|
+
# insika harvest --agent ID [--last-sessions N] [--since ISO8601] [--full] [--json]
|
|
897
|
+
#
|
|
898
|
+
# mines one window of the agent's finished traffic and records
|
|
899
|
+
# the candidates. Writes nothing but the run + candidates + markers (D2's
|
|
900
|
+
# fork discipline); a cron or the operator's own habit runs it.
|
|
901
|
+
def harvest(argv)
|
|
902
|
+
opts = { json: false, full: false }
|
|
903
|
+
OptionParser.new do |o|
|
|
904
|
+
o.banner = "Usage: insika harvest --agent ID [options]"
|
|
905
|
+
o.on("--agent ID", "agent whose traffic to mine (required)") { |v| opts[:agent] = v }
|
|
906
|
+
o.on("--last-sessions N", Integer, "mine the N most recent conversations") { |v| opts[:last_sessions] = v }
|
|
907
|
+
o.on("--since ISO8601", "mine only turns from this instant on") { |v| opts[:since] = v }
|
|
908
|
+
o.on("--full", "ignore the mined markers and re-read the window") { opts[:full] = true }
|
|
909
|
+
o.on("--json", "emit the summary as JSON") { opts[:json] = true }
|
|
910
|
+
end.parse!(argv)
|
|
911
|
+
|
|
912
|
+
abort "insika harvest: --agent is required" if Insika::Coercion.blank?(opts[:agent])
|
|
913
|
+
|
|
914
|
+
result = begin
|
|
915
|
+
run_harvest(opts)
|
|
916
|
+
rescue Insika::Error, ArgumentError => e
|
|
917
|
+
warn "insika harvest: #{e.message}"
|
|
918
|
+
exit 2
|
|
919
|
+
end
|
|
920
|
+
if opts[:json]
|
|
921
|
+
require "json"
|
|
922
|
+
puts JSON.pretty_generate(result)
|
|
923
|
+
else
|
|
924
|
+
puts harvest_report(result)
|
|
925
|
+
end
|
|
926
|
+
exit(1) if result[:mined] && result[:run_id] && run_failed?(result)
|
|
927
|
+
end
|
|
928
|
+
|
|
929
|
+
# insika harvest:negative import --agent ID --file F
|
|
930
|
+
#
|
|
931
|
+
# the seed path: reads the VERSIONED rules file (parse! — loud), and
|
|
932
|
+
# writes the rules into the profile's harvest.negative_list via UpdateAgent
|
|
933
|
+
# (the operative, hot-editable list the engine applies).
|
|
934
|
+
def harvest_negative_import(argv)
|
|
935
|
+
opts = {}
|
|
936
|
+
OptionParser.new do |o|
|
|
937
|
+
o.banner = "Usage: insika harvest:negative import --agent ID --file F"
|
|
938
|
+
o.on("--agent ID", "agent whose profile receives the rules (required)") { |v| opts[:agent] = v }
|
|
939
|
+
o.on("--file F", "the negative-list rules file (required)") { |v| opts[:file] = v }
|
|
940
|
+
end.parse!(argv)
|
|
941
|
+
|
|
942
|
+
abort "insika harvest:negative import: --agent is required" if Insika::Coercion.blank?(opts[:agent])
|
|
943
|
+
abort "insika harvest:negative import: --file is required" if Insika::Coercion.blank?(opts[:file])
|
|
944
|
+
|
|
945
|
+
rules = begin
|
|
946
|
+
list = Insika::Harvest::NegativeList.parse!(File.read(opts[:file]))
|
|
947
|
+
list.rules.map { |r| { "rule" => r.rule, "pattern" => r.pattern, "note" => r.note } }
|
|
948
|
+
rescue Insika::Error, Errno::ENOENT => e
|
|
949
|
+
warn "insika harvest:negative import: #{e.message}"
|
|
950
|
+
exit 2
|
|
951
|
+
end
|
|
952
|
+
|
|
953
|
+
backend = Insika::Wiring::Graph.backend_from_env
|
|
954
|
+
config_store = Insika::ConfigStore.new(store: backend)
|
|
955
|
+
profiles = Insika::StoredProfileSource.new(config_store: config_store)
|
|
956
|
+
profile = profiles.fetch(opts[:agent]) ||
|
|
957
|
+
(abort "insika harvest:negative import: agent '#{opts[:agent]}' not configured")
|
|
958
|
+
handler = Insika::Commands::UpdateAgent.new(profile_source: profiles,
|
|
959
|
+
event_stream: Insika::EventStream.new)
|
|
960
|
+
merged = (profile.harvest || {}).merge("negative_list" => rules)
|
|
961
|
+
handler.call(Insika::Command.build(:update_agent,
|
|
962
|
+
{ id: opts[:agent], harvest: merged }, transport: :cli))
|
|
963
|
+
puts "imported #{rules.size} rule(s) into #{opts[:agent]}.harvest.negative_list"
|
|
964
|
+
end
|
|
965
|
+
|
|
966
|
+
# insika harvest:criterion check --file F
|
|
967
|
+
#
|
|
968
|
+
# strict-loads the frozen conversion criterion — a missing key, an unknown
|
|
969
|
+
# key or an absent file exits non-zero. The hook before any promotion is
|
|
970
|
+
# allowed.
|
|
971
|
+
def harvest_criterion_check(argv)
|
|
972
|
+
opts = {}
|
|
973
|
+
OptionParser.new do |o|
|
|
974
|
+
o.banner = "Usage: insika harvest:criterion check --file F"
|
|
975
|
+
o.on("--file F", "the frozen criterion (required)") { |v| opts[:file] = v }
|
|
976
|
+
end.parse!(argv)
|
|
977
|
+
|
|
978
|
+
abort "insika harvest:criterion check: --file is required" if Insika::Coercion.blank?(opts[:file])
|
|
979
|
+
|
|
980
|
+
criterion = Insika::Harvest::Criterion.load(opts[:file])
|
|
981
|
+
puts "criterion ok: #{criterion.rule.metric} / #{criterion.rule.window} " \
|
|
982
|
+
"threshold #{criterion.rule.threshold} min_span #{criterion.rule.min_span} " \
|
|
983
|
+
"(#{criterion.sha})"
|
|
984
|
+
0
|
|
985
|
+
rescue Insika::Error => e
|
|
986
|
+
warn "insika harvest:criterion check: #{e.message}"
|
|
987
|
+
exit 1
|
|
988
|
+
end
|
|
989
|
+
|
|
990
|
+
# insika knowledge:backfill --agent ID [--since ISO8601] [--json]
|
|
991
|
+
#
|
|
992
|
+
# the recovery path (RFC's "re-scan, not a queue"): replays the agent's
|
|
993
|
+
# stored sessions through the SAME extractor a live turn's terminal hook
|
|
994
|
+
# uses. Writes nothing but the concepts themselves.
|
|
995
|
+
def knowledge_backfill(argv)
|
|
996
|
+
opts = { json: false }
|
|
997
|
+
OptionParser.new do |o|
|
|
998
|
+
o.banner = "Usage: insika knowledge:backfill --agent ID [options]"
|
|
999
|
+
o.on("--agent ID", "agent whose sessions to replay (required)") { |v| opts[:agent] = v }
|
|
1000
|
+
o.on("--since ISO8601", "replay only sessions with turns from this instant on") { |v| opts[:since] = v }
|
|
1001
|
+
o.on("--json", "emit the summary as JSON") { opts[:json] = true }
|
|
1002
|
+
end.parse!(argv)
|
|
1003
|
+
|
|
1004
|
+
abort "insika knowledge:backfill: --agent is required" if Insika::Coercion.blank?(opts[:agent])
|
|
1005
|
+
|
|
1006
|
+
result = begin
|
|
1007
|
+
run_knowledge_backfill(opts)
|
|
1008
|
+
rescue Insika::Error, ArgumentError => e
|
|
1009
|
+
warn "insika knowledge:backfill: #{e.message}"
|
|
1010
|
+
exit 2
|
|
1011
|
+
end
|
|
1012
|
+
if opts[:json]
|
|
1013
|
+
require "json"
|
|
1014
|
+
puts JSON.pretty_generate(result)
|
|
1015
|
+
else
|
|
1016
|
+
puts knowledge_backfill_report(result)
|
|
1017
|
+
end
|
|
1018
|
+
end
|
|
1019
|
+
|
|
1020
|
+
# Builds the handler over the real durable backend — same "no app boot"
|
|
1021
|
+
# rule as `doctor`/`harvest`: no DEEPSEEK, no seeding, no reactor.
|
|
1022
|
+
def run_knowledge_backfill(opts)
|
|
1023
|
+
configure_cli_llm!
|
|
1024
|
+
backend = Insika::Wiring::Graph.backend_from_env
|
|
1025
|
+
config_store = Insika::ConfigStore.new(store: backend)
|
|
1026
|
+
profiles = Insika::StoredProfileSource.new(config_store: config_store)
|
|
1027
|
+
handler = Insika::Commands::BackfillKnowledge.new(
|
|
1028
|
+
profiles: profiles,
|
|
1029
|
+
knowledge_store: Insika::KnowledgeStore.new(store: backend),
|
|
1030
|
+
session_store: Insika::SessionStore.new(store: backend),
|
|
1031
|
+
task_store: Insika::TaskStore.new(store: backend),
|
|
1032
|
+
settings_store: Insika::SettingsStore.new(config_store: config_store),
|
|
1033
|
+
event_stream: Insika::EventStream.new
|
|
1034
|
+
)
|
|
1035
|
+
handler.call(Insika::Command.build(:backfill_knowledge,
|
|
1036
|
+
{ agent: opts[:agent], since: opts[:since] }.compact, transport: :cli))
|
|
1037
|
+
end
|
|
1038
|
+
|
|
1039
|
+
def knowledge_backfill_report(result)
|
|
1040
|
+
return "knowledge:backfill: skipped (#{result[:skipped]})" unless result[:backfilled]
|
|
1041
|
+
|
|
1042
|
+
head = "knowledge:backfill: #{result[:concepts]} concept(s) learned from #{result[:sessions]} session(s)"
|
|
1043
|
+
head += " — #{result[:conflicts]} conflict(s), needs a human" if result[:conflicts].to_i.positive?
|
|
1044
|
+
dropped = result[:dropped].reject { |_k, v| v.to_i.zero? }
|
|
1045
|
+
dropped.empty? ? head : "#{head} — dropped: #{dropped.map { |k, v| "#{k}: #{v}" }.join(', ')}"
|
|
1046
|
+
end
|
|
1047
|
+
|
|
1048
|
+
# insika knowledge:export --agent ID [--out DIR] [--tenant T]
|
|
1049
|
+
#
|
|
1050
|
+
# writes one <name>.md per concept — the storage format IS the export
|
|
1051
|
+
# format, so this is a dump, not a converter, for okf-gem/graphify to
|
|
1052
|
+
# read directly. Safe to re-run: unlike evals:export's YAML.dump, there
|
|
1053
|
+
# is nothing lossy here to guard with --force.
|
|
1054
|
+
KNOWLEDGE_EXPORT_FORMATS = %w[md graphml].freeze
|
|
1055
|
+
|
|
1056
|
+
def knowledge_export(argv)
|
|
1057
|
+
opts = { out: ".", format: "md" }
|
|
1058
|
+
OptionParser.new do |o|
|
|
1059
|
+
o.banner = "Usage: insika knowledge:export --agent ID [options]"
|
|
1060
|
+
o.on("--agent ID", "agent whose concepts to export (required)") { |v| opts[:agent] = v }
|
|
1061
|
+
o.on("--out DIR", "destination directory (default: current directory)") { |v| opts[:out] = v }
|
|
1062
|
+
o.on("--tenant T", "explicit tenant scope (default: none)") { |v| opts[:tenant] = v }
|
|
1063
|
+
o.on("--format FORMAT", "md (default, one file per concept) | graphml (one graph file)") { |v| opts[:format] = v }
|
|
1064
|
+
end.parse!(argv)
|
|
1065
|
+
|
|
1066
|
+
abort "insika knowledge:export: --agent is required" if Insika::Coercion.blank?(opts[:agent])
|
|
1067
|
+
unless KNOWLEDGE_EXPORT_FORMATS.include?(opts[:format])
|
|
1068
|
+
abort "insika knowledge:export: --format must be #{KNOWLEDGE_EXPORT_FORMATS.join(' or ')} (got #{opts[:format].inspect})"
|
|
1069
|
+
end
|
|
1070
|
+
|
|
1071
|
+
backend = Insika::Wiring::Graph.backend_from_env
|
|
1072
|
+
store = Insika::KnowledgeStore.new(store: backend)
|
|
1073
|
+
tenant = Insika::Coercion.presence(opts[:tenant])
|
|
1074
|
+
|
|
1075
|
+
if opts[:format] == "graphml"
|
|
1076
|
+
FileUtils.mkdir_p(opts[:out])
|
|
1077
|
+
path = File.join(opts[:out], "knowledge.graphml")
|
|
1078
|
+
File.write(path, store.export_graphml(opts[:agent], tenant: tenant))
|
|
1079
|
+
puts "exported the concept graph to #{path}"
|
|
1080
|
+
else
|
|
1081
|
+
paths = store.export_dir(opts[:agent], opts[:out], tenant: tenant)
|
|
1082
|
+
puts "exported #{paths.size} concept(s) to #{opts[:out]}"
|
|
1083
|
+
end
|
|
1084
|
+
end
|
|
1085
|
+
|
|
1086
|
+
def run_failed?(result)
|
|
1087
|
+
backend = Insika::Wiring::Graph.backend_from_env
|
|
1088
|
+
run = Insika::HarvestStore.new(store: backend).find_run(result[:run_id])
|
|
1089
|
+
run && run.status == "failed"
|
|
1090
|
+
end
|
|
1091
|
+
|
|
1092
|
+
# Builds the handler over the real durable backend — same "no app boot" rule as
|
|
1093
|
+
# `doctor`: no DEEPSEEK, no seeding, no reactor.
|
|
1094
|
+
def run_harvest(opts)
|
|
1095
|
+
configure_cli_llm!
|
|
1096
|
+
backend = Insika::Wiring::Graph.backend_from_env
|
|
1097
|
+
config_store = Insika::ConfigStore.new(store: backend)
|
|
1098
|
+
profiles = Insika::StoredProfileSource.new(config_store: config_store)
|
|
1099
|
+
handler = Insika::Commands::RunHarvest.new(
|
|
1100
|
+
profiles: profiles,
|
|
1101
|
+
harvest_store: Insika::HarvestStore.new(store: backend),
|
|
1102
|
+
session_store: Insika::SessionStore.new(store: backend),
|
|
1103
|
+
task_store: Insika::TaskStore.new(store: backend),
|
|
1104
|
+
skill_store: Insika::SkillStore.new(config_store: config_store),
|
|
1105
|
+
settings_store: Insika::SettingsStore.new(config_store: config_store),
|
|
1106
|
+
negative_list: begin
|
|
1107
|
+
path = Insika::EnvSchema.read("INSIKA_HARVEST_NEGATIVE")
|
|
1108
|
+
path && Insika::Harvest::NegativeList.parse(File.read(path))
|
|
1109
|
+
rescue Errno::ENOENT
|
|
1110
|
+
nil
|
|
1111
|
+
end,
|
|
1112
|
+
miner_factory: nil,
|
|
1113
|
+
event_stream: Insika::EventStream.new
|
|
1114
|
+
)
|
|
1115
|
+
handler.call(Insika::Command.build(:run_harvest, {
|
|
1116
|
+
agent: opts[:agent], since: opts[:since],
|
|
1117
|
+
last_sessions: opts[:last_sessions], full: opts[:full]
|
|
1118
|
+
}.compact, transport: :cli))
|
|
1119
|
+
end
|
|
1120
|
+
|
|
1121
|
+
def harvest_report(result)
|
|
1122
|
+
head = if result[:mined]
|
|
1123
|
+
"harvest: #{result[:candidates]} candidate(s) — #{harvest_rejected(result[:rejected])}"
|
|
1124
|
+
else
|
|
1125
|
+
"harvest: skipped (#{result[:skipped]})"
|
|
1126
|
+
end
|
|
1127
|
+
if result[:cost] && result[:cost]["spent"].to_i.positive?
|
|
1128
|
+
head += " — #{result[:cost]['spent']} tokens" \
|
|
1129
|
+
"#{result[:cost]['cached'].to_i.positive? ? " (#{result[:cost]['cached']} cached)" : ''}"
|
|
1130
|
+
end
|
|
1131
|
+
head
|
|
1132
|
+
end
|
|
1133
|
+
|
|
1134
|
+
def harvest_rejected(rejected)
|
|
1135
|
+
return "nothing rejected" if rejected.nil? || rejected.values.all?(&:zero?)
|
|
1136
|
+
|
|
1137
|
+
rejected.reject { |_k, v| v.to_i.zero? }.map { |k, v| "#{k}: #{v}" }.join(", ")
|
|
1138
|
+
end
|
|
1139
|
+
|
|
299
1140
|
INDENT = " " * 23 # aligns the provenance line under the finding's title
|
|
300
1141
|
|
|
301
1142
|
def refine_report(run)
|
|
@@ -337,10 +1178,56 @@ end
|
|
|
337
1178
|
# a skill and the agents holding it, so neither half alone is enough. The
|
|
338
1179
|
# catalog brings the DISK seeds into the sweep (deploy/skills in a deployment,
|
|
339
1180
|
# skills/ in the minimal wiring — whichever exists here); without it a stale
|
|
340
|
-
# `eager:` or a
|
|
1181
|
+
# `eager:` or a store-specific phrase sitting in a shared seed pack is invisible.
|
|
341
1182
|
skill_store: Insika::SkillStore.new(config_store: config_store),
|
|
342
1183
|
skill_catalog: Insika::SkillCatalog.new(skill_roots),
|
|
343
1184
|
profile_source: Insika::StoredProfileSource.new(config_store: config_store),
|
|
1185
|
+
# the shadow-parity check reads the pair store over the same
|
|
1186
|
+
# runtime backend (pairs hold raw customer text — the check reports their
|
|
1187
|
+
# age and the criterion's loadability before boot does).
|
|
1188
|
+
shadow_pair_store: Insika::ShadowPairStore.new(store: backend),
|
|
1189
|
+
# the cache-layers check — the BUILTIN set as classes (the
|
|
1190
|
+
# check reads the layer declaration off the class, no instances needed).
|
|
1191
|
+
context_providers: [
|
|
1192
|
+
Insika::Context::Providers::Request, Insika::Context::Providers::Prompt,
|
|
1193
|
+
Insika::Context::Providers::Skill, Insika::Context::Providers::SkillTrigger,
|
|
1194
|
+
Insika::Context::Providers::ToolSearch, Insika::Context::Providers::Memory,
|
|
1195
|
+
Insika::Context::Providers::Session
|
|
1196
|
+
],
|
|
1197
|
+
# the memory-scopes check reads the cells over the same
|
|
1198
|
+
# runtime backend; `agent_ids` excuses the agent-memory tab's cells
|
|
1199
|
+
# (a bare cell named like an agent is a profile, not a customer —
|
|
1200
|
+
# the profile source is right here, so the CLI names them).
|
|
1201
|
+
memory_store: Insika::MemoryStore.new(store: backend),
|
|
1202
|
+
agent_ids: Insika::StoredProfileSource.new(config_store: config_store).all.map(&:id),
|
|
1203
|
+
# the outcome-funnel check reads the fold's cells and the
|
|
1204
|
+
# outcomes over the same runtime backend — a profile with outcomes but
|
|
1205
|
+
# no funnel warns, and a frozen baseline is confirmed before it is
|
|
1206
|
+
# relied on.
|
|
1207
|
+
funnel_store: Insika::FunnelStore.new(store: backend),
|
|
1208
|
+
outcome_store: Insika::OutcomeStore.new(store: backend),
|
|
1209
|
+
# the follow-up check reads the schedule records and the
|
|
1210
|
+
# contact cells over the same runtime backend (nil collaborators = the
|
|
1211
|
+
# CLI reports declarations only).
|
|
1212
|
+
followup_store: Insika::FollowupStore.new(store: backend),
|
|
1213
|
+
contact_store: Insika::ContactStore.new(store: backend),
|
|
1214
|
+
# the schedules check reads the rows over the same runtime
|
|
1215
|
+
# backend (nil = the CLI reports declarations only).
|
|
1216
|
+
schedule_store: Insika::ScheduleStore.new(store: backend),
|
|
1217
|
+
# the distillation check reads the proposal store over
|
|
1218
|
+
# the same runtime backend (nil = the CLI reports declarations only).
|
|
1219
|
+
proposal_store: Insika::ProposalStore.new(store: backend),
|
|
1220
|
+
# the harvest check reads the candidates over the same
|
|
1221
|
+
# runtime backend + the frozen criterion (nil = declarations only).
|
|
1222
|
+
harvest_store: Insika::HarvestStore.new(store: backend),
|
|
1223
|
+
harvest_criterion: begin
|
|
1224
|
+
path = Insika::EnvSchema.read("INSIKA_HARVEST_CRITERION")
|
|
1225
|
+
path && Insika::Harvest::Criterion.load(path)
|
|
1226
|
+
rescue Insika::ConfigError, Insika::ValidationError
|
|
1227
|
+
nil
|
|
1228
|
+
end,
|
|
1229
|
+
# the mcp check reads the instances over the same runtime backend.
|
|
1230
|
+
mcp_store: Insika::McpStore.new(config_store: config_store),
|
|
344
1231
|
backend: backend
|
|
345
1232
|
)
|
|
346
1233
|
end
|