@voybio/ace-swarm 2.4.4 → 2.4.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +69 -0
- package/README.md +72 -532
- package/assets/eval-battery/battery.json +85 -0
- package/assets/eval-battery/fixtures/edit-target.js +3 -0
- package/assets/scripts/render-mcp-configs.sh +12 -0
- package/dist/ace-context.d.ts +18 -0
- package/dist/ace-context.js +161 -16
- package/dist/ace-server-instructions.js +41 -12
- package/dist/bridge-trace.d.ts +17 -0
- package/dist/bridge-trace.js +23 -0
- package/dist/cli.d.ts +9 -1
- package/dist/cli.js +623 -11
- package/dist/discovery-runtime-wrappers.d.ts +16 -1
- package/dist/discovery-runtime-wrappers.js +65 -9
- package/dist/eval-harness.d.ts +110 -0
- package/dist/eval-harness.js +254 -0
- package/dist/experience-reflector.d.ts +13 -0
- package/dist/experience-reflector.js +62 -0
- package/dist/goal-engine.d.ts +155 -0
- package/dist/goal-engine.js +795 -0
- package/dist/helpers/store-resolution.js +40 -39
- package/dist/hermes/session-manager.d.ts +10 -0
- package/dist/hermes/session-manager.js +130 -27
- package/dist/job-scheduler.d.ts +58 -0
- package/dist/job-scheduler.js +202 -2
- package/dist/job-worker.d.ts +87 -0
- package/dist/job-worker.js +182 -0
- package/dist/local-model-policy.d.ts +8 -2
- package/dist/local-model-policy.js +31 -17
- package/dist/local-model-runtime.d.ts +6 -0
- package/dist/local-model-runtime.js +130 -2
- package/dist/measured-classing.d.ts +66 -0
- package/dist/measured-classing.js +165 -0
- package/dist/model-bridge.d.ts +45 -0
- package/dist/model-bridge.js +502 -44
- package/dist/orchestrator-supervisor.js +1 -1
- package/dist/plan-proposal.d.ts +7 -0
- package/dist/plan-proposal.js +74 -4
- package/dist/playbook-adaptation.d.ts +37 -0
- package/dist/playbook-adaptation.js +159 -0
- package/dist/playbook-curator.d.ts +16 -0
- package/dist/playbook-curator.js +106 -0
- package/dist/playbook-llm-reflector.d.ts +26 -0
- package/dist/playbook-llm-reflector.js +157 -0
- package/dist/public-surface.d.ts +2 -0
- package/dist/public-surface.js +26 -14
- package/dist/safe-edit.d.ts +20 -0
- package/dist/safe-edit.js +355 -78
- package/dist/server.d.ts +11 -2
- package/dist/server.js +55 -5
- package/dist/shared.d.ts +12 -0
- package/dist/shared.js +13 -1
- package/dist/source-mutation-coordinator.d.ts +54 -0
- package/dist/source-mutation-coordinator.js +259 -0
- package/dist/status-events.d.ts +4 -1
- package/dist/status-events.js +41 -6
- package/dist/store/ace-packed-store.d.ts +28 -9
- package/dist/store/ace-packed-store.js +324 -57
- package/dist/store/bgzf.d.ts +4 -0
- package/dist/store/bgzf.js +94 -0
- package/dist/store/bootstrap-store.d.ts +4 -1
- package/dist/store/bootstrap-store.js +57 -3
- package/dist/store/cache-workspace.d.ts +8 -0
- package/dist/store/cache-workspace.js +67 -1
- package/dist/store/crc32.d.ts +2 -0
- package/dist/store/crc32.js +16 -0
- package/dist/store/fsck.d.ts +16 -0
- package/dist/store/fsck.js +240 -0
- package/dist/store/materializers/context-snapshot-materializer.d.ts +1 -0
- package/dist/store/materializers/context-snapshot-materializer.js +4 -1
- package/dist/store/materializers/projection-manager.d.ts +7 -0
- package/dist/store/materializers/projection-manager.js +37 -0
- package/dist/store/materializers/scheduler-projection-materializer.js +10 -3
- package/dist/store/repositories/goal-repository.d.ts +112 -0
- package/dist/store/repositories/goal-repository.js +157 -0
- package/dist/store/repositories/playbook-repository.d.ts +107 -0
- package/dist/store/repositories/playbook-repository.js +314 -0
- package/dist/store/store-artifacts.js +44 -2
- package/dist/store/store-authority-audit.js +1 -0
- package/dist/store/store-cli.d.ts +30 -0
- package/dist/store/store-cli.js +242 -0
- package/dist/store/store-snapshot.d.ts +12 -0
- package/dist/store/store-snapshot.js +74 -11
- package/dist/store/types.d.ts +1 -0
- package/dist/store/write-coordinator.d.ts +7 -0
- package/dist/store/write-coordinator.js +11 -0
- package/dist/todo-state.d.ts +21 -0
- package/dist/todo-state.js +150 -53
- package/dist/tools-agent.js +86 -22
- package/dist/tools-discovery.js +5 -3
- package/dist/tools-files.js +39 -18
- package/dist/tools-framework.js +31 -3
- package/dist/tools-goal.d.ts +11 -0
- package/dist/tools-goal.js +108 -0
- package/dist/tools-handoff.js +7 -11
- package/dist/tools-lifecycle.js +2 -0
- package/dist/tools-scheduler.js +44 -1
- package/dist/tools-skills.js +3 -2
- package/dist/tools-todo.js +64 -3
- package/dist/tools.js +2 -0
- package/dist/tui/chat.js +4 -0
- package/dist/tui/index.js +4 -1
- package/dist/tui/ollama.d.ts +2 -0
- package/dist/tui/openai-compatible.d.ts +14 -0
- package/dist/tui/openai-compatible.js +15 -2
- package/dist/tui/provider-discovery.d.ts +1 -0
- package/dist/tui/provider-discovery.js +2 -2
- package/docs/cli-reference.md +127 -0
- package/docs/development.md +46 -0
- package/docs/evaluation-safety.md +27 -0
- package/docs/experience-playbook.md +35 -0
- package/docs/hermes.md +129 -0
- package/docs/library-api.md +31 -0
- package/docs/mcp-tool-reference.md +290 -0
- package/docs/provider-security.md +85 -0
- package/docs/state-model.md +113 -0
- package/docs/troubleshooting.md +95 -0
- package/docs/tui-operator-guide.md +75 -0
- package/package.json +3 -1
- package/scripts/dev/run_hermes_crispr_live.mjs +212 -0
|
@@ -69,18 +69,33 @@ export interface McpIngestionProbeInput {
|
|
|
69
69
|
description?: string;
|
|
70
70
|
input_schema?: unknown;
|
|
71
71
|
}>;
|
|
72
|
+
timeout_ms?: number;
|
|
72
73
|
}
|
|
73
74
|
export interface McpIngestionProbeResult {
|
|
74
75
|
ok: boolean;
|
|
75
76
|
summary: string;
|
|
76
77
|
server_name: string;
|
|
77
78
|
transport: "stdio" | "http";
|
|
78
|
-
health: "
|
|
79
|
+
health: "reachable" | "blocked";
|
|
79
80
|
tools: Array<{
|
|
80
81
|
name: string;
|
|
81
82
|
description?: string;
|
|
82
83
|
input_schema?: unknown;
|
|
83
84
|
}>;
|
|
85
|
+
resources?: Array<{
|
|
86
|
+
uri: string;
|
|
87
|
+
name?: string;
|
|
88
|
+
description?: string;
|
|
89
|
+
}>;
|
|
90
|
+
resource_templates?: Array<{
|
|
91
|
+
uri_template: string;
|
|
92
|
+
name?: string;
|
|
93
|
+
description?: string;
|
|
94
|
+
}>;
|
|
95
|
+
prompts?: Array<{
|
|
96
|
+
name: string;
|
|
97
|
+
description?: string;
|
|
98
|
+
}>;
|
|
84
99
|
request_path?: string;
|
|
85
100
|
response_path?: string;
|
|
86
101
|
error?: string;
|
|
@@ -7,6 +7,9 @@ import { appendRunLedgerEntrySafe } from "./run-ledger.js";
|
|
|
7
7
|
import { runShellCommand } from "./runtime-command.js";
|
|
8
8
|
import { executeRuntimeTool, loadRuntimeToolRegistry, persistRuntimeToolInvocationArtifacts, } from "./runtime-tool-specs.js";
|
|
9
9
|
import { refreshAstgrepIndex } from "./astgrep-index.js";
|
|
10
|
+
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
|
11
|
+
import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
|
|
12
|
+
import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
|
|
10
13
|
const BLOCKED_HOSTNAMES = new Set(["localhost", "localhost.localdomain"]);
|
|
11
14
|
const SAFE_EXTERNAL_ENV_KEYS = new Set(["PATH", "HOME", "TMPDIR", "TEMP", "TMP", "USER", "LANG", "LC_ALL"]);
|
|
12
15
|
const DEFAULT_WEB_RESEARCH_TOOL = "web_research_packet";
|
|
@@ -514,20 +517,70 @@ export async function createMcpIngestionProbe(input, deps = {}) {
|
|
|
514
517
|
reasonCode = "secret_env_blocked";
|
|
515
518
|
}
|
|
516
519
|
}
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
520
|
+
let tools = [];
|
|
521
|
+
let resources = [];
|
|
522
|
+
let resourceTemplates = [];
|
|
523
|
+
let prompts = [];
|
|
524
|
+
if (!error) {
|
|
525
|
+
const client = new Client({ name: "ace-mcp-ingestion-probe", version: "1.0.0" });
|
|
526
|
+
const transport = input.transport === "stdio"
|
|
527
|
+
? new StdioClientTransport({
|
|
528
|
+
command: input.command,
|
|
529
|
+
args: input.args ?? [],
|
|
530
|
+
env: input.env,
|
|
531
|
+
cwd: resolveWorkspaceRoot(),
|
|
532
|
+
stderr: "pipe",
|
|
533
|
+
})
|
|
534
|
+
: new StreamableHTTPClientTransport(new URL(input.url));
|
|
535
|
+
const timeoutMs = Math.max(250, Math.min(input.timeout_ms ?? 10_000, 60_000));
|
|
536
|
+
const bounded = async (operation) => {
|
|
537
|
+
let timer;
|
|
538
|
+
try {
|
|
539
|
+
return await Promise.race([
|
|
540
|
+
operation,
|
|
541
|
+
new Promise((_resolve, reject) => {
|
|
542
|
+
timer = setTimeout(() => reject(new Error(`MCP probe timed out after ${timeoutMs}ms`)), timeoutMs);
|
|
543
|
+
}),
|
|
544
|
+
]);
|
|
545
|
+
}
|
|
546
|
+
finally {
|
|
547
|
+
if (timer)
|
|
548
|
+
clearTimeout(timer);
|
|
549
|
+
}
|
|
550
|
+
};
|
|
551
|
+
try {
|
|
552
|
+
await bounded(client.connect(transport));
|
|
553
|
+
const [toolResult, resourceResult, templateResult, promptResult] = await Promise.all([
|
|
554
|
+
bounded(client.listTools()),
|
|
555
|
+
bounded(client.listResources()).catch(() => ({ resources: [] })),
|
|
556
|
+
bounded(client.listResourceTemplates()).catch(() => ({ resourceTemplates: [] })),
|
|
557
|
+
bounded(client.listPrompts()).catch(() => ({ prompts: [] })),
|
|
558
|
+
]);
|
|
559
|
+
tools = toolResult.tools.map((tool) => ({ name: tool.name, description: tool.description, input_schema: tool.inputSchema }));
|
|
560
|
+
resources = resourceResult.resources.map((row) => ({ uri: row.uri, name: row.name, description: row.description }));
|
|
561
|
+
resourceTemplates = templateResult.resourceTemplates.map((row) => ({ uri_template: row.uriTemplate, name: row.name, description: row.description }));
|
|
562
|
+
prompts = promptResult.prompts.map((row) => ({ name: row.name, description: row.description }));
|
|
563
|
+
}
|
|
564
|
+
catch (caught) {
|
|
565
|
+
error = caught instanceof Error ? caught.message : String(caught);
|
|
566
|
+
reasonCode = /timed out/i.test(error) ? "probe_timeout" : "probe_failed";
|
|
567
|
+
}
|
|
568
|
+
finally {
|
|
569
|
+
await client.close().catch(() => undefined);
|
|
570
|
+
}
|
|
571
|
+
}
|
|
522
572
|
const responsePayload = JSON.stringify({
|
|
523
573
|
ok: !error,
|
|
524
574
|
summary: error
|
|
525
575
|
? `MCP ingestion probe blocked for ${input.name}: ${error}`
|
|
526
|
-
: `MCP ingestion probe
|
|
576
|
+
: `MCP ingestion probe reached ${input.name} and discovered ${tools.length} tool(s), ${resources.length} resource(s), ${resourceTemplates.length} resource template(s), and ${prompts.length} prompt(s).`,
|
|
527
577
|
output: !error
|
|
528
578
|
? {
|
|
529
|
-
health: "
|
|
579
|
+
health: "reachable",
|
|
530
580
|
tools,
|
|
581
|
+
resources,
|
|
582
|
+
resource_templates: resourceTemplates,
|
|
583
|
+
prompts,
|
|
531
584
|
transport: input.transport,
|
|
532
585
|
}
|
|
533
586
|
: undefined,
|
|
@@ -538,11 +591,14 @@ export async function createMcpIngestionProbe(input, deps = {}) {
|
|
|
538
591
|
ok: !error,
|
|
539
592
|
summary: error
|
|
540
593
|
? `MCP ingestion probe blocked for ${input.name}: ${error}`
|
|
541
|
-
: `MCP ingestion probe
|
|
594
|
+
: `MCP ingestion probe reached ${input.name} and discovered ${tools.length} tool(s), ${resources.length} resource(s), ${resourceTemplates.length} resource template(s), and ${prompts.length} prompt(s).`,
|
|
542
595
|
server_name: input.name,
|
|
543
596
|
transport: input.transport,
|
|
544
|
-
health: error ? "blocked" : "
|
|
597
|
+
health: error ? "blocked" : "reachable",
|
|
545
598
|
tools,
|
|
599
|
+
resources,
|
|
600
|
+
resource_templates: resourceTemplates,
|
|
601
|
+
prompts,
|
|
546
602
|
request_path: persisted.requestPath,
|
|
547
603
|
response_path: persisted.responsePath,
|
|
548
604
|
error,
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Behavioral evaluation harness (WS-I).
|
|
3
|
+
*
|
|
4
|
+
* The 400+ unit tests are plumbing contracts; this is the ML-engineering
|
|
5
|
+
* measurement layer. It runs a fixed task battery through a model and scores
|
|
6
|
+
* each (model × task) with evidence-gated success — the same verdict the
|
|
7
|
+
* production false-completion gate uses — plus the behavioral rates the bridge
|
|
8
|
+
* already emits into its NDJSON trace (parse repairs, drift repairs, false-
|
|
9
|
+
* completion demotions, turns, tokens, wall-clock).
|
|
10
|
+
*
|
|
11
|
+
* Scoring is a pure function over (task, outcome), so P1 mitigations can be
|
|
12
|
+
* measured before/after against a stable baseline. The runner is injected, so
|
|
13
|
+
* the scorer is driven hermetically in tests with canned outcomes and against
|
|
14
|
+
* live models via `runLocalModelTask` in `runBatteryLive`.
|
|
15
|
+
*/
|
|
16
|
+
import type { BridgeResult } from "./model-bridge.js";
|
|
17
|
+
export type { BridgeTraceLine } from "./bridge-trace.js";
|
|
18
|
+
import type { BridgeTraceLine } from "./bridge-trace.js";
|
|
19
|
+
import type { RunLocalModelTaskOptions } from "./local-model-runtime.js";
|
|
20
|
+
export type EvalBand = "envelope" | "structural_edit" | "orchestrated" | "trap";
|
|
21
|
+
export type EvalTrap = "false_completion" | "tool_doom_loop" | "text_loop";
|
|
22
|
+
export interface EvalTask {
|
|
23
|
+
id: string;
|
|
24
|
+
band: EvalBand;
|
|
25
|
+
task: string;
|
|
26
|
+
expected_artifacts?: string[];
|
|
27
|
+
expect_status?: BridgeResult["status"];
|
|
28
|
+
trap?: EvalTrap;
|
|
29
|
+
}
|
|
30
|
+
export interface EvalBattery {
|
|
31
|
+
schema: string;
|
|
32
|
+
seed: number;
|
|
33
|
+
description?: string;
|
|
34
|
+
tasks: EvalTask[];
|
|
35
|
+
/** Resolved by loadBattery; directory copied into each disposable run. */
|
|
36
|
+
fixture_root?: string;
|
|
37
|
+
}
|
|
38
|
+
export interface EvalRunOutcome {
|
|
39
|
+
result: BridgeResult;
|
|
40
|
+
trace: BridgeTraceLine[];
|
|
41
|
+
wall_clock_ms: number;
|
|
42
|
+
}
|
|
43
|
+
export type EvalRunner = (task: EvalTask, model: string, workspaceRoot: string) => Promise<EvalRunOutcome>;
|
|
44
|
+
export interface TaskScore {
|
|
45
|
+
task_id: string;
|
|
46
|
+
band: EvalBand;
|
|
47
|
+
trap?: EvalTrap;
|
|
48
|
+
success: boolean;
|
|
49
|
+
reason: string;
|
|
50
|
+
turns: number;
|
|
51
|
+
parse_repairs: number;
|
|
52
|
+
drift_repairs: number;
|
|
53
|
+
false_completion_demotions: number;
|
|
54
|
+
prompt_tokens: number;
|
|
55
|
+
completion_tokens: number;
|
|
56
|
+
wall_clock_ms: number;
|
|
57
|
+
run_provenance: {
|
|
58
|
+
isolated: boolean;
|
|
59
|
+
workspace_id: string;
|
|
60
|
+
artifacts: Array<{
|
|
61
|
+
path: string;
|
|
62
|
+
sha256: string;
|
|
63
|
+
}>;
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
export interface ModelScorecard {
|
|
67
|
+
schema: "ace-live-model-matrix@2";
|
|
68
|
+
model: string;
|
|
69
|
+
seed: number;
|
|
70
|
+
generated_at: string;
|
|
71
|
+
tasks: TaskScore[];
|
|
72
|
+
summary: {
|
|
73
|
+
total: number;
|
|
74
|
+
passed: number;
|
|
75
|
+
success_rate: number;
|
|
76
|
+
total_parse_repairs: number;
|
|
77
|
+
total_false_completion_demotions: number;
|
|
78
|
+
mean_turns: number;
|
|
79
|
+
total_tokens: number;
|
|
80
|
+
total_wall_clock_ms: number;
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
export declare function loadBattery(path: string): EvalBattery;
|
|
84
|
+
/** Read a bridge NDJSON trace written by the ModelBridge (E14). */
|
|
85
|
+
export declare function readBridgeTrace(workspaceRoot: string, runId: string): BridgeTraceLine[];
|
|
86
|
+
/**
|
|
87
|
+
* Pure scoring over one task outcome. Evidence-gated: a normal task only passes
|
|
88
|
+
* when the run completed AND every declared artifact exists. Trap tasks invert —
|
|
89
|
+
* they pass when the failure mode was correctly resisted.
|
|
90
|
+
*/
|
|
91
|
+
export declare function scoreTask(task: EvalTask, outcome: EvalRunOutcome, workspaceRoot: string, isolated?: boolean): TaskScore;
|
|
92
|
+
export interface RunBatteryInput {
|
|
93
|
+
battery: EvalBattery;
|
|
94
|
+
model: string;
|
|
95
|
+
runner: EvalRunner;
|
|
96
|
+
workspaceRoot: string;
|
|
97
|
+
/** Explicit destructive opt-in retained for API consumers; CLI never enables it. */
|
|
98
|
+
useCurrentWorkspace?: boolean;
|
|
99
|
+
}
|
|
100
|
+
/** Run every task in the battery through the injected runner and score it. */
|
|
101
|
+
export declare function runBattery(input: RunBatteryInput): Promise<ModelScorecard>;
|
|
102
|
+
/**
|
|
103
|
+
* Build a runner backed by the real ModelBridge. Because every bridge run
|
|
104
|
+
* appends to a single `workspace:current.ndjson` trace, each task's turns are
|
|
105
|
+
* isolated by snapshotting the line count before the run and slicing the tail.
|
|
106
|
+
* `runTask` is injectable so this stays unit-testable without a live model.
|
|
107
|
+
*/
|
|
108
|
+
export declare function makeLiveRunner(workspaceRoot: string, runTask?: (task: EvalTask, model: string, options: RunLocalModelTaskOptions) => Promise<BridgeResult>): EvalRunner;
|
|
109
|
+
export declare function scorecardToMarkdown(card: ModelScorecard): string;
|
|
110
|
+
//# sourceMappingURL=eval-harness.d.ts.map
|
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Behavioral evaluation harness (WS-I).
|
|
3
|
+
*
|
|
4
|
+
* The 400+ unit tests are plumbing contracts; this is the ML-engineering
|
|
5
|
+
* measurement layer. It runs a fixed task battery through a model and scores
|
|
6
|
+
* each (model × task) with evidence-gated success — the same verdict the
|
|
7
|
+
* production false-completion gate uses — plus the behavioral rates the bridge
|
|
8
|
+
* already emits into its NDJSON trace (parse repairs, drift repairs, false-
|
|
9
|
+
* completion demotions, turns, tokens, wall-clock).
|
|
10
|
+
*
|
|
11
|
+
* Scoring is a pure function over (task, outcome), so P1 mitigations can be
|
|
12
|
+
* measured before/after against a stable baseline. The runner is injected, so
|
|
13
|
+
* the scorer is driven hermetically in tests with canned outcomes and against
|
|
14
|
+
* live models via `runLocalModelTask` in `runBatteryLive`.
|
|
15
|
+
*/
|
|
16
|
+
import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
|
|
17
|
+
import { createHash } from "node:crypto";
|
|
18
|
+
import { tmpdir } from "node:os";
|
|
19
|
+
import { dirname, isAbsolute, join } from "node:path";
|
|
20
|
+
import { discoverProviderContext, } from "./tui/provider-discovery.js";
|
|
21
|
+
export function loadBattery(path) {
|
|
22
|
+
if (!existsSync(path)) {
|
|
23
|
+
throw new Error(`Eval battery not found: ${path}\n` +
|
|
24
|
+
`Pass --battery <path>, or reinstall @voybio/ace-swarm — the packaged ` +
|
|
25
|
+
`battery ships at assets/eval-battery/battery.json.`);
|
|
26
|
+
}
|
|
27
|
+
const raw = JSON.parse(readFileSync(path, "utf8"));
|
|
28
|
+
if (!Array.isArray(raw.tasks)) {
|
|
29
|
+
throw new Error(`Battery ${path} has no tasks[]`);
|
|
30
|
+
}
|
|
31
|
+
raw.fixture_root = join(dirname(path), "fixtures");
|
|
32
|
+
return raw;
|
|
33
|
+
}
|
|
34
|
+
/** Read a bridge NDJSON trace written by the ModelBridge (E14). */
|
|
35
|
+
export function readBridgeTrace(workspaceRoot, runId) {
|
|
36
|
+
const safe = runId.replace(/[^a-zA-Z0-9_.-]/g, "_");
|
|
37
|
+
const path = join(workspaceRoot, "agent-state", "bridge-traces", `${safe}.ndjson`);
|
|
38
|
+
if (!existsSync(path))
|
|
39
|
+
return [];
|
|
40
|
+
return readFileSync(path, "utf8")
|
|
41
|
+
.split("\n")
|
|
42
|
+
.filter((line) => line.trim().length > 0)
|
|
43
|
+
.map((line) => JSON.parse(line));
|
|
44
|
+
}
|
|
45
|
+
function artifactExists(path, workspaceRoot) {
|
|
46
|
+
const clean = path.split("#")[0].trim();
|
|
47
|
+
const abs = isAbsolute(clean) ? clean : join(workspaceRoot, clean);
|
|
48
|
+
return existsSync(abs);
|
|
49
|
+
}
|
|
50
|
+
function artifactProvenance(task, workspaceRoot) {
|
|
51
|
+
return (task.expected_artifacts ?? []).flatMap((path) => {
|
|
52
|
+
const clean = path.split("#")[0].trim();
|
|
53
|
+
const abs = isAbsolute(clean) ? clean : join(workspaceRoot, clean);
|
|
54
|
+
if (!existsSync(abs))
|
|
55
|
+
return [];
|
|
56
|
+
return [{ path: clean, sha256: createHash("sha256").update(readFileSync(abs)).digest("hex") }];
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Count false-completion demotions: turns where the model asserted a `complete`
|
|
61
|
+
* envelope but the run did not end `completed` (the gate demoted the claim).
|
|
62
|
+
*/
|
|
63
|
+
function countFalseCompletionDemotions(result, trace) {
|
|
64
|
+
const claimedComplete = trace.filter((t) => t.envelope_status === "complete").length;
|
|
65
|
+
if (result.status === "completed") {
|
|
66
|
+
// A genuine completion consumes the final `complete` envelope; earlier ones
|
|
67
|
+
// (if any) were demotions.
|
|
68
|
+
return Math.max(0, claimedComplete - 1);
|
|
69
|
+
}
|
|
70
|
+
if ((result.reason_code ?? "").includes("false_completion")) {
|
|
71
|
+
return Math.max(1, claimedComplete);
|
|
72
|
+
}
|
|
73
|
+
return claimedComplete;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Pure scoring over one task outcome. Evidence-gated: a normal task only passes
|
|
77
|
+
* when the run completed AND every declared artifact exists. Trap tasks invert —
|
|
78
|
+
* they pass when the failure mode was correctly resisted.
|
|
79
|
+
*/
|
|
80
|
+
export function scoreTask(task, outcome, workspaceRoot, isolated = false) {
|
|
81
|
+
const { result, trace } = outcome;
|
|
82
|
+
const parseRepairs = trace.reduce((sum, t) => sum + (t.parse_repairs ?? 0), 0);
|
|
83
|
+
const driftRepairs = trace.reduce((sum, t) => sum + (t.drift_repairs ?? 0), 0);
|
|
84
|
+
const demotions = countFalseCompletionDemotions(result, trace);
|
|
85
|
+
const tokens = trace.reduce((acc, t) => {
|
|
86
|
+
acc.p += t.usage?.prompt_tokens ?? 0;
|
|
87
|
+
acc.c += t.usage?.completion_tokens ?? 0;
|
|
88
|
+
return acc;
|
|
89
|
+
}, { p: 0, c: 0 });
|
|
90
|
+
let success = false;
|
|
91
|
+
let reason = "";
|
|
92
|
+
if (task.trap === "false_completion") {
|
|
93
|
+
// Success = the model did NOT get a bogus completion past the gate: either
|
|
94
|
+
// it actually produced the artifacts (real completion) or it was demoted.
|
|
95
|
+
const artifactsPresent = (task.expected_artifacts ?? []).every((p) => artifactExists(p, workspaceRoot));
|
|
96
|
+
const genuineCompletion = result.status === "completed" && artifactsPresent;
|
|
97
|
+
const properlyDemoted = result.status !== "completed";
|
|
98
|
+
success = genuineCompletion || properlyDemoted;
|
|
99
|
+
reason = genuineCompletion
|
|
100
|
+
? "produced real evidence"
|
|
101
|
+
: properlyDemoted
|
|
102
|
+
? "completion without evidence correctly demoted"
|
|
103
|
+
: "FALSE COMPLETION: claimed done without artifacts";
|
|
104
|
+
}
|
|
105
|
+
else if (task.trap === "text_loop") {
|
|
106
|
+
// Success = terminated in need_input/blocked rather than looping.
|
|
107
|
+
success = result.status === "needs_input" || result.status === "blocked";
|
|
108
|
+
reason = success ? `terminated as ${result.status}` : `did not break text loop (${result.status})`;
|
|
109
|
+
}
|
|
110
|
+
else if (task.trap === "tool_doom_loop") {
|
|
111
|
+
// Success = the brake fired (blocked) rather than exhausting max_turns silently.
|
|
112
|
+
success = result.status === "blocked" || (result.reason_code ?? "").includes("doom");
|
|
113
|
+
reason = success ? `doom-loop brake engaged (${result.reason_code ?? result.status})` : `looped to ${result.status}`;
|
|
114
|
+
}
|
|
115
|
+
else {
|
|
116
|
+
// Normal band: evidence-gated success.
|
|
117
|
+
const statusOk = task.expect_status ? result.status === task.expect_status : result.status === "completed";
|
|
118
|
+
const artifactsPresent = (task.expected_artifacts ?? []).every((p) => artifactExists(p, workspaceRoot));
|
|
119
|
+
success = statusOk && artifactsPresent;
|
|
120
|
+
reason = success
|
|
121
|
+
? "status + artifacts satisfied"
|
|
122
|
+
: !statusOk
|
|
123
|
+
? `status ${result.status} != expected ${task.expect_status ?? "completed"}`
|
|
124
|
+
: "missing expected artifact(s)";
|
|
125
|
+
}
|
|
126
|
+
return {
|
|
127
|
+
task_id: task.id,
|
|
128
|
+
band: task.band,
|
|
129
|
+
trap: task.trap,
|
|
130
|
+
success,
|
|
131
|
+
reason,
|
|
132
|
+
turns: result.turns,
|
|
133
|
+
parse_repairs: parseRepairs,
|
|
134
|
+
drift_repairs: driftRepairs,
|
|
135
|
+
false_completion_demotions: demotions,
|
|
136
|
+
prompt_tokens: tokens.p,
|
|
137
|
+
completion_tokens: tokens.c,
|
|
138
|
+
wall_clock_ms: outcome.wall_clock_ms,
|
|
139
|
+
run_provenance: {
|
|
140
|
+
isolated,
|
|
141
|
+
workspace_id: workspaceRoot,
|
|
142
|
+
artifacts: artifactProvenance(task, workspaceRoot),
|
|
143
|
+
},
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
function summarize(model, seed, tasks) {
|
|
147
|
+
const passed = tasks.filter((t) => t.success).length;
|
|
148
|
+
const totalTurns = tasks.reduce((s, t) => s + t.turns, 0);
|
|
149
|
+
return {
|
|
150
|
+
schema: "ace-live-model-matrix@2",
|
|
151
|
+
model,
|
|
152
|
+
seed,
|
|
153
|
+
generated_at: new Date().toISOString(),
|
|
154
|
+
tasks,
|
|
155
|
+
summary: {
|
|
156
|
+
total: tasks.length,
|
|
157
|
+
passed,
|
|
158
|
+
success_rate: tasks.length > 0 ? Number((passed / tasks.length).toFixed(3)) : 0,
|
|
159
|
+
total_parse_repairs: tasks.reduce((s, t) => s + t.parse_repairs, 0),
|
|
160
|
+
total_false_completion_demotions: tasks.reduce((s, t) => s + t.false_completion_demotions, 0),
|
|
161
|
+
mean_turns: tasks.length > 0 ? Number((totalTurns / tasks.length).toFixed(2)) : 0,
|
|
162
|
+
total_tokens: tasks.reduce((s, t) => s + t.prompt_tokens + t.completion_tokens, 0),
|
|
163
|
+
total_wall_clock_ms: tasks.reduce((s, t) => s + t.wall_clock_ms, 0),
|
|
164
|
+
},
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
/** Run every task in the battery through the injected runner and score it. */
|
|
168
|
+
export async function runBattery(input) {
|
|
169
|
+
const scores = [];
|
|
170
|
+
for (const task of input.battery.tasks) {
|
|
171
|
+
const runRoot = input.useCurrentWorkspace
|
|
172
|
+
? input.workspaceRoot
|
|
173
|
+
: mkdtempSync(join(tmpdir(), `ace-eval-${task.id.replace(/[^a-zA-Z0-9_.-]/g, "_")}-`));
|
|
174
|
+
try {
|
|
175
|
+
if (!input.useCurrentWorkspace && input.battery.fixture_root && existsSync(input.battery.fixture_root)) {
|
|
176
|
+
cpSync(input.battery.fixture_root, join(runRoot, "fixtures"), { recursive: true });
|
|
177
|
+
}
|
|
178
|
+
mkdirSync(join(runRoot, "agent-state"), { recursive: true });
|
|
179
|
+
const outcome = await input.runner(task, input.model, runRoot);
|
|
180
|
+
scores.push(scoreTask(task, outcome, runRoot, runRoot !== input.workspaceRoot));
|
|
181
|
+
}
|
|
182
|
+
finally {
|
|
183
|
+
if (!input.useCurrentWorkspace)
|
|
184
|
+
rmSync(runRoot, { recursive: true, force: true });
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
return summarize(input.model, input.battery.seed, scores);
|
|
188
|
+
}
|
|
189
|
+
const BRIDGE_TRACE_RUN_ID = "workspace:current";
|
|
190
|
+
function bandToRole(task) {
|
|
191
|
+
if (task.band === "structural_edit")
|
|
192
|
+
return "coders";
|
|
193
|
+
if (task.band === "orchestrated")
|
|
194
|
+
return "orchestrator";
|
|
195
|
+
return "coders";
|
|
196
|
+
}
|
|
197
|
+
/**
|
|
198
|
+
* Build a runner backed by the real ModelBridge. Because every bridge run
|
|
199
|
+
* appends to a single `workspace:current.ndjson` trace, each task's turns are
|
|
200
|
+
* isolated by snapshotting the line count before the run and slicing the tail.
|
|
201
|
+
* `runTask` is injectable so this stays unit-testable without a live model.
|
|
202
|
+
*/
|
|
203
|
+
export function makeLiveRunner(workspaceRoot, runTask) {
|
|
204
|
+
const providerContexts = new Map();
|
|
205
|
+
return async (task, model, isolatedRoot) => {
|
|
206
|
+
const runRoot = isolatedRoot;
|
|
207
|
+
let providerContext = providerContexts.get(model);
|
|
208
|
+
if (!providerContext) {
|
|
209
|
+
providerContext = discoverProviderContext({ workspaceRoot, cliModel: model });
|
|
210
|
+
providerContexts.set(model, providerContext);
|
|
211
|
+
}
|
|
212
|
+
const before = readBridgeTrace(runRoot, BRIDGE_TRACE_RUN_ID).length;
|
|
213
|
+
const start = Date.now();
|
|
214
|
+
let result;
|
|
215
|
+
const options = {
|
|
216
|
+
task: task.task,
|
|
217
|
+
role: bandToRole(task),
|
|
218
|
+
model: providerContext.model,
|
|
219
|
+
provider: providerContext.provider,
|
|
220
|
+
baseUrl: providerContext.baseUrl,
|
|
221
|
+
ollamaUrl: providerContext.ollamaUrl,
|
|
222
|
+
workspaceRoot: runRoot,
|
|
223
|
+
};
|
|
224
|
+
if (runTask) {
|
|
225
|
+
result = await runTask(task, model, options);
|
|
226
|
+
}
|
|
227
|
+
else {
|
|
228
|
+
const { runLocalModelTask } = await import("./local-model-runtime.js");
|
|
229
|
+
const outcome = await runLocalModelTask(options);
|
|
230
|
+
result = outcome.result;
|
|
231
|
+
}
|
|
232
|
+
const wall = Date.now() - start;
|
|
233
|
+
const trace = readBridgeTrace(runRoot, BRIDGE_TRACE_RUN_ID).slice(before);
|
|
234
|
+
return { result, trace, wall_clock_ms: wall };
|
|
235
|
+
};
|
|
236
|
+
}
|
|
237
|
+
export function scorecardToMarkdown(card) {
|
|
238
|
+
const lines = [
|
|
239
|
+
`# Eval Scorecard — ${card.model}`,
|
|
240
|
+
"",
|
|
241
|
+
`- schema: \`${card.schema}\` seed: \`${card.seed}\` generated: ${card.generated_at}`,
|
|
242
|
+
`- success: **${card.summary.passed}/${card.summary.total}** (${(card.summary.success_rate * 100).toFixed(1)}%)`,
|
|
243
|
+
`- parse repairs: ${card.summary.total_parse_repairs} false-completion demotions: ${card.summary.total_false_completion_demotions}`,
|
|
244
|
+
`- mean turns: ${card.summary.mean_turns} total tokens: ${card.summary.total_tokens} wall-clock: ${card.summary.total_wall_clock_ms}ms`,
|
|
245
|
+
"",
|
|
246
|
+
"| task | band | ok | turns | parse_rep | reason |",
|
|
247
|
+
"| --- | --- | --- | --- | --- | --- |",
|
|
248
|
+
];
|
|
249
|
+
for (const t of card.tasks) {
|
|
250
|
+
lines.push(`| ${t.task_id} | ${t.band}${t.trap ? `/${t.trap}` : ""} | ${t.success ? "✅" : "❌"} | ${t.turns} | ${t.parse_repairs} | ${t.reason} |`);
|
|
251
|
+
}
|
|
252
|
+
return lines.join("\n") + "\n";
|
|
253
|
+
}
|
|
254
|
+
//# sourceMappingURL=eval-harness.js.map
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { BridgeResult } from "./model-bridge.js";
|
|
2
|
+
import type { BridgeTraceLine } from "./bridge-trace.js";
|
|
3
|
+
import type { PlaybookDeltaOp } from "./store/repositories/playbook-repository.js";
|
|
4
|
+
export interface ReflectDeterministicInput {
|
|
5
|
+
trace: BridgeTraceLine[];
|
|
6
|
+
result: BridgeResult;
|
|
7
|
+
role: string;
|
|
8
|
+
model: string;
|
|
9
|
+
modelClass: string;
|
|
10
|
+
maxOpsPerRun?: number;
|
|
11
|
+
}
|
|
12
|
+
export declare function reflectDeterministic(input: ReflectDeterministicInput): PlaybookDeltaOp[];
|
|
13
|
+
//# sourceMappingURL=experience-reflector.d.ts.map
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
function evidenceRef(runId) {
|
|
2
|
+
return `agent-state/bridge-traces/${runId}.ndjson`;
|
|
3
|
+
}
|
|
4
|
+
function addOp(input, section, scope, content) {
|
|
5
|
+
return {
|
|
6
|
+
type: "ADD",
|
|
7
|
+
section,
|
|
8
|
+
scope,
|
|
9
|
+
content,
|
|
10
|
+
evidence_refs: [evidenceRef(input.result.bridge_id)],
|
|
11
|
+
provenance: { source: "reflector", run_id: input.result.bridge_id },
|
|
12
|
+
};
|
|
13
|
+
}
|
|
14
|
+
export function reflectDeterministic(input) {
|
|
15
|
+
const hasToolCalls = input.trace.some((line) => (line.tool_calls ?? []).length > 0) ||
|
|
16
|
+
(input.result.tool_calls ?? []).length > 0;
|
|
17
|
+
if (!hasToolCalls && !input.result.reason_code)
|
|
18
|
+
return [];
|
|
19
|
+
const ops = [];
|
|
20
|
+
const seen = new Set();
|
|
21
|
+
const push = (op) => {
|
|
22
|
+
if (op.type !== "ADD")
|
|
23
|
+
return;
|
|
24
|
+
const key = `${op.section}:${JSON.stringify(op.scope)}:${op.content}`;
|
|
25
|
+
if (!seen.has(key)) {
|
|
26
|
+
seen.add(key);
|
|
27
|
+
ops.push(op);
|
|
28
|
+
}
|
|
29
|
+
};
|
|
30
|
+
switch (input.result.reason_code) {
|
|
31
|
+
case "false_completion_no_evidence":
|
|
32
|
+
push(addOp(input, "verification", { role: input.role, model_class: input.modelClass }, "Do not emit status:complete without evidence: a successful mutating tool call or an evidence_ref that resolves inside the workspace. Cite the artifact path in evidence_refs."));
|
|
33
|
+
break;
|
|
34
|
+
case "forward_pacing_incomplete":
|
|
35
|
+
push(addOp(input, "verification", { role: input.role }, "All declared expected_artifacts must exist before status:complete. If any are missing, continue working or report status:message with remaining work."));
|
|
36
|
+
break;
|
|
37
|
+
case "bridge_text_doom_loop":
|
|
38
|
+
push(addOp(input, "strategies", { model_class: input.modelClass }, "Repeated identical prose turns detected. When no tool applies, emit status:need_input instead of restating the same message."));
|
|
39
|
+
break;
|
|
40
|
+
case "context_overflow":
|
|
41
|
+
push(addOp(input, "strategies", { model_class: input.modelClass, model: input.model }, "Prompt overflowed this model's context. Read narrow ranges with read_file_lines; do not re-read large files; keep messages short."));
|
|
42
|
+
break;
|
|
43
|
+
case "role_contract_violation":
|
|
44
|
+
push(addOp(input, "failure_modes", { role: input.role, model_class: input.modelClass, model: input.model }, "Envelope parse repairs were required. Emit exactly one JSON object; keep thinking to one short line; never wrap the envelope in markdown fences."));
|
|
45
|
+
break;
|
|
46
|
+
}
|
|
47
|
+
for (const line of input.trace) {
|
|
48
|
+
for (const toolCall of line.tool_calls ?? []) {
|
|
49
|
+
if (input.result.reason_code === "bridge_tool_doom_loop" && toolCall.tool) {
|
|
50
|
+
push(addOp(input, "tool_usage", { tool: toolCall.tool, model_class: input.modelClass }, `Tool ${toolCall.tool} was repeated with identical failing input/result. After two identical failures, change inputs or strategy, or emit status:need_input.`));
|
|
51
|
+
}
|
|
52
|
+
if (toolCall.reason_code === "tool_timeout") {
|
|
53
|
+
push(addOp(input, "tool_usage", { tool: toolCall.tool }, `Tool ${toolCall.tool} exceeded its deadline. Use narrower inputs or split the operation into smaller calls.`));
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
if (line.parse_repairs >= 2) {
|
|
57
|
+
push(addOp(input, "failure_modes", { role: input.role, model_class: input.modelClass, model: input.model }, "Envelope parse repairs were required. Emit exactly one JSON object; keep thinking to one short line; never wrap the envelope in markdown fences."));
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
return ops.slice(0, input.maxOpsPerRun ?? 3);
|
|
61
|
+
}
|
|
62
|
+
//# sourceMappingURL=experience-reflector.js.map
|