@voybio/ace-swarm 2.4.4 → 2.4.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/CHANGELOG.md +69 -0
  2. package/README.md +72 -532
  3. package/assets/eval-battery/battery.json +85 -0
  4. package/assets/eval-battery/fixtures/edit-target.js +3 -0
  5. package/assets/scripts/render-mcp-configs.sh +12 -0
  6. package/dist/ace-context.d.ts +18 -0
  7. package/dist/ace-context.js +161 -16
  8. package/dist/ace-server-instructions.js +36 -9
  9. package/dist/bridge-trace.d.ts +17 -0
  10. package/dist/bridge-trace.js +23 -0
  11. package/dist/cli.d.ts +9 -1
  12. package/dist/cli.js +619 -10
  13. package/dist/discovery-runtime-wrappers.d.ts +16 -1
  14. package/dist/discovery-runtime-wrappers.js +65 -9
  15. package/dist/eval-harness.d.ts +110 -0
  16. package/dist/eval-harness.js +254 -0
  17. package/dist/experience-reflector.d.ts +13 -0
  18. package/dist/experience-reflector.js +62 -0
  19. package/dist/goal-engine.d.ts +155 -0
  20. package/dist/goal-engine.js +795 -0
  21. package/dist/helpers/store-resolution.js +40 -39
  22. package/dist/hermes/session-manager.d.ts +10 -0
  23. package/dist/hermes/session-manager.js +130 -27
  24. package/dist/job-scheduler.d.ts +58 -0
  25. package/dist/job-scheduler.js +202 -2
  26. package/dist/job-worker.d.ts +87 -0
  27. package/dist/job-worker.js +182 -0
  28. package/dist/local-model-policy.d.ts +8 -2
  29. package/dist/local-model-policy.js +31 -17
  30. package/dist/local-model-runtime.d.ts +6 -0
  31. package/dist/local-model-runtime.js +130 -2
  32. package/dist/measured-classing.d.ts +66 -0
  33. package/dist/measured-classing.js +165 -0
  34. package/dist/model-bridge.d.ts +45 -0
  35. package/dist/model-bridge.js +502 -44
  36. package/dist/orchestrator-supervisor.js +1 -1
  37. package/dist/plan-proposal.d.ts +7 -0
  38. package/dist/plan-proposal.js +74 -4
  39. package/dist/playbook-adaptation.d.ts +37 -0
  40. package/dist/playbook-adaptation.js +159 -0
  41. package/dist/playbook-curator.d.ts +16 -0
  42. package/dist/playbook-curator.js +106 -0
  43. package/dist/playbook-llm-reflector.d.ts +26 -0
  44. package/dist/playbook-llm-reflector.js +157 -0
  45. package/dist/public-surface.d.ts +2 -0
  46. package/dist/public-surface.js +26 -14
  47. package/dist/safe-edit.d.ts +20 -0
  48. package/dist/safe-edit.js +355 -78
  49. package/dist/server.d.ts +11 -2
  50. package/dist/server.js +55 -5
  51. package/dist/shared.d.ts +12 -0
  52. package/dist/shared.js +13 -1
  53. package/dist/source-mutation-coordinator.d.ts +54 -0
  54. package/dist/source-mutation-coordinator.js +259 -0
  55. package/dist/status-events.d.ts +4 -1
  56. package/dist/status-events.js +41 -6
  57. package/dist/store/ace-packed-store.d.ts +28 -9
  58. package/dist/store/ace-packed-store.js +324 -57
  59. package/dist/store/bgzf.d.ts +4 -0
  60. package/dist/store/bgzf.js +94 -0
  61. package/dist/store/bootstrap-store.d.ts +4 -1
  62. package/dist/store/bootstrap-store.js +57 -3
  63. package/dist/store/cache-workspace.d.ts +8 -0
  64. package/dist/store/cache-workspace.js +67 -1
  65. package/dist/store/crc32.d.ts +2 -0
  66. package/dist/store/crc32.js +16 -0
  67. package/dist/store/fsck.d.ts +16 -0
  68. package/dist/store/fsck.js +240 -0
  69. package/dist/store/materializers/context-snapshot-materializer.d.ts +1 -0
  70. package/dist/store/materializers/context-snapshot-materializer.js +4 -1
  71. package/dist/store/materializers/projection-manager.d.ts +7 -0
  72. package/dist/store/materializers/projection-manager.js +37 -0
  73. package/dist/store/materializers/scheduler-projection-materializer.js +10 -3
  74. package/dist/store/repositories/goal-repository.d.ts +112 -0
  75. package/dist/store/repositories/goal-repository.js +157 -0
  76. package/dist/store/repositories/playbook-repository.d.ts +107 -0
  77. package/dist/store/repositories/playbook-repository.js +314 -0
  78. package/dist/store/store-artifacts.js +44 -2
  79. package/dist/store/store-authority-audit.js +1 -0
  80. package/dist/store/store-cli.d.ts +30 -0
  81. package/dist/store/store-cli.js +242 -0
  82. package/dist/store/store-snapshot.d.ts +12 -0
  83. package/dist/store/store-snapshot.js +74 -11
  84. package/dist/store/types.d.ts +1 -0
  85. package/dist/store/write-coordinator.d.ts +7 -0
  86. package/dist/store/write-coordinator.js +11 -0
  87. package/dist/todo-state.d.ts +21 -0
  88. package/dist/todo-state.js +150 -53
  89. package/dist/tools-agent.js +86 -22
  90. package/dist/tools-discovery.js +5 -3
  91. package/dist/tools-files.js +39 -18
  92. package/dist/tools-framework.js +31 -3
  93. package/dist/tools-goal.d.ts +11 -0
  94. package/dist/tools-goal.js +108 -0
  95. package/dist/tools-handoff.js +7 -11
  96. package/dist/tools-lifecycle.js +2 -0
  97. package/dist/tools-scheduler.js +44 -1
  98. package/dist/tools-skills.js +3 -2
  99. package/dist/tools-todo.js +64 -3
  100. package/dist/tools.js +2 -0
  101. package/dist/tui/chat.js +4 -0
  102. package/dist/tui/index.js +4 -1
  103. package/dist/tui/ollama.d.ts +2 -0
  104. package/dist/tui/openai-compatible.d.ts +14 -0
  105. package/dist/tui/openai-compatible.js +15 -2
  106. package/dist/tui/provider-discovery.d.ts +1 -0
  107. package/dist/tui/provider-discovery.js +2 -2
  108. package/docs/cli-reference.md +125 -0
  109. package/docs/development.md +46 -0
  110. package/docs/evaluation-safety.md +27 -0
  111. package/docs/experience-playbook.md +35 -0
  112. package/docs/hermes.md +129 -0
  113. package/docs/library-api.md +31 -0
  114. package/docs/mcp-tool-reference.md +290 -0
  115. package/docs/provider-security.md +85 -0
  116. package/docs/state-model.md +113 -0
  117. package/docs/troubleshooting.md +95 -0
  118. package/docs/tui-operator-guide.md +75 -0
  119. package/package.json +3 -1
  120. package/scripts/dev/run_hermes_crispr_live.mjs +212 -0
@@ -69,18 +69,33 @@ export interface McpIngestionProbeInput {
69
69
  description?: string;
70
70
  input_schema?: unknown;
71
71
  }>;
72
+ timeout_ms?: number;
72
73
  }
73
74
  export interface McpIngestionProbeResult {
74
75
  ok: boolean;
75
76
  summary: string;
76
77
  server_name: string;
77
78
  transport: "stdio" | "http";
78
- health: "configured" | "blocked";
79
+ health: "reachable" | "blocked";
79
80
  tools: Array<{
80
81
  name: string;
81
82
  description?: string;
82
83
  input_schema?: unknown;
83
84
  }>;
85
+ resources?: Array<{
86
+ uri: string;
87
+ name?: string;
88
+ description?: string;
89
+ }>;
90
+ resource_templates?: Array<{
91
+ uri_template: string;
92
+ name?: string;
93
+ description?: string;
94
+ }>;
95
+ prompts?: Array<{
96
+ name: string;
97
+ description?: string;
98
+ }>;
84
99
  request_path?: string;
85
100
  response_path?: string;
86
101
  error?: string;
@@ -7,6 +7,9 @@ import { appendRunLedgerEntrySafe } from "./run-ledger.js";
7
7
  import { runShellCommand } from "./runtime-command.js";
8
8
  import { executeRuntimeTool, loadRuntimeToolRegistry, persistRuntimeToolInvocationArtifacts, } from "./runtime-tool-specs.js";
9
9
  import { refreshAstgrepIndex } from "./astgrep-index.js";
10
+ import { Client } from "@modelcontextprotocol/sdk/client/index.js";
11
+ import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
12
+ import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
10
13
  const BLOCKED_HOSTNAMES = new Set(["localhost", "localhost.localdomain"]);
11
14
  const SAFE_EXTERNAL_ENV_KEYS = new Set(["PATH", "HOME", "TMPDIR", "TEMP", "TMP", "USER", "LANG", "LC_ALL"]);
12
15
  const DEFAULT_WEB_RESEARCH_TOOL = "web_research_packet";
@@ -514,20 +517,70 @@ export async function createMcpIngestionProbe(input, deps = {}) {
514
517
  reasonCode = "secret_env_blocked";
515
518
  }
516
519
  }
517
- const tools = (input.declared_tools ?? []).map((tool) => ({
518
- name: tool.name,
519
- description: tool.description,
520
- input_schema: tool.input_schema,
521
- })).filter((tool) => tool.name?.trim());
520
+ let tools = [];
521
+ let resources = [];
522
+ let resourceTemplates = [];
523
+ let prompts = [];
524
+ if (!error) {
525
+ const client = new Client({ name: "ace-mcp-ingestion-probe", version: "1.0.0" });
526
+ const transport = input.transport === "stdio"
527
+ ? new StdioClientTransport({
528
+ command: input.command,
529
+ args: input.args ?? [],
530
+ env: input.env,
531
+ cwd: resolveWorkspaceRoot(),
532
+ stderr: "pipe",
533
+ })
534
+ : new StreamableHTTPClientTransport(new URL(input.url));
535
+ const timeoutMs = Math.max(250, Math.min(input.timeout_ms ?? 10_000, 60_000));
536
+ const bounded = async (operation) => {
537
+ let timer;
538
+ try {
539
+ return await Promise.race([
540
+ operation,
541
+ new Promise((_resolve, reject) => {
542
+ timer = setTimeout(() => reject(new Error(`MCP probe timed out after ${timeoutMs}ms`)), timeoutMs);
543
+ }),
544
+ ]);
545
+ }
546
+ finally {
547
+ if (timer)
548
+ clearTimeout(timer);
549
+ }
550
+ };
551
+ try {
552
+ await bounded(client.connect(transport));
553
+ const [toolResult, resourceResult, templateResult, promptResult] = await Promise.all([
554
+ bounded(client.listTools()),
555
+ bounded(client.listResources()).catch(() => ({ resources: [] })),
556
+ bounded(client.listResourceTemplates()).catch(() => ({ resourceTemplates: [] })),
557
+ bounded(client.listPrompts()).catch(() => ({ prompts: [] })),
558
+ ]);
559
+ tools = toolResult.tools.map((tool) => ({ name: tool.name, description: tool.description, input_schema: tool.inputSchema }));
560
+ resources = resourceResult.resources.map((row) => ({ uri: row.uri, name: row.name, description: row.description }));
561
+ resourceTemplates = templateResult.resourceTemplates.map((row) => ({ uri_template: row.uriTemplate, name: row.name, description: row.description }));
562
+ prompts = promptResult.prompts.map((row) => ({ name: row.name, description: row.description }));
563
+ }
564
+ catch (caught) {
565
+ error = caught instanceof Error ? caught.message : String(caught);
566
+ reasonCode = /timed out/i.test(error) ? "probe_timeout" : "probe_failed";
567
+ }
568
+ finally {
569
+ await client.close().catch(() => undefined);
570
+ }
571
+ }
522
572
  const responsePayload = JSON.stringify({
523
573
  ok: !error,
524
574
  summary: error
525
575
  ? `MCP ingestion probe blocked for ${input.name}: ${error}`
526
- : `MCP ingestion probe configured ${tools.length} declared tool(s) for ${input.name}.`,
576
+ : `MCP ingestion probe reached ${input.name} and discovered ${tools.length} tool(s), ${resources.length} resource(s), ${resourceTemplates.length} resource template(s), and ${prompts.length} prompt(s).`,
527
577
  output: !error
528
578
  ? {
529
- health: "configured",
579
+ health: "reachable",
530
580
  tools,
581
+ resources,
582
+ resource_templates: resourceTemplates,
583
+ prompts,
531
584
  transport: input.transport,
532
585
  }
533
586
  : undefined,
@@ -538,11 +591,14 @@ export async function createMcpIngestionProbe(input, deps = {}) {
538
591
  ok: !error,
539
592
  summary: error
540
593
  ? `MCP ingestion probe blocked for ${input.name}: ${error}`
541
- : `MCP ingestion probe configured ${tools.length} declared tool(s) for ${input.name}.`,
594
+ : `MCP ingestion probe reached ${input.name} and discovered ${tools.length} tool(s), ${resources.length} resource(s), ${resourceTemplates.length} resource template(s), and ${prompts.length} prompt(s).`,
542
595
  server_name: input.name,
543
596
  transport: input.transport,
544
- health: error ? "blocked" : "configured",
597
+ health: error ? "blocked" : "reachable",
545
598
  tools,
599
+ resources,
600
+ resource_templates: resourceTemplates,
601
+ prompts,
546
602
  request_path: persisted.requestPath,
547
603
  response_path: persisted.responsePath,
548
604
  error,
@@ -0,0 +1,110 @@
1
+ /**
2
+ * Behavioral evaluation harness (WS-I).
3
+ *
4
+ * The 400+ unit tests are plumbing contracts; this is the ML-engineering
5
+ * measurement layer. It runs a fixed task battery through a model and scores
6
+ * each (model × task) with evidence-gated success — the same verdict the
7
+ * production false-completion gate uses — plus the behavioral rates the bridge
8
+ * already emits into its NDJSON trace (parse repairs, drift repairs, false-
9
+ * completion demotions, turns, tokens, wall-clock).
10
+ *
11
+ * Scoring is a pure function over (task, outcome), so P1 mitigations can be
12
+ * measured before/after against a stable baseline. The runner is injected, so
13
+ * the scorer is driven hermetically in tests with canned outcomes and against
14
+ * live models via `runLocalModelTask` in `runBatteryLive`.
15
+ */
16
+ import type { BridgeResult } from "./model-bridge.js";
17
+ export type { BridgeTraceLine } from "./bridge-trace.js";
18
+ import type { BridgeTraceLine } from "./bridge-trace.js";
19
+ import type { RunLocalModelTaskOptions } from "./local-model-runtime.js";
20
+ export type EvalBand = "envelope" | "structural_edit" | "orchestrated" | "trap";
21
+ export type EvalTrap = "false_completion" | "tool_doom_loop" | "text_loop";
22
+ export interface EvalTask {
23
+ id: string;
24
+ band: EvalBand;
25
+ task: string;
26
+ expected_artifacts?: string[];
27
+ expect_status?: BridgeResult["status"];
28
+ trap?: EvalTrap;
29
+ }
30
+ export interface EvalBattery {
31
+ schema: string;
32
+ seed: number;
33
+ description?: string;
34
+ tasks: EvalTask[];
35
+ /** Resolved by loadBattery; directory copied into each disposable run. */
36
+ fixture_root?: string;
37
+ }
38
+ export interface EvalRunOutcome {
39
+ result: BridgeResult;
40
+ trace: BridgeTraceLine[];
41
+ wall_clock_ms: number;
42
+ }
43
+ export type EvalRunner = (task: EvalTask, model: string, workspaceRoot: string) => Promise<EvalRunOutcome>;
44
+ export interface TaskScore {
45
+ task_id: string;
46
+ band: EvalBand;
47
+ trap?: EvalTrap;
48
+ success: boolean;
49
+ reason: string;
50
+ turns: number;
51
+ parse_repairs: number;
52
+ drift_repairs: number;
53
+ false_completion_demotions: number;
54
+ prompt_tokens: number;
55
+ completion_tokens: number;
56
+ wall_clock_ms: number;
57
+ run_provenance: {
58
+ isolated: boolean;
59
+ workspace_id: string;
60
+ artifacts: Array<{
61
+ path: string;
62
+ sha256: string;
63
+ }>;
64
+ };
65
+ }
66
+ export interface ModelScorecard {
67
+ schema: "ace-live-model-matrix@2";
68
+ model: string;
69
+ seed: number;
70
+ generated_at: string;
71
+ tasks: TaskScore[];
72
+ summary: {
73
+ total: number;
74
+ passed: number;
75
+ success_rate: number;
76
+ total_parse_repairs: number;
77
+ total_false_completion_demotions: number;
78
+ mean_turns: number;
79
+ total_tokens: number;
80
+ total_wall_clock_ms: number;
81
+ };
82
+ }
83
+ export declare function loadBattery(path: string): EvalBattery;
84
+ /** Read a bridge NDJSON trace written by the ModelBridge (E14). */
85
+ export declare function readBridgeTrace(workspaceRoot: string, runId: string): BridgeTraceLine[];
86
+ /**
87
+ * Pure scoring over one task outcome. Evidence-gated: a normal task only passes
88
+ * when the run completed AND every declared artifact exists. Trap tasks invert —
89
+ * they pass when the failure mode was correctly resisted.
90
+ */
91
+ export declare function scoreTask(task: EvalTask, outcome: EvalRunOutcome, workspaceRoot: string, isolated?: boolean): TaskScore;
92
+ export interface RunBatteryInput {
93
+ battery: EvalBattery;
94
+ model: string;
95
+ runner: EvalRunner;
96
+ workspaceRoot: string;
97
+ /** Explicit destructive opt-in retained for API consumers; CLI never enables it. */
98
+ useCurrentWorkspace?: boolean;
99
+ }
100
+ /** Run every task in the battery through the injected runner and score it. */
101
+ export declare function runBattery(input: RunBatteryInput): Promise<ModelScorecard>;
102
+ /**
103
+ * Build a runner backed by the real ModelBridge. Because every bridge run
104
+ * appends to a single `workspace:current.ndjson` trace, each task's turns are
105
+ * isolated by snapshotting the line count before the run and slicing the tail.
106
+ * `runTask` is injectable so this stays unit-testable without a live model.
107
+ */
108
+ export declare function makeLiveRunner(workspaceRoot: string, runTask?: (task: EvalTask, model: string, options: RunLocalModelTaskOptions) => Promise<BridgeResult>): EvalRunner;
109
+ export declare function scorecardToMarkdown(card: ModelScorecard): string;
110
+ //# sourceMappingURL=eval-harness.d.ts.map
@@ -0,0 +1,254 @@
1
+ /**
2
+ * Behavioral evaluation harness (WS-I).
3
+ *
4
+ * The 400+ unit tests are plumbing contracts; this is the ML-engineering
5
+ * measurement layer. It runs a fixed task battery through a model and scores
6
+ * each (model × task) with evidence-gated success — the same verdict the
7
+ * production false-completion gate uses — plus the behavioral rates the bridge
8
+ * already emits into its NDJSON trace (parse repairs, drift repairs, false-
9
+ * completion demotions, turns, tokens, wall-clock).
10
+ *
11
+ * Scoring is a pure function over (task, outcome), so P1 mitigations can be
12
+ * measured before/after against a stable baseline. The runner is injected, so
13
+ * the scorer is driven hermetically in tests with canned outcomes and against
14
+ * live models via `runLocalModelTask` in `runBatteryLive`.
15
+ */
16
+ import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
17
+ import { createHash } from "node:crypto";
18
+ import { tmpdir } from "node:os";
19
+ import { dirname, isAbsolute, join } from "node:path";
20
+ import { discoverProviderContext, } from "./tui/provider-discovery.js";
21
+ export function loadBattery(path) {
22
+ if (!existsSync(path)) {
23
+ throw new Error(`Eval battery not found: ${path}\n` +
24
+ `Pass --battery <path>, or reinstall @voybio/ace-swarm — the packaged ` +
25
+ `battery ships at assets/eval-battery/battery.json.`);
26
+ }
27
+ const raw = JSON.parse(readFileSync(path, "utf8"));
28
+ if (!Array.isArray(raw.tasks)) {
29
+ throw new Error(`Battery ${path} has no tasks[]`);
30
+ }
31
+ raw.fixture_root = join(dirname(path), "fixtures");
32
+ return raw;
33
+ }
34
+ /** Read a bridge NDJSON trace written by the ModelBridge (E14). */
35
+ export function readBridgeTrace(workspaceRoot, runId) {
36
+ const safe = runId.replace(/[^a-zA-Z0-9_.-]/g, "_");
37
+ const path = join(workspaceRoot, "agent-state", "bridge-traces", `${safe}.ndjson`);
38
+ if (!existsSync(path))
39
+ return [];
40
+ return readFileSync(path, "utf8")
41
+ .split("\n")
42
+ .filter((line) => line.trim().length > 0)
43
+ .map((line) => JSON.parse(line));
44
+ }
45
+ function artifactExists(path, workspaceRoot) {
46
+ const clean = path.split("#")[0].trim();
47
+ const abs = isAbsolute(clean) ? clean : join(workspaceRoot, clean);
48
+ return existsSync(abs);
49
+ }
50
+ function artifactProvenance(task, workspaceRoot) {
51
+ return (task.expected_artifacts ?? []).flatMap((path) => {
52
+ const clean = path.split("#")[0].trim();
53
+ const abs = isAbsolute(clean) ? clean : join(workspaceRoot, clean);
54
+ if (!existsSync(abs))
55
+ return [];
56
+ return [{ path: clean, sha256: createHash("sha256").update(readFileSync(abs)).digest("hex") }];
57
+ });
58
+ }
59
+ /**
60
+ * Count false-completion demotions: turns where the model asserted a `complete`
61
+ * envelope but the run did not end `completed` (the gate demoted the claim).
62
+ */
63
+ function countFalseCompletionDemotions(result, trace) {
64
+ const claimedComplete = trace.filter((t) => t.envelope_status === "complete").length;
65
+ if (result.status === "completed") {
66
+ // A genuine completion consumes the final `complete` envelope; earlier ones
67
+ // (if any) were demotions.
68
+ return Math.max(0, claimedComplete - 1);
69
+ }
70
+ if ((result.reason_code ?? "").includes("false_completion")) {
71
+ return Math.max(1, claimedComplete);
72
+ }
73
+ return claimedComplete;
74
+ }
75
+ /**
76
+ * Pure scoring over one task outcome. Evidence-gated: a normal task only passes
77
+ * when the run completed AND every declared artifact exists. Trap tasks invert —
78
+ * they pass when the failure mode was correctly resisted.
79
+ */
80
+ export function scoreTask(task, outcome, workspaceRoot, isolated = false) {
81
+ const { result, trace } = outcome;
82
+ const parseRepairs = trace.reduce((sum, t) => sum + (t.parse_repairs ?? 0), 0);
83
+ const driftRepairs = trace.reduce((sum, t) => sum + (t.drift_repairs ?? 0), 0);
84
+ const demotions = countFalseCompletionDemotions(result, trace);
85
+ const tokens = trace.reduce((acc, t) => {
86
+ acc.p += t.usage?.prompt_tokens ?? 0;
87
+ acc.c += t.usage?.completion_tokens ?? 0;
88
+ return acc;
89
+ }, { p: 0, c: 0 });
90
+ let success = false;
91
+ let reason = "";
92
+ if (task.trap === "false_completion") {
93
+ // Success = the model did NOT get a bogus completion past the gate: either
94
+ // it actually produced the artifacts (real completion) or it was demoted.
95
+ const artifactsPresent = (task.expected_artifacts ?? []).every((p) => artifactExists(p, workspaceRoot));
96
+ const genuineCompletion = result.status === "completed" && artifactsPresent;
97
+ const properlyDemoted = result.status !== "completed";
98
+ success = genuineCompletion || properlyDemoted;
99
+ reason = genuineCompletion
100
+ ? "produced real evidence"
101
+ : properlyDemoted
102
+ ? "completion without evidence correctly demoted"
103
+ : "FALSE COMPLETION: claimed done without artifacts";
104
+ }
105
+ else if (task.trap === "text_loop") {
106
+ // Success = terminated in need_input/blocked rather than looping.
107
+ success = result.status === "needs_input" || result.status === "blocked";
108
+ reason = success ? `terminated as ${result.status}` : `did not break text loop (${result.status})`;
109
+ }
110
+ else if (task.trap === "tool_doom_loop") {
111
+ // Success = the brake fired (blocked) rather than exhausting max_turns silently.
112
+ success = result.status === "blocked" || (result.reason_code ?? "").includes("doom");
113
+ reason = success ? `doom-loop brake engaged (${result.reason_code ?? result.status})` : `looped to ${result.status}`;
114
+ }
115
+ else {
116
+ // Normal band: evidence-gated success.
117
+ const statusOk = task.expect_status ? result.status === task.expect_status : result.status === "completed";
118
+ const artifactsPresent = (task.expected_artifacts ?? []).every((p) => artifactExists(p, workspaceRoot));
119
+ success = statusOk && artifactsPresent;
120
+ reason = success
121
+ ? "status + artifacts satisfied"
122
+ : !statusOk
123
+ ? `status ${result.status} != expected ${task.expect_status ?? "completed"}`
124
+ : "missing expected artifact(s)";
125
+ }
126
+ return {
127
+ task_id: task.id,
128
+ band: task.band,
129
+ trap: task.trap,
130
+ success,
131
+ reason,
132
+ turns: result.turns,
133
+ parse_repairs: parseRepairs,
134
+ drift_repairs: driftRepairs,
135
+ false_completion_demotions: demotions,
136
+ prompt_tokens: tokens.p,
137
+ completion_tokens: tokens.c,
138
+ wall_clock_ms: outcome.wall_clock_ms,
139
+ run_provenance: {
140
+ isolated,
141
+ workspace_id: workspaceRoot,
142
+ artifacts: artifactProvenance(task, workspaceRoot),
143
+ },
144
+ };
145
+ }
146
+ function summarize(model, seed, tasks) {
147
+ const passed = tasks.filter((t) => t.success).length;
148
+ const totalTurns = tasks.reduce((s, t) => s + t.turns, 0);
149
+ return {
150
+ schema: "ace-live-model-matrix@2",
151
+ model,
152
+ seed,
153
+ generated_at: new Date().toISOString(),
154
+ tasks,
155
+ summary: {
156
+ total: tasks.length,
157
+ passed,
158
+ success_rate: tasks.length > 0 ? Number((passed / tasks.length).toFixed(3)) : 0,
159
+ total_parse_repairs: tasks.reduce((s, t) => s + t.parse_repairs, 0),
160
+ total_false_completion_demotions: tasks.reduce((s, t) => s + t.false_completion_demotions, 0),
161
+ mean_turns: tasks.length > 0 ? Number((totalTurns / tasks.length).toFixed(2)) : 0,
162
+ total_tokens: tasks.reduce((s, t) => s + t.prompt_tokens + t.completion_tokens, 0),
163
+ total_wall_clock_ms: tasks.reduce((s, t) => s + t.wall_clock_ms, 0),
164
+ },
165
+ };
166
+ }
167
+ /** Run every task in the battery through the injected runner and score it. */
168
+ export async function runBattery(input) {
169
+ const scores = [];
170
+ for (const task of input.battery.tasks) {
171
+ const runRoot = input.useCurrentWorkspace
172
+ ? input.workspaceRoot
173
+ : mkdtempSync(join(tmpdir(), `ace-eval-${task.id.replace(/[^a-zA-Z0-9_.-]/g, "_")}-`));
174
+ try {
175
+ if (!input.useCurrentWorkspace && input.battery.fixture_root && existsSync(input.battery.fixture_root)) {
176
+ cpSync(input.battery.fixture_root, join(runRoot, "fixtures"), { recursive: true });
177
+ }
178
+ mkdirSync(join(runRoot, "agent-state"), { recursive: true });
179
+ const outcome = await input.runner(task, input.model, runRoot);
180
+ scores.push(scoreTask(task, outcome, runRoot, runRoot !== input.workspaceRoot));
181
+ }
182
+ finally {
183
+ if (!input.useCurrentWorkspace)
184
+ rmSync(runRoot, { recursive: true, force: true });
185
+ }
186
+ }
187
+ return summarize(input.model, input.battery.seed, scores);
188
+ }
189
+ const BRIDGE_TRACE_RUN_ID = "workspace:current";
190
+ function bandToRole(task) {
191
+ if (task.band === "structural_edit")
192
+ return "coders";
193
+ if (task.band === "orchestrated")
194
+ return "orchestrator";
195
+ return "coders";
196
+ }
197
+ /**
198
+ * Build a runner backed by the real ModelBridge. Because every bridge run
199
+ * appends to a single `workspace:current.ndjson` trace, each task's turns are
200
+ * isolated by snapshotting the line count before the run and slicing the tail.
201
+ * `runTask` is injectable so this stays unit-testable without a live model.
202
+ */
203
+ export function makeLiveRunner(workspaceRoot, runTask) {
204
+ const providerContexts = new Map();
205
+ return async (task, model, isolatedRoot) => {
206
+ const runRoot = isolatedRoot;
207
+ let providerContext = providerContexts.get(model);
208
+ if (!providerContext) {
209
+ providerContext = discoverProviderContext({ workspaceRoot, cliModel: model });
210
+ providerContexts.set(model, providerContext);
211
+ }
212
+ const before = readBridgeTrace(runRoot, BRIDGE_TRACE_RUN_ID).length;
213
+ const start = Date.now();
214
+ let result;
215
+ const options = {
216
+ task: task.task,
217
+ role: bandToRole(task),
218
+ model: providerContext.model,
219
+ provider: providerContext.provider,
220
+ baseUrl: providerContext.baseUrl,
221
+ ollamaUrl: providerContext.ollamaUrl,
222
+ workspaceRoot: runRoot,
223
+ };
224
+ if (runTask) {
225
+ result = await runTask(task, model, options);
226
+ }
227
+ else {
228
+ const { runLocalModelTask } = await import("./local-model-runtime.js");
229
+ const outcome = await runLocalModelTask(options);
230
+ result = outcome.result;
231
+ }
232
+ const wall = Date.now() - start;
233
+ const trace = readBridgeTrace(runRoot, BRIDGE_TRACE_RUN_ID).slice(before);
234
+ return { result, trace, wall_clock_ms: wall };
235
+ };
236
+ }
237
+ export function scorecardToMarkdown(card) {
238
+ const lines = [
239
+ `# Eval Scorecard — ${card.model}`,
240
+ "",
241
+ `- schema: \`${card.schema}\` seed: \`${card.seed}\` generated: ${card.generated_at}`,
242
+ `- success: **${card.summary.passed}/${card.summary.total}** (${(card.summary.success_rate * 100).toFixed(1)}%)`,
243
+ `- parse repairs: ${card.summary.total_parse_repairs} false-completion demotions: ${card.summary.total_false_completion_demotions}`,
244
+ `- mean turns: ${card.summary.mean_turns} total tokens: ${card.summary.total_tokens} wall-clock: ${card.summary.total_wall_clock_ms}ms`,
245
+ "",
246
+ "| task | band | ok | turns | parse_rep | reason |",
247
+ "| --- | --- | --- | --- | --- | --- |",
248
+ ];
249
+ for (const t of card.tasks) {
250
+ lines.push(`| ${t.task_id} | ${t.band}${t.trap ? `/${t.trap}` : ""} | ${t.success ? "✅" : "❌"} | ${t.turns} | ${t.parse_repairs} | ${t.reason} |`);
251
+ }
252
+ return lines.join("\n") + "\n";
253
+ }
254
+ //# sourceMappingURL=eval-harness.js.map
@@ -0,0 +1,13 @@
1
+ import type { BridgeResult } from "./model-bridge.js";
2
+ import type { BridgeTraceLine } from "./bridge-trace.js";
3
+ import type { PlaybookDeltaOp } from "./store/repositories/playbook-repository.js";
4
+ export interface ReflectDeterministicInput {
5
+ trace: BridgeTraceLine[];
6
+ result: BridgeResult;
7
+ role: string;
8
+ model: string;
9
+ modelClass: string;
10
+ maxOpsPerRun?: number;
11
+ }
12
+ export declare function reflectDeterministic(input: ReflectDeterministicInput): PlaybookDeltaOp[];
13
+ //# sourceMappingURL=experience-reflector.d.ts.map
@@ -0,0 +1,62 @@
1
+ function evidenceRef(runId) {
2
+ return `agent-state/bridge-traces/${runId}.ndjson`;
3
+ }
4
+ function addOp(input, section, scope, content) {
5
+ return {
6
+ type: "ADD",
7
+ section,
8
+ scope,
9
+ content,
10
+ evidence_refs: [evidenceRef(input.result.bridge_id)],
11
+ provenance: { source: "reflector", run_id: input.result.bridge_id },
12
+ };
13
+ }
14
+ export function reflectDeterministic(input) {
15
+ const hasToolCalls = input.trace.some((line) => (line.tool_calls ?? []).length > 0) ||
16
+ (input.result.tool_calls ?? []).length > 0;
17
+ if (!hasToolCalls && !input.result.reason_code)
18
+ return [];
19
+ const ops = [];
20
+ const seen = new Set();
21
+ const push = (op) => {
22
+ if (op.type !== "ADD")
23
+ return;
24
+ const key = `${op.section}:${JSON.stringify(op.scope)}:${op.content}`;
25
+ if (!seen.has(key)) {
26
+ seen.add(key);
27
+ ops.push(op);
28
+ }
29
+ };
30
+ switch (input.result.reason_code) {
31
+ case "false_completion_no_evidence":
32
+ push(addOp(input, "verification", { role: input.role, model_class: input.modelClass }, "Do not emit status:complete without evidence: a successful mutating tool call or an evidence_ref that resolves inside the workspace. Cite the artifact path in evidence_refs."));
33
+ break;
34
+ case "forward_pacing_incomplete":
35
+ push(addOp(input, "verification", { role: input.role }, "All declared expected_artifacts must exist before status:complete. If any are missing, continue working or report status:message with remaining work."));
36
+ break;
37
+ case "bridge_text_doom_loop":
38
+ push(addOp(input, "strategies", { model_class: input.modelClass }, "Repeated identical prose turns detected. When no tool applies, emit status:need_input instead of restating the same message."));
39
+ break;
40
+ case "context_overflow":
41
+ push(addOp(input, "strategies", { model_class: input.modelClass, model: input.model }, "Prompt overflowed this model's context. Read narrow ranges with read_file_lines; do not re-read large files; keep messages short."));
42
+ break;
43
+ case "role_contract_violation":
44
+ push(addOp(input, "failure_modes", { role: input.role, model_class: input.modelClass, model: input.model }, "Envelope parse repairs were required. Emit exactly one JSON object; keep thinking to one short line; never wrap the envelope in markdown fences."));
45
+ break;
46
+ }
47
+ for (const line of input.trace) {
48
+ for (const toolCall of line.tool_calls ?? []) {
49
+ if (input.result.reason_code === "bridge_tool_doom_loop" && toolCall.tool) {
50
+ push(addOp(input, "tool_usage", { tool: toolCall.tool, model_class: input.modelClass }, `Tool ${toolCall.tool} was repeated with identical failing input/result. After two identical failures, change inputs or strategy, or emit status:need_input.`));
51
+ }
52
+ if (toolCall.reason_code === "tool_timeout") {
53
+ push(addOp(input, "tool_usage", { tool: toolCall.tool }, `Tool ${toolCall.tool} exceeded its deadline. Use narrower inputs or split the operation into smaller calls.`));
54
+ }
55
+ }
56
+ if (line.parse_repairs >= 2) {
57
+ push(addOp(input, "failure_modes", { role: input.role, model_class: input.modelClass, model: input.model }, "Envelope parse repairs were required. Emit exactly one JSON object; keep thinking to one short line; never wrap the envelope in markdown fences."));
58
+ }
59
+ }
60
+ return ops.slice(0, input.maxOpsPerRun ?? 3);
61
+ }
62
+ //# sourceMappingURL=experience-reflector.js.map