@tangle-network/traces 0.15.0 → 0.15.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1017,9 +1017,9 @@ function serializeSpans(spans) {
1017
1017
  }
1018
1018
  async function writeOtlpFile(spans, outPath) {
1019
1019
  const { mkdtemp: mkdtemp8, writeFile: writeFile13 } = await import("fs/promises");
1020
- const { tmpdir: tmpdir9 } = await import("os");
1021
- const { join: join38 } = await import("path");
1022
- const path = outPath ?? join38(await mkdtemp8(join38(tmpdir9(), "traces-")), "spans.otlp.jsonl");
1020
+ const { tmpdir: tmpdir8 } = await import("os");
1021
+ const { join: join34 } = await import("path");
1022
+ const path = outPath ?? join34(await mkdtemp8(join34(tmpdir8(), "traces-")), "spans.otlp.jsonl");
1023
1023
  await writeFile13(path, serializeSpans(spans), "utf8");
1024
1024
  return path;
1025
1025
  }
@@ -1130,6 +1130,10 @@ var KIND = {
1130
1130
  LLM: "llm",
1131
1131
  TOOL: "tool",
1132
1132
  CHAIN: "custom",
1133
+ EMBEDDING: "custom",
1134
+ RERANKER: "custom",
1135
+ GUARDRAIL: "custom",
1136
+ PROMPT: "custom",
1133
1137
  EVALUATOR: "custom",
1134
1138
  RETRIEVER: "custom",
1135
1139
  UNKNOWN: "custom"
@@ -1818,7 +1822,7 @@ import {
1818
1822
  LLM_INPUT_TOKENS as LLM_INPUT_TOKENS2,
1819
1823
  LLM_OUTPUT_TOKENS as LLM_OUTPUT_TOKENS2
1820
1824
  } from "@tangle-network/agent-eval/trace-attributes";
1821
- import { deriveHexId as deriveHexId2 } from "@tangle-network/agent-trace-contract";
1825
+ import { ATTR as ATTR3, deriveHexId as deriveHexId2 } from "@tangle-network/agent-trace-contract";
1822
1826
 
1823
1827
  // src/adapters/claude-subagent-sources.ts
1824
1828
  import { readdir as readdir4, realpath } from "fs/promises";
@@ -1984,7 +1988,10 @@ function selectWorkflowBinding(file, bindings, childStartedAt, explicitToolUseId
1984
1988
  `Cannot link Claude Workflow subagent ${file}: parent call is ambiguous`
1985
1989
  );
1986
1990
  }
1987
- return latest[0];
1991
+ return {
1992
+ binding: latest[0],
1993
+ confidence: explicitToolUseId || bindings.length === 1 ? "correlated" : "heuristic"
1994
+ };
1988
1995
  }
1989
1996
 
1990
1997
  // src/adapters/claude-subagent-sources.ts
@@ -2092,6 +2099,21 @@ async function collectClaudeSubagentSources(ref, bindings, signal) {
2092
2099
 
2093
2100
  // src/adapters/claude.ts
2094
2101
  var SERVICE2 = "claude-code";
2102
+ var AGENT_TOOL_RENAMED_IN = [2, 1, 63];
2103
+ function modelFacingToolNames(tools, version) {
2104
+ if (typeof version !== "string" || !atLeast(version, AGENT_TOOL_RENAMED_IN)) return [...tools];
2105
+ return tools.map((name) => name === "Task" ? "Agent" : name);
2106
+ }
2107
+ function atLeast(version, min) {
2108
+ const parts = version.split(".").map((p) => Number.parseInt(p, 10));
2109
+ if (parts.length < min.length || parts.some((p) => !Number.isFinite(p))) return false;
2110
+ for (let i = 0; i < min.length; i++) {
2111
+ if (parts[i] !== min[i]) return parts[i] > min[i];
2112
+ }
2113
+ return true;
2114
+ }
2115
+ var TOOL_DEFINITIONS_ATTR = "gen_ai.tool.definitions";
2116
+ var RUN_STATUS_ATTR = "run.status";
2095
2117
  var EPOCH = (/* @__PURE__ */ new Date(0)).toISOString();
2096
2118
  var HUMAN_ORIGIN = "human";
2097
2119
  var ORIGIN_EVIDENCE_ATTR = "traces.claude.actor_evidence";
@@ -2788,9 +2810,21 @@ var ClaudeAdapter = class {
2788
2810
  let selectedTurnId;
2789
2811
  let selectedTurnFound = false;
2790
2812
  let active = taskScope === "all";
2813
+ let offeredTools;
2814
+ let runResult;
2791
2815
  for await (const event of readJsonl(ref.path, sessionJsonlOptions(ref, options))) {
2792
2816
  options.signal?.throwIfAborted();
2793
- if (!discoveredTraceId && event.sessionId) discoveredTraceId = event.sessionId;
2817
+ const eventSessionId = event.sessionId ?? event.session_id;
2818
+ if (!discoveredTraceId && eventSessionId) discoveredTraceId = eventSessionId;
2819
+ if (event.type === "system" && event.subtype === "init" && Array.isArray(event.tools)) {
2820
+ offeredTools = modelFacingToolNames(
2821
+ event.tools.filter((t) => typeof t === "string"),
2822
+ event.claude_code_version
2823
+ );
2824
+ }
2825
+ if (event.type === "result" && typeof event.is_error === "boolean") {
2826
+ runResult = { failed: event.is_error, subtype: event.subtype ?? "unknown" };
2827
+ }
2794
2828
  const accepted = distinctClaudeEvent(event, seen, ref.path, `step${state.step}`);
2795
2829
  if (!accepted) continue;
2796
2830
  indexWorkflowProjection(accepted.projection, workflowIndex);
@@ -2845,6 +2879,13 @@ var ClaudeAdapter = class {
2845
2879
  service: SERVICE2,
2846
2880
  agent: SERVICE2
2847
2881
  });
2882
+ if (offeredTools) {
2883
+ root.attributes[TOOL_DEFINITIONS_ATTR] = JSON.stringify(offeredTools.map((name) => ({ type: "function", name })));
2884
+ }
2885
+ if (runResult) {
2886
+ root.attributes[RUN_STATUS_ATTR] = runResult.failed ? "failed" : "completed";
2887
+ root.status = runResult.failed ? { code: "ERROR", message: `the run ended with ${runResult.subtype}` } : { code: "OK" };
2888
+ }
2848
2889
  const spans = [root];
2849
2890
  appendAll(spans, main.spans);
2850
2891
  await this.foldSubagents(
@@ -2905,13 +2946,14 @@ var ClaudeAdapter = class {
2905
2946
  workflowBindings
2906
2947
  );
2907
2948
  const parsed = await parseClaudeSubagent(ref, traceId, agent, options);
2908
- const binding = selectWorkflowBinding(
2949
+ const { binding, confidence } = selectWorkflowBinding(
2909
2950
  basename5(file),
2910
2951
  runBindings,
2911
2952
  firstSubagentTimestamp(agent, parsed),
2912
2953
  meta.toolUseId
2913
2954
  );
2914
2955
  agent.parentToolUseId = binding.toolUseId;
2956
+ agent.parentConfidence = confidence;
2915
2957
  }
2916
2958
  agents.push(agent);
2917
2959
  }
@@ -2968,10 +3010,14 @@ var ClaudeAdapter = class {
2968
3010
  const parentAgentMissing = Boolean(
2969
3011
  agent.meta.parentAgentId && !parsedByAgentId.has(agent.meta.parentAgentId)
2970
3012
  );
3013
+ const confidence = parentSpan ? agent.parentConfidence ?? "correlated" : "unknown";
2971
3014
  const task = subagentTaskName(agent);
2972
3015
  const lifecycle = subagentLifecycleSpan(agent, parsed, traceId, parent, task, subDir);
2973
3016
  for (const item of [...parsed.spans, ...lifecycle ? [lifecycle] : []]) {
2974
- if (item.parent_span_id === `root:${traceId}`) item.parent_span_id = parent;
3017
+ if (item.parent_span_id === `root:${traceId}` || item === lifecycle) {
3018
+ item.parent_span_id = parent;
3019
+ item.attributes[ATTR3.parentConfidence] = confidence;
3020
+ }
2975
3021
  item.attributes[SUBAGENT_SPAN_ATTR] = true;
2976
3022
  if (task) item.attributes[SUBAGENT_TASK_ATTR] = task;
2977
3023
  if (agent.parentToolUseId && !parentSpan) {
@@ -3787,9 +3833,9 @@ function itemCountsJson(counts) {
3787
3833
  }
3788
3834
  function placeInnerSpans(pending, windows) {
3789
3835
  return pending.map((entry) => {
3790
- const { parent, join: join38 } = joinItem(windows, entry.startMs, entry.endMs);
3836
+ const { parent, join: join34 } = joinItem(windows, entry.startMs, entry.endMs);
3791
3837
  if (parent) entry.span.parent_span_id = parent.span_id;
3792
- entry.span.attributes["traces.codex.item_join"] = join38;
3838
+ entry.span.attributes["traces.codex.item_join"] = join34;
3793
3839
  return entry.span;
3794
3840
  });
3795
3841
  }
@@ -6434,9 +6480,9 @@ function normalizeRemote(url) {
6434
6480
  }
6435
6481
  async function readGit(cwd) {
6436
6482
  await stat12(`${cwd}/.git`);
6437
- const { execFile: execFile2 } = await import("child_process");
6438
- const { promisify: promisify2 } = await import("util");
6439
- const run = promisify2(execFile2);
6483
+ const { execFile } = await import("child_process");
6484
+ const { promisify } = await import("util");
6485
+ const run = promisify(execFile);
6440
6486
  const git = async (args) => {
6441
6487
  try {
6442
6488
  const { stdout } = await run("git", ["-C", cwd, ...args], { timeout: 5e3 });
@@ -6462,9 +6508,9 @@ async function readGit(cwd) {
6462
6508
  };
6463
6509
  }
6464
6510
  async function gitOutput(cwd, args) {
6465
- const { execFile: execFile2 } = await import("child_process");
6466
- const { promisify: promisify2 } = await import("util");
6467
- const run = promisify2(execFile2);
6511
+ const { execFile } = await import("child_process");
6512
+ const { promisify } = await import("util");
6513
+ const run = promisify(execFile);
6468
6514
  try {
6469
6515
  const { stdout } = await run("git", ["-C", cwd, ...args], { timeout: 5e3 });
6470
6516
  const v = stdout.trim();
@@ -7530,11 +7576,11 @@ function decodeEvidenceRef(value, label, knownSpanUris) {
7530
7576
  }
7531
7577
  const uri = nonEmptyString3(ref.uri, `${label}.uri`);
7532
7578
  if (kind === "span") validateSpanEvidence(uri, knownSpanUris, label);
7533
- const excerpt2 = optionalString(ref.excerpt, `${label}.excerpt`);
7579
+ const excerpt = optionalString(ref.excerpt, `${label}.excerpt`);
7534
7580
  return {
7535
7581
  kind,
7536
7582
  uri,
7537
- ...excerpt2 === void 0 ? {} : { excerpt: excerpt2 }
7583
+ ...excerpt === void 0 ? {} : { excerpt }
7538
7584
  };
7539
7585
  }
7540
7586
  function decodeFinding(value, label, knownSpanUris) {
@@ -7741,14 +7787,14 @@ function normalizeFinding(row, analystId, index, context) {
7741
7787
  to: uri
7742
7788
  });
7743
7789
  }
7744
- const excerpt2 = citation.excerpt === void 0 ? void 0 : normalizeExcerpt(citation.excerpt, span2);
7745
- if (excerpt2 !== citation.excerpt) {
7790
+ const excerpt = citation.excerpt === void 0 ? void 0 : normalizeExcerpt(citation.excerpt, span2);
7791
+ if (excerpt !== citation.excerpt) {
7746
7792
  context.log?.("finding citation normalized: json-escaped excerpt", {
7747
7793
  analyst_id: analystId,
7748
7794
  uri
7749
7795
  });
7750
7796
  }
7751
- return excerpt2 === void 0 ? { uri } : { uri, excerpt: excerpt2 };
7797
+ return excerpt === void 0 ? { uri } : { uri, excerpt };
7752
7798
  });
7753
7799
  const normalized = { ...row, evidence: evidence3 };
7754
7800
  const allowed = KIND_EXPECTED_SUBJECTS[analystId];
@@ -7797,10 +7843,10 @@ function decodeComponent(value) {
7797
7843
  return value;
7798
7844
  }
7799
7845
  }
7800
- function normalizeExcerpt(excerpt2, span2) {
7801
- if (containsText(span2.content, excerpt2)) return excerpt2;
7802
- const decoded = decodeJsonEscapes(excerpt2);
7803
- return decoded !== excerpt2 && containsText(span2.content, decoded) ? decoded : excerpt2;
7846
+ function normalizeExcerpt(excerpt, span2) {
7847
+ if (containsText(span2.content, excerpt)) return excerpt;
7848
+ const decoded = decodeJsonEscapes(excerpt);
7849
+ return decoded !== excerpt && containsText(span2.content, decoded) ? decoded : excerpt;
7804
7850
  }
7805
7851
  function decodeJsonEscapes(text2) {
7806
7852
  return text2.replace(
@@ -9118,13 +9164,13 @@ import { OtlpFileTraceStore } from "@tangle-network/agent-eval/traces";
9118
9164
  import { isUtf8 as isUtf82 } from "buffer";
9119
9165
  import { createHash as createHash9 } from "crypto";
9120
9166
  import { constants } from "fs";
9121
- import { lstat, open, realpath as realpath3 } from "fs/promises";
9167
+ import { lstat as lstat2, open, realpath as realpath3 } from "fs/promises";
9122
9168
  import { dirname as dirname8, isAbsolute as isAbsolute4, join as join20, relative as relative5, resolve as resolve7, sep as sep2 } from "path";
9123
9169
 
9124
9170
  // src/bundle.ts
9125
9171
  import { createHash as createHash8 } from "crypto";
9126
9172
  import { readFileSync as readFileSync2 } from "fs";
9127
- import { copyFile, cp, mkdir as mkdir2, readdir as readdir16, readFile as readFile7, stat as stat14, writeFile as writeFile5 } from "fs/promises";
9173
+ import { copyFile, cp, lstat, mkdir as mkdir2, readdir as readdir16, readFile as readFile7, stat as stat14, writeFile as writeFile5 } from "fs/promises";
9128
9174
  import { dirname as dirname7, join as join19, relative as relative4, resolve as resolve6, sep } from "path";
9129
9175
 
9130
9176
  // src/improvement.ts
@@ -9444,7 +9490,7 @@ function validationExitCode(validation) {
9444
9490
 
9445
9491
  // src/loop-analysis.ts
9446
9492
  import {
9447
- ATTR as ATTR3,
9493
+ ATTR as ATTR4,
9448
9494
  firstNumberAttr,
9449
9495
  firstStringAttr,
9450
9496
  ITERATION_ATTR_KEYS,
@@ -9458,11 +9504,11 @@ function attributes(span2) {
9458
9504
  return span2.attributes;
9459
9505
  }
9460
9506
  function outcomeOf(span2) {
9461
- const raw = firstStringAttr(attributes(span2), [ATTR3.outcome]);
9507
+ const raw = firstStringAttr(attributes(span2), [ATTR4.outcome]);
9462
9508
  return raw !== void 0 && OUTCOMES.includes(raw) ? raw : null;
9463
9509
  }
9464
9510
  function scoreOf(span2) {
9465
- return firstNumberAttr(attributes(span2), [ATTR3.score]) ?? null;
9511
+ return firstNumberAttr(attributes(span2), [ATTR4.score]) ?? null;
9466
9512
  }
9467
9513
  function iterationOf(span2) {
9468
9514
  const value = firstNumberAttr(attributes(span2), ITERATION_ATTR_KEYS);
@@ -9535,7 +9581,7 @@ function trendOf(iterations) {
9535
9581
  return {
9536
9582
  trend: "unscored",
9537
9583
  basis: "none",
9538
- detail: `${iterations.length} round(s) recorded, ${scored.length + graded.length} graded: the trace carries neither ${ATTR3.outcome} nor ${ATTR3.score} on enough rounds to compare them`
9584
+ detail: `${iterations.length} round(s) recorded, ${scored.length + graded.length} graded: the trace carries neither ${ATTR4.outcome} nor ${ATTR4.score} on enough rounds to compare them`
9539
9585
  };
9540
9586
  }
9541
9587
  function analyzeLoopConvergence(spans) {
@@ -9546,7 +9592,7 @@ function analyzeLoopConvergence(spans) {
9546
9592
  const iteration = iterationOf(span2);
9547
9593
  if (iteration === null) continue;
9548
9594
  iterationSpans += 1;
9549
- const loopId = firstStringAttr(attributes(span2), [ATTR3.loopId]) ?? `(unnamed loop in ${span2.trace_id})`;
9595
+ const loopId = firstStringAttr(attributes(span2), [ATTR4.loopId]) ?? `(unnamed loop in ${span2.trace_id})`;
9550
9596
  const key = `${span2.trace_id}\0${loopId}`;
9551
9597
  const verdict = verdictFor(span2, children);
9552
9598
  const bucket = byLoop.get(key) ?? { traceId: span2.trace_id, iterations: [] };
@@ -10641,8 +10687,8 @@ function renderReport(result, meta) {
10641
10687
  if (f.recommended_action) lines.push(`- **Fix:** ${f.recommended_action}`);
10642
10688
  if (f.validation_plan) lines.push(`- **Check:** ${f.validation_plan}`);
10643
10689
  for (const ev of f.evidence_refs.slice(0, 3)) {
10644
- const excerpt2 = ev.excerpt ? ` \u2014 \`${ev.excerpt.slice(0, 200)}\`` : "";
10645
- lines.push(`- **Evidence:** ${ev.kind} ${ev.uri}${excerpt2}`);
10690
+ const excerpt = ev.excerpt ? ` \u2014 \`${ev.excerpt.slice(0, 200)}\`` : "";
10691
+ lines.push(`- **Evidence:** ${ev.kind} ${ev.uri}${excerpt}`);
10646
10692
  }
10647
10693
  lines.push("");
10648
10694
  }
@@ -10966,8 +11012,8 @@ function jsonl(rows) {
10966
11012
  return rows.length === 0 ? "" : `${rows.map((row) => JSON.stringify(row)).join("\n")}
10967
11013
  `;
10968
11014
  }
10969
- function evidence(kind, uri, excerpt2) {
10970
- return excerpt2 ? { kind, uri, excerpt: excerpt2 } : { kind, uri };
11015
+ function evidence(kind, uri, excerpt) {
11016
+ return excerpt ? { kind, uri, excerpt } : { kind, uri };
10971
11017
  }
10972
11018
  function totalReactionSignals(reactions) {
10973
11019
  return Object.values(reactions.signals).reduce((sum, count2) => sum + count2, 0);
@@ -11679,8 +11725,8 @@ async function collectContextRoot(root) {
11679
11725
  if (summary) files.push(summary);
11680
11726
  }
11681
11727
  }
11682
- const progress = await summarizeFile(join18(evolve, "progress.md"), "other");
11683
- if (progress) files.push(progress);
11728
+ const progress2 = await summarizeFile(join18(evolve, "progress.md"), "other");
11729
+ if (progress2) files.push(progress2);
11684
11730
  return { root, files: files.sort((a, b) => a.path.localeCompare(b.path)) };
11685
11731
  }
11686
11732
  async function listEvolveRootFiles(evolve) {
@@ -12209,6 +12255,9 @@ var KNOWN_LIMITS = [
12209
12255
  "progress.md is copied whole; no session-dated section extraction is performed",
12210
12256
  "open pull-request state is not captured: the bundler makes no network calls"
12211
12257
  ];
12258
+ function sha256Hex(bytes) {
12259
+ return createHash8("sha256").update(bytes).digest("hex");
12260
+ }
12212
12261
  function bundlePath(...parts) {
12213
12262
  return parts.join("/");
12214
12263
  }
@@ -12244,9 +12293,9 @@ function sliceJsonlByWindow(text2, windowStartMs, windowEndMs) {
12244
12293
  return { kept, totalRows, unparseableTsRows };
12245
12294
  }
12246
12295
  async function runGitLog(cwd, sinceIso, untilIso) {
12247
- const { execFile: execFile2 } = await import("child_process");
12248
- const { promisify: promisify2 } = await import("util");
12249
- const run = promisify2(execFile2);
12296
+ const { execFile } = await import("child_process");
12297
+ const { promisify } = await import("util");
12298
+ const run = promisify(execFile);
12250
12299
  try {
12251
12300
  const { stdout } = await run(
12252
12301
  "git",
@@ -12549,6 +12598,90 @@ async function readSessionBundleManifest(bundleDir) {
12549
12598
  }
12550
12599
  return manifest;
12551
12600
  }
12601
+ function unsafeBundlePath(path) {
12602
+ if (path.length === 0) return "empty path";
12603
+ if (path.includes("\\")) return "contains a backslash";
12604
+ if (path.startsWith("/") || /^[A-Za-z]:/.test(path)) return "is absolute";
12605
+ if (path.split("/").some((part) => part === ".." || part === "." || part === "")) {
12606
+ return 'has an empty, "." or ".." segment';
12607
+ }
12608
+ return null;
12609
+ }
12610
+ async function nonRegularEntries(root) {
12611
+ const out = [];
12612
+ const pending = [resolve6(root)];
12613
+ while (pending.length > 0) {
12614
+ const dir = pending.pop();
12615
+ for (const entry of await readdir16(dir, { withFileTypes: true })) {
12616
+ const path = join19(dir, entry.name);
12617
+ if (entry.isDirectory()) pending.push(path);
12618
+ else if (!entry.isFile()) out.push(relative4(resolve6(root), path).split(sep).join("/"));
12619
+ }
12620
+ }
12621
+ return out.sort();
12622
+ }
12623
+ async function verifySessionBundle(bundleDir) {
12624
+ const directory = resolve6(bundleDir);
12625
+ const manifest = await readSessionBundleManifest(directory);
12626
+ const issues = [];
12627
+ const listed = /* @__PURE__ */ new Set();
12628
+ let matched = 0;
12629
+ for (const file of manifest.files) {
12630
+ const unsafe = unsafeBundlePath(file.path);
12631
+ if (unsafe) {
12632
+ issues.push({ kind: "unsafe-path", path: file.path, detail: `manifest path ${unsafe}` });
12633
+ continue;
12634
+ }
12635
+ listed.add(file.path);
12636
+ const info = await lstat(join19(directory, file.path)).catch((error) => {
12637
+ if (isMissingPathError(error)) return null;
12638
+ throw error;
12639
+ });
12640
+ if (!info) {
12641
+ issues.push({ kind: "missing", path: file.path, detail: "listed in the manifest, not on disk" });
12642
+ continue;
12643
+ }
12644
+ if (!info.isFile()) continue;
12645
+ const sha2562 = await fileSha256(join19(directory, file.path));
12646
+ if (info.size !== file.bytes || sha2562 !== file.sha256) {
12647
+ issues.push({
12648
+ kind: "mismatch",
12649
+ path: file.path,
12650
+ detail: `recorded ${file.bytes} bytes sha256 ${file.sha256}, found ${info.size} bytes sha256 ${sha2562}`
12651
+ });
12652
+ continue;
12653
+ }
12654
+ matched += 1;
12655
+ }
12656
+ for (const path of await listSessionBundleFiles(directory)) {
12657
+ if (path === "manifest.json" || listed.has(path)) continue;
12658
+ issues.push({ kind: "unexpected", path, detail: "on disk, not listed in the manifest" });
12659
+ }
12660
+ for (const path of await nonRegularEntries(directory)) {
12661
+ issues.push({ kind: "not-a-file", path, detail: "a symbolic link or special file; a bundle holds regular files only" });
12662
+ }
12663
+ let source;
12664
+ if (manifest.projection) {
12665
+ const sourceManifest = join19(manifest.projection.sourceDirectory, "manifest.json");
12666
+ const bytes = await readFile7(sourceManifest).catch((error) => {
12667
+ if (isMissingPathError(error)) return null;
12668
+ throw error;
12669
+ });
12670
+ source = {
12671
+ directory: manifest.projection.sourceDirectory,
12672
+ status: bytes === null ? "unavailable" : sha256Hex(bytes) === manifest.projection.sourceManifestSha256 ? "match" : "mismatch"
12673
+ };
12674
+ }
12675
+ return {
12676
+ ok: issues.length === 0 && source?.status !== "mismatch",
12677
+ directory,
12678
+ view: manifest.view,
12679
+ listed: manifest.files.length,
12680
+ matched,
12681
+ issues,
12682
+ ...source ? { source } : {}
12683
+ };
12684
+ }
12552
12685
  var cachedVersion;
12553
12686
  function tracesPackageVersion() {
12554
12687
  if (cachedVersion) return cachedVersion;
@@ -12578,7 +12711,7 @@ async function openBundleFile(root, file) {
12578
12711
  let current = root;
12579
12712
  for (const part of relative5(root, path).split(sep2)) {
12580
12713
  current = join20(current, part);
12581
- if ((await lstat(current)).isSymbolicLink()) throw new Error("bundle source symlinks are not permitted");
12714
+ if ((await lstat2(current)).isSymbolicLink()) throw new Error("bundle source symlinks are not permitted");
12582
12715
  }
12583
12716
  if (await realpath3(path) !== path) throw new Error("bundle source resolved outside its declared path");
12584
12717
  const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
@@ -12614,9 +12747,9 @@ async function createBundleSourceReader(directory, spans, options = {}) {
12614
12747
  const maxRecordBytes = options.maxRecordBytes ?? 16 * 1024 * 1024;
12615
12748
  if (!Number.isSafeInteger(maxRecordBytes) || maxRecordBytes < 1) throw new Error("maxRecordBytes must be a positive safe integer");
12616
12749
  options.signal?.throwIfAborted();
12617
- if ((await lstat(resolve7(directory))).isSymbolicLink()) throw new Error("bundle directory must not be a symlink");
12750
+ if ((await lstat2(resolve7(directory))).isSymbolicLink()) throw new Error("bundle directory must not be a symlink");
12618
12751
  const root = await realpath3(resolve7(directory));
12619
- if ((await lstat(join20(root, "manifest.json"))).isSymbolicLink()) throw new Error("bundle manifest must not be a symlink");
12752
+ if ((await lstat2(join20(root, "manifest.json"))).isSymbolicLink()) throw new Error("bundle manifest must not be a symlink");
12620
12753
  const manifest = await readSessionBundleManifest(root);
12621
12754
  if (manifest.view !== "full") throw new Error("original source reads require an explicitly authorized full bundle");
12622
12755
  if (!Array.isArray(manifest.files)) throw new Error("bundle manifest has no file receipts");
@@ -13042,7 +13175,7 @@ import { basename as basename14, dirname as dirname10, join as join23, resolve a
13042
13175
  import { createHash as createHash11, randomUUID } from "crypto";
13043
13176
  import {
13044
13177
  link,
13045
- lstat as lstat2,
13178
+ lstat as lstat3,
13046
13179
  mkdir as mkdir3,
13047
13180
  readFile as readFile8,
13048
13181
  realpath as realpath4,
@@ -13211,7 +13344,7 @@ async function optionalContainedFile(root, candidate) {
13211
13344
  }
13212
13345
  async function assertPathMissing(path) {
13213
13346
  try {
13214
- await lstat2(path);
13347
+ await lstat3(path);
13215
13348
  } catch (error) {
13216
13349
  if (isErrno(error, "ENOENT")) return;
13217
13350
  throw error;
@@ -13725,54 +13858,18 @@ function validateRevision(value) {
13725
13858
  return value;
13726
13859
  }
13727
13860
 
13728
- // src/replay-verify.ts
13729
- import { mkdirSync, readFileSync as readFileSync3, writeFileSync } from "fs";
13730
- import { join as join24 } from "path";
13861
+ // src/replay.ts
13862
+ import { readFileSync as readFileSync3 } from "fs";
13731
13863
  import {
13732
- InMemoryTraceStore as InMemoryTraceStore2,
13733
- runCounterfactual,
13734
- TraceEmitter
13735
- } from "@tangle-network/agent-eval";
13864
+ enumerateReplayableCases,
13865
+ parseCorpusFlag,
13866
+ readFindingsFile,
13867
+ replayVerify,
13868
+ runReplayBatch,
13869
+ verifyFindings
13870
+ } from "@tangle-network/agent-eval/trajectory-replay";
13736
13871
  import { Sandbox } from "@tangle-network/sandbox";
13737
- function parseRecordedReturncode(observation) {
13738
- if (!observation) return null;
13739
- const m = /<returncode>(-?\d+)<\/returncode>/.exec(observation);
13740
- return m ? Number(m[1]) : null;
13741
- }
13742
- function parseObservationOutput(observation) {
13743
- if (!observation) return "";
13744
- const m = /<output>\n?([\s\S]*?)\n?<\/output>/.exec(observation);
13745
- return m ? m[1] : observation;
13746
- }
13747
- function deriveFailureSignature(observation) {
13748
- const line = parseObservationOutput(observation).split("\n").find((l) => /\berror\b/i.test(l));
13749
- return line ? line.trim().slice(0, 200) : null;
13750
- }
13751
- async function ingestCodeTraceBenchSteps(store, steps, caseId) {
13752
- let tick = 0;
13753
- const emitter = new TraceEmitter(store, { now: () => ++tick });
13754
- await emitter.startRun({
13755
- scenarioId: caseId,
13756
- tags: { source: "codetracebench-replay", caseId }
13757
- });
13758
- for (const step of steps) {
13759
- if (typeof step.action !== "string" || step.action.length === 0) {
13760
- throw new Error(`replay-verify: step ${step.step_id} has no action`);
13761
- }
13762
- const handle = await emitter.tool({
13763
- name: `step:${step.step_id}`,
13764
- toolName: "shell",
13765
- args: { command: step.action },
13766
- result: {
13767
- returncode: parseRecordedReturncode(step.observation),
13768
- observation: step.observation
13769
- }
13770
- });
13771
- await handle.end();
13772
- }
13773
- await emitter.endRun({ pass: true, notes: "recorded trajectory (ingested for replay)" });
13774
- return { runId: emitter.runId, stepCount: steps.length };
13775
- }
13872
+ var DEFAULT_SANDBOX_BASE_URL = "http://127.0.0.1:4097";
13776
13873
  function sandboxReplayBackend(options) {
13777
13874
  const client = new Sandbox({ apiKey: options.apiKey, baseUrl: options.baseUrl });
13778
13875
  return {
@@ -13797,365 +13894,117 @@ function sandboxReplayBackend(options) {
13797
13894
  }
13798
13895
  };
13799
13896
  }
13800
- function wrapActionForExec(action, cwd) {
13801
- const b64 = Buffer.from(action, "utf8").toString("base64");
13802
- const quotedCwd = `'${cwd.replaceAll("'", `'\\''`)}'`;
13803
- return `cd ${quotedCwd} && printf %s ${b64} | base64 -d | sh`;
13804
- }
13805
- function toolCommand(span2) {
13806
- const args = span2.args;
13807
- if (!args || typeof args.command !== "string") {
13808
- throw new Error(`replay-verify: span ${span2.name} carries no command`);
13809
- }
13810
- return args.command;
13811
- }
13812
- function recordedReturncodeOf(span2) {
13813
- const result = span2.result;
13814
- return typeof result?.returncode === "number" ? result.returncode : null;
13815
- }
13816
- var SandboxCounterfactualRunner = class {
13817
- constructor(backend, options) {
13818
- this.backend = backend;
13819
- this.options = options;
13820
- }
13821
- backend;
13822
- options;
13823
- lastPrefix = null;
13824
- lastArm = null;
13825
- async executeFrom(ctx, emitter) {
13826
- const { cwd, stepTimeoutMs, prefixLimit, onProgress } = this.options;
13827
- const session = await this.backend.open();
13828
- try {
13829
- const toolSteps = ctx.prefix.filter(
13830
- (s) => s.span.kind === "tool"
13831
- );
13832
- const toExecute = prefixLimit !== void 0 ? toolSteps.slice(0, prefixLimit) : toolSteps;
13833
- const divergences = [];
13834
- const prefixStart = Date.now();
13835
- for (const step of toExecute) {
13836
- const command = toolCommand(step.span);
13837
- const expected = recordedReturncodeOf(step.span);
13838
- const started = Date.now();
13839
- const result = await session.exec(wrapActionForExec(command, cwd), stepTimeoutMs);
13840
- const handle = await emitter.sandbox({
13841
- name: step.span.name,
13842
- command,
13843
- exitCode: result.exitCode,
13844
- wallMs: Date.now() - started
13845
- });
13846
- await handle.end();
13847
- if (expected !== null && expected !== result.exitCode) {
13848
- divergences.push({
13849
- step: step.index + 1,
13850
- expectedReturncode: expected,
13851
- actualExit: result.exitCode
13852
- });
13853
- }
13854
- onProgress?.(
13855
- `prefix ${step.span.name}: exit=${result.exitCode}` + (expected !== null && expected !== result.exitCode ? ` (recorded ${expected})` : "")
13856
- );
13857
- }
13858
- this.lastPrefix = {
13859
- prefixExecuted: toExecute.length,
13860
- prefixDivergences: divergences,
13861
- wallMs: Date.now() - prefixStart
13862
- };
13863
- if (ctx.mutatedStep.span.kind !== "tool") {
13864
- throw new Error("replay-verify: mutation target is not a tool span");
13865
- }
13866
- const armCommand = toolCommand(ctx.mutatedStep.span);
13867
- const armStart = Date.now();
13868
- const armResult = await session.exec(wrapActionForExec(armCommand, cwd), stepTimeoutMs);
13869
- const armWallMs = Date.now() - armStart;
13870
- const armHandle = await emitter.sandbox({
13871
- name: `candidate:${ctx.mutatedStep.span.name}`,
13872
- command: armCommand,
13873
- exitCode: armResult.exitCode,
13874
- wallMs: armWallMs
13875
- });
13876
- await armHandle.end();
13877
- this.lastArm = {
13878
- command: armCommand,
13879
- exitCode: armResult.exitCode,
13880
- stdout: armResult.stdout,
13881
- stderr: armResult.stderr,
13882
- wallMs: armWallMs
13883
- };
13884
- onProgress?.(`candidate step: exit=${armResult.exitCode} in ${armWallMs}ms`);
13885
- await emitter.endRun({
13886
- pass: armResult.exitCode === 0,
13887
- notes: `prefix ${toExecute.length} steps, ${divergences.length} divergences`
13888
- });
13889
- } finally {
13890
- await session.close();
13891
- }
13892
- }
13893
- };
13894
- function excerpt(text2, limit = 2400) {
13895
- if (text2.length <= limit) return text2;
13896
- const head = text2.slice(0, limit / 2);
13897
- const tail = text2.slice(-limit / 2);
13898
- return `${head}
13899
- \u2026 [${text2.length - limit} chars elided] \u2026
13900
- ${tail}`;
13901
- }
13902
- async function replayVerify(options) {
13903
- const totalStart = Date.now();
13904
- const steps = JSON.parse(readFileSync3(options.stepsPath, "utf8"));
13905
- if (!Array.isArray(steps) || steps.length === 0) {
13906
- throw new Error(`replay-verify: ${options.stepsPath} is not a non-empty steps array`);
13907
- }
13908
- const index = options.at - 1;
13909
- if (index < 0 || index >= steps.length) {
13910
- throw new Error(`replay-verify: --at ${options.at} out of range [1, ${steps.length}]`);
13911
- }
13912
- const target = steps[index];
13913
- if (target.step_id !== options.at) {
13897
+ function sandboxBackendFactory(options) {
13898
+ return (image) => sandboxReplayBackend({ ...options, image });
13899
+ }
13900
+ async function assertSandboxReachable(baseUrl) {
13901
+ let response;
13902
+ try {
13903
+ response = await fetch(new URL("/health", baseUrl), { signal: AbortSignal.timeout(5e3) });
13904
+ } catch (err) {
13914
13905
  throw new Error(
13915
- `replay-verify: steps[${index}].step_id=${target.step_id} != --at ${options.at}; steps.json must be ordered with 1-based contiguous step_ids`
13906
+ `verify-findings: sandbox API unreachable at ${baseUrl} \u2014 ${err instanceof Error ? err.message : String(err)}. Executing findings requires a running sandbox orchestrator + SDK adapter (see docs/replay-verify.md); start them or pass --base-url.`
13916
13907
  );
13917
13908
  }
13918
- const caseId = options.caseId ?? options.stepsPath;
13919
- const recordedReturncode = parseRecordedReturncode(target.observation);
13920
- const signature = options.signature ?? deriveFailureSignature(target.observation);
13921
- const signatureBasis = signature ? "returncode+output-substring" : "returncode-only";
13922
- const backend = options.backend ?? sandboxReplayBackend({
13923
- image: options.image,
13924
- apiKey: requireOption(options.apiKey, "apiKey"),
13925
- baseUrl: requireOption(options.baseUrl, "baseUrl"),
13926
- maxLifetimeSeconds: options.maxLifetimeSeconds
13927
- });
13928
- const runnerOptions = {
13929
- cwd: options.cwd,
13930
- stepTimeoutMs: options.stepTimeoutMs ?? 3e5,
13931
- prefixLimit: options.prefixLimit,
13932
- onProgress: options.onProgress
13933
- };
13934
- const store = new InMemoryTraceStore2();
13935
- const { runId } = await ingestCodeTraceBenchSteps(store, steps, caseId);
13936
- options.onProgress?.(`arm A: replaying ${index} prefix steps then recorded step ${options.at}`);
13937
- const armARunner = new SandboxCounterfactualRunner(backend, runnerOptions);
13938
- const armAStart = Date.now();
13939
- const armAResult = await runCounterfactual(
13940
- store,
13941
- runId,
13942
- { kind: "custom", at: index, describe: "arm-A identity replay", apply: (s) => s },
13943
- armARunner
13944
- );
13945
- const armAMs = Date.now() - armAStart;
13946
- const armAExec = requireExec(armARunner, "arm A");
13947
- const armAOutput = `${armAExec.stdout}
13948
- ${armAExec.stderr}`;
13949
- const failureSignatureMatch = recordedReturncode !== null && armAExec.exitCode === recordedReturncode && (signature ? armAOutput.includes(signature) : true);
13950
- let armBRunner = null;
13951
- let armBExec = null;
13952
- let armBRunId = null;
13953
- let armBMs = null;
13954
- if (options.fixCommand) {
13955
- const fixCommand = options.fixCommand;
13956
- options.onProgress?.(`arm B: replaying prefix then corrected step`);
13957
- armBRunner = new SandboxCounterfactualRunner(backend, runnerOptions);
13958
- const armBStart = Date.now();
13959
- const armBResult = await runCounterfactual(
13960
- store,
13961
- runId,
13962
- {
13963
- kind: "custom",
13964
- at: index,
13965
- describe: "arm-B corrected step",
13966
- apply: (step) => ({
13967
- ...step,
13968
- span: { ...step.span, args: { command: fixCommand } }
13969
- })
13970
- },
13971
- armBRunner
13909
+ if (!response.ok) {
13910
+ throw new Error(
13911
+ `verify-findings: sandbox API at ${baseUrl} answered /health with HTTP ${response.status} \u2014 refusing to run proofs against degraded infrastructure`
13972
13912
  );
13973
- armBMs = Date.now() - armBStart;
13974
- armBRunId = armBResult.counterfactualRunId;
13975
- armBExec = requireExec(armBRunner, "arm B");
13976
- }
13977
- const armBOutput = armBExec ? `${armBExec.stdout}
13978
- ${armBExec.stderr}` : null;
13979
- const verdict = {
13980
- case: caseId,
13981
- image: options.image,
13982
- driver: options.driverLabel ?? "docker",
13983
- k: options.at,
13984
- cwd: options.cwd,
13985
- recordedReturncode,
13986
- signature,
13987
- signatureBasis,
13988
- prefixExecuted: armARunner.lastPrefix?.prefixExecuted ?? 0,
13989
- prefixDivergences: armARunner.lastPrefix?.prefixDivergences ?? [],
13990
- armA: {
13991
- command: armAExec.command,
13992
- exitCode: armAExec.exitCode,
13993
- wallMs: armAExec.wallMs,
13994
- prefix: armARunner.lastPrefix,
13995
- failureSignatureMatch
13996
- },
13997
- armB: armBExec && armBRunner ? {
13998
- command: armBExec.command,
13999
- exitCode: armBExec.exitCode,
14000
- wallMs: armBExec.wallMs,
14001
- prefix: armBRunner.lastPrefix,
14002
- failureVanished: armBExec.exitCode === 0 && (signature && armBOutput ? !armBOutput.includes(signature) : true)
14003
- } : null,
14004
- timings: { armAMs, armBMs, totalMs: Date.now() - totalStart },
14005
- runIds: {
14006
- original: runId,
14007
- armA: armAResult.counterfactualRunId,
14008
- armB: armBRunId
14009
- }
14010
- };
14011
- mkdirSync(options.out, { recursive: true });
14012
- writeFileSync(join24(options.out, "replay-verdict.json"), `${JSON.stringify(verdict, null, 2)}
14013
- `);
14014
- writeFileSync(
14015
- join24(options.out, "report.md"),
14016
- renderReport2(verdict, target, armAExec, armBExec)
14017
- );
14018
- return verdict;
14019
- }
14020
- function requireOption(value, name) {
14021
- if (value === void 0) throw new Error(`replay-verify: ${name} is required`);
14022
- return value;
14023
- }
14024
- function requireExec(runner, arm) {
14025
- if (!runner.lastArm) throw new Error(`replay-verify: ${arm} finished without executing step k`);
14026
- return runner.lastArm;
13913
+ }
14027
13914
  }
14028
- function renderReport2(verdict, target, armA, armB) {
14029
- const lines = [];
14030
- const divergencePct = verdict.prefixExecuted > 0 ? (verdict.prefixDivergences.length / verdict.prefixExecuted * 100).toFixed(1) : "0.0";
14031
- lines.push(`# Replay verification \u2014 ${verdict.case}`);
14032
- lines.push("");
14033
- lines.push(`- image: \`${verdict.image}\` (driver: ${verdict.driver})`);
14034
- lines.push(`- error-critical step k = ${verdict.k}, cwd \`${verdict.cwd}\``);
14035
- lines.push(`- recorded returncode at k: ${verdict.recordedReturncode ?? "none recorded"}`);
14036
- lines.push(
14037
- `- failure signature (${verdict.signatureBasis}): ${verdict.signature ? `\`${verdict.signature}\`` : "\u2014"}`
14038
- );
14039
- lines.push("");
14040
- lines.push(`## Prefix replay (steps 1..${verdict.k - 1})`);
14041
- lines.push("");
14042
- lines.push(
14043
- `Executed ${verdict.prefixExecuted} steps; ${verdict.prefixDivergences.length} returncode divergences (${divergencePct}%).`
14044
- );
14045
- if (verdict.prefixDivergences.length > 0) {
14046
- lines.push("");
14047
- lines.push("| step | recorded rc | replayed exit |");
14048
- lines.push("| --- | --- | --- |");
14049
- for (const d of verdict.prefixDivergences) {
14050
- lines.push(`| ${d.step} | ${d.expectedReturncode} | ${d.actualExit} |`);
13915
+ function zaiChatCaller(options) {
13916
+ async function callOnce(system, user, maxTokens) {
13917
+ try {
13918
+ const response = await fetch(`${options.baseUrl.replace(/\/$/, "")}/chat/completions`, {
13919
+ method: "POST",
13920
+ headers: { authorization: `Bearer ${options.apiKey}`, "content-type": "application/json" },
13921
+ body: JSON.stringify({
13922
+ model: options.model,
13923
+ messages: [
13924
+ { role: "system", content: system },
13925
+ { role: "user", content: user }
13926
+ ],
13927
+ temperature: 0,
13928
+ max_tokens: maxTokens
13929
+ }),
13930
+ signal: AbortSignal.timeout(options.timeoutMs ?? 18e4)
13931
+ });
13932
+ const bodyText = await response.text();
13933
+ if (!response.ok) return { succeeded: false, error: `HTTP ${response.status}: ${bodyText.slice(0, 400)}` };
13934
+ const body = JSON.parse(bodyText);
13935
+ if (body.error) return { succeeded: false, error: String(body.error.message ?? "provider error") };
13936
+ const content = body.choices?.[0]?.message?.content;
13937
+ if (typeof content !== "string" || content.length === 0) {
13938
+ return { succeeded: false, error: `empty completion content: ${bodyText.slice(0, 400)}` };
13939
+ }
13940
+ const promptTokens = body.usage?.prompt_tokens;
13941
+ const completionTokens = body.usage?.completion_tokens;
13942
+ const usage = typeof promptTokens === "number" && typeof completionTokens === "number" ? { promptTokens, completionTokens } : null;
13943
+ return { succeeded: true, value: { content, usage } };
13944
+ } catch (err) {
13945
+ return { succeeded: false, error: err instanceof Error ? err.message : String(err) };
14051
13946
  }
14052
13947
  }
14053
- lines.push("");
14054
- lines.push("## Arm A \u2014 recorded step k, replayed");
14055
- lines.push("");
14056
- lines.push("```sh");
14057
- lines.push(armA.command);
14058
- lines.push("```");
14059
- lines.push("");
14060
- lines.push(
14061
- `Exit ${armA.exitCode} in ${armA.wallMs}ms \u2014 failureSignatureMatch: **${verdict.armA.failureSignatureMatch}**`
14062
- );
14063
- lines.push("");
14064
- lines.push("Recorded observation excerpt:");
14065
- lines.push("");
14066
- lines.push("```");
14067
- lines.push(excerpt(parseObservationOutput(target.observation)));
14068
- lines.push("```");
14069
- lines.push("");
14070
- lines.push("Replayed stdout+stderr excerpt:");
14071
- lines.push("");
14072
- lines.push("```");
14073
- lines.push(excerpt(`${armA.stdout}
14074
- ${armA.stderr}`.trim()));
14075
- lines.push("```");
14076
- if (armB && verdict.armB) {
14077
- lines.push("");
14078
- lines.push("## Arm B \u2014 corrected step k");
14079
- lines.push("");
14080
- lines.push("```sh");
14081
- lines.push(armB.command);
14082
- lines.push("```");
14083
- lines.push("");
14084
- lines.push(
14085
- `Exit ${armB.exitCode} in ${armB.wallMs}ms \u2014 failureVanished: **${verdict.armB.failureVanished}** (prefix re-replayed in a fresh sandbox: ${verdict.armB.prefix.prefixExecuted} steps, ${verdict.armB.prefix.prefixDivergences.length} divergences)`
14086
- );
14087
- lines.push("");
14088
- lines.push("Replayed stdout+stderr excerpt:");
14089
- lines.push("");
14090
- lines.push("```");
14091
- lines.push(excerpt(`${armB.stdout}
14092
- ${armB.stderr}`.trim()));
14093
- lines.push("```");
14094
- }
14095
- lines.push("");
14096
- lines.push("## Timings");
14097
- lines.push("");
14098
- lines.push(
14099
- `arm A ${verdict.timings.armAMs}ms, arm B ${verdict.timings.armBMs ?? "\u2014"}ms, total ${verdict.timings.totalMs}ms.`
14100
- );
14101
- lines.push("");
14102
- return lines.join("\n");
13948
+ return {
13949
+ async complete(system, user) {
13950
+ const baseCap = options.maxTokens ?? 16384;
13951
+ const first = await callOnce(system, user, baseCap);
13952
+ if (first.succeeded || !/finish_reason":"length/.test(first.error)) return first;
13953
+ return callOnce(system, user, baseCap * 2);
13954
+ }
13955
+ };
14103
13956
  }
14104
- function parseReplayVerifyArgs(argv) {
13957
+ function parseFlags2(command, argv, switches = []) {
14105
13958
  const values = /* @__PURE__ */ new Map();
13959
+ const corpora = [];
13960
+ const seen = /* @__PURE__ */ new Set();
14106
13961
  for (let i = 0; i < argv.length; i++) {
14107
13962
  const arg = argv[i];
14108
13963
  if (arg === "--help" || arg === "-h") return "help";
14109
- if (!arg.startsWith("--")) throw new Error(`unexpected positional argument: ${arg}`);
13964
+ if (switches.includes(arg)) {
13965
+ seen.add(arg);
13966
+ continue;
13967
+ }
13968
+ if (!arg.startsWith("--")) throw new Error(`${command}: unexpected positional argument: ${arg}`);
14110
13969
  const value = argv[++i];
14111
- if (value === void 0) throw new Error(`missing value for ${arg}`);
14112
- values.set(arg, value);
13970
+ if (value === void 0) throw new Error(`${command}: missing value for ${arg}`);
13971
+ if (arg === "--corpus") corpora.push(parseCorpusFlag(value));
13972
+ else values.set(arg, value);
14113
13973
  }
14114
- const required = (flag) => {
14115
- const v = values.get(flag);
14116
- if (v === void 0) throw new Error(`replay-verify: ${flag} is required`);
14117
- return v;
14118
- };
14119
- const optionalNumber = (flag) => {
14120
- const v = values.get(flag);
14121
- if (v === void 0) return void 0;
14122
- const n = Number(v);
14123
- if (!Number.isFinite(n)) throw new Error(`replay-verify: ${flag} must be a number, got ${v}`);
14124
- return n;
13974
+ return { values, corpora, switches: seen };
13975
+ }
13976
+ function flagReader(command, values) {
13977
+ return {
13978
+ required(flag) {
13979
+ const v = values.get(flag);
13980
+ if (v === void 0) throw new Error(`${command}: ${flag} is required`);
13981
+ return v;
13982
+ },
13983
+ number(flag) {
13984
+ const v = values.get(flag);
13985
+ if (v === void 0) return void 0;
13986
+ const n = Number(v);
13987
+ if (!Number.isFinite(n)) throw new Error(`${command}: ${flag} must be a number, got ${v}`);
13988
+ return n;
13989
+ }
14125
13990
  };
14126
- const stepsPath = required("--steps");
14127
- const image = required("--image");
14128
- const at = Number(required("--at"));
14129
- const cwd = required("--cwd");
14130
- const out = required("--out");
14131
- if (!Number.isInteger(at) || at < 1) {
14132
- throw new Error(`replay-verify: --at must be a positive step_id, got ${values.get("--at")}`);
14133
- }
14134
- let fixCommand = values.get("--fix-command");
14135
- const fixFile = values.get("--fix-command-file");
14136
- if (fixCommand !== void 0 && fixFile !== void 0) {
14137
- throw new Error("replay-verify: pass --fix-command or --fix-command-file, not both");
13991
+ }
13992
+ function sandboxOptions(command, values, requireKey) {
13993
+ const apiKeyEnv = values.get("--api-key-env") ?? "SANDBOX_API_KEY";
13994
+ const apiKey = process.env[apiKeyEnv];
13995
+ if (requireKey && !apiKey) {
13996
+ throw new Error(`${command}: env var ${apiKeyEnv} is empty \u2014 pass --api-key-env or export it`);
14138
13997
  }
14139
- if (fixFile !== void 0) fixCommand = readFileSync3(fixFile, "utf8").trimEnd();
13998
+ const maxLifetime = flagReader(command, values).number("--max-lifetime");
14140
13999
  return {
14141
- stepsPath,
14142
- image,
14143
- at,
14144
- fixCommand,
14145
- cwd,
14146
- out,
14147
- caseId: values.get("--case"),
14148
- signature: values.get("--signature"),
14149
- stepTimeoutMs: optionalNumber("--step-timeout"),
14150
- prefixLimit: optionalNumber("--prefix-limit"),
14151
- driverLabel: values.get("--driver-label"),
14152
- baseUrl: values.get("--base-url") ?? process.env.SANDBOX_API_URL ?? "http://127.0.0.1:4097",
14153
- apiKeyEnv: values.get("--api-key-env") ?? "SANDBOX_API_KEY",
14154
- maxLifetimeSeconds: optionalNumber("--max-lifetime")
14000
+ apiKey: apiKey ?? "",
14001
+ baseUrl: values.get("--base-url") ?? process.env.SANDBOX_API_URL ?? DEFAULT_SANDBOX_BASE_URL,
14002
+ ...maxLifetime === void 0 ? {} : { maxLifetimeSeconds: maxLifetime }
14155
14003
  };
14156
14004
  }
14157
- function replayVerifyUsage() {
14158
- return `traces replay-verify \u2014 replay a trajectory prefix in a sandbox and execute the counterfactual at step k
14005
+ var progress = (message) => process.stderr.write(`${message}
14006
+ `);
14007
+ var REPLAY_VERIFY_USAGE = `traces replay-verify \u2014 replay a trajectory prefix in a sandbox and execute the counterfactual at step k
14159
14008
 
14160
14009
  Usage:
14161
14010
  traces replay-verify --steps STEPS.json --image IMG --at K --cwd DIR --out DIR \\
@@ -14173,1928 +14022,153 @@ Usage:
14173
14022
  line of the recorded observation containing "error")
14174
14023
  --prefix-limit execute at most N prefix steps (weakens fidelity; reported)
14175
14024
  --step-timeout per-exec timeout in ms (default 300000)
14176
- --base-url sandbox API url (default $SANDBOX_API_URL or http://127.0.0.1:4097)
14025
+ --base-url sandbox API url (default $SANDBOX_API_URL or ${DEFAULT_SANDBOX_BASE_URL})
14177
14026
  --api-key-env env var holding the sandbox API key (default SANDBOX_API_KEY)
14178
14027
 
14179
14028
  Requires a running sandbox orchestrator; only SWE-style trajectories that carry a
14180
14029
  docker image are replayable. See docs/replay-verify.md for setup and limits.
14181
14030
  `;
14182
- }
14183
14031
  async function cmdReplayVerify(argv) {
14184
- const parsed = parseReplayVerifyArgs(argv);
14032
+ const parsed = parseFlags2("replay-verify", argv);
14185
14033
  if (parsed === "help") {
14186
- process.stdout.write(replayVerifyUsage());
14034
+ process.stdout.write(REPLAY_VERIFY_USAGE);
14187
14035
  return;
14188
14036
  }
14189
- const apiKey = process.env[parsed.apiKeyEnv];
14190
- if (!apiKey) {
14191
- throw new Error(
14192
- `replay-verify: env var ${parsed.apiKeyEnv} is empty \u2014 pass --api-key-env or export it`
14193
- );
14037
+ const { values } = parsed;
14038
+ const flags = flagReader("replay-verify", values);
14039
+ const at = Number(flags.required("--at"));
14040
+ if (!Number.isInteger(at) || at < 1) {
14041
+ throw new Error(`replay-verify: --at must be a positive step_id, got ${values.get("--at")}`);
14042
+ }
14043
+ const fixFile = values.get("--fix-command-file");
14044
+ if (values.has("--fix-command") && fixFile !== void 0) {
14045
+ throw new Error("replay-verify: pass --fix-command or --fix-command-file, not both");
14194
14046
  }
14047
+ const fixCommand = fixFile === void 0 ? values.get("--fix-command") : readFileSync3(fixFile, "utf8").trimEnd();
14048
+ const image = flags.required("--image");
14049
+ const out = flags.required("--out");
14195
14050
  const verdict = await replayVerify({
14196
- stepsPath: parsed.stepsPath,
14197
- image: parsed.image,
14198
- at: parsed.at,
14199
- fixCommand: parsed.fixCommand,
14200
- cwd: parsed.cwd,
14201
- out: parsed.out,
14202
- caseId: parsed.caseId,
14203
- signature: parsed.signature,
14204
- stepTimeoutMs: parsed.stepTimeoutMs,
14205
- prefixLimit: parsed.prefixLimit,
14206
- driverLabel: parsed.driverLabel,
14207
- baseUrl: parsed.baseUrl,
14208
- apiKey,
14209
- maxLifetimeSeconds: parsed.maxLifetimeSeconds,
14210
- onProgress: (message) => process.stderr.write(`${message}
14211
- `)
14051
+ stepsPath: flags.required("--steps"),
14052
+ image,
14053
+ at,
14054
+ fixCommand,
14055
+ cwd: flags.required("--cwd"),
14056
+ out,
14057
+ caseId: values.get("--case"),
14058
+ signature: values.get("--signature"),
14059
+ stepTimeoutMs: flags.number("--step-timeout"),
14060
+ prefixLimit: flags.number("--prefix-limit"),
14061
+ driverLabel: values.get("--driver-label"),
14062
+ backend: sandboxReplayBackend({ ...sandboxOptions("replay-verify", values, true), image }),
14063
+ onProgress: progress
14212
14064
  });
14213
14065
  process.stdout.write(`${JSON.stringify(verdict, null, 2)}
14214
14066
  `);
14215
14067
  const verdictLine = verdict.armB ? `armA.failureSignatureMatch=${verdict.armA.failureSignatureMatch} armB.failureVanished=${verdict.armB.failureVanished}` : `armA.failureSignatureMatch=${verdict.armA.failureSignatureMatch}`;
14216
- process.stderr.write(
14217
- `replay-verify: ${verdictLine} (${verdict.prefixDivergences.length}/${verdict.prefixExecuted} prefix divergences) \u2192 ${parsed.out}
14218
- `
14068
+ progress(
14069
+ `replay-verify: ${verdictLine} (${verdict.prefixDivergences.length}/${verdict.prefixExecuted} prefix divergences) \u2192 ${out}`
14219
14070
  );
14220
14071
  }
14072
+ var REPLAY_BATCH_USAGE = `traces replay-verify-batch \u2014 measure replayability and fix-flip rates across gold-labeled corpora
14221
14073
 
14222
- // src/replay-corpus.ts
14223
- import { existsSync as existsSync2, readdirSync as readdirSync2, readFileSync as readFileSync4 } from "fs";
14224
- import { join as join25 } from "path";
14225
- function parseCorpusFlag(value) {
14226
- const eq = value.indexOf("=");
14227
- if (eq <= 0) throw new Error(`--corpus must be name=<labels>::<prepared>, got: ${value}`);
14228
- const name = value.slice(0, eq);
14229
- const rest = value.slice(eq + 1);
14230
- const sep4 = rest.indexOf("::");
14231
- if (sep4 <= 0 || sep4 === rest.length - 2) {
14232
- throw new Error(`--corpus must be name=<labels>::<prepared>, got: ${value}`);
14074
+ Usage:
14075
+ traces replay-verify-batch \\
14076
+ --corpus NAME=<labels.json>::<prepared-dir> [--corpus ...] \\
14077
+ --out DIR [--fix none|generate|loop] [--fix-attempts 3] [--enumerate-only] \\
14078
+ [--fix-model glm-5.2] [--fix-base-url URL] [--fix-api-key-env ZAI_GLM_API_KEY] \\
14079
+ [--max-fix-cases 30] [--seed 17] [--step-timeout MS] [--prefix-limit N] \\
14080
+ [--case-filter SUBSTRING] [--case-limit N] \\
14081
+ [--base-url URL] [--api-key-env SANDBOX_API_KEY] [--max-lifetime SECONDS]
14082
+
14083
+ Replayable = the raw trajectory carries info.docker_config.base_image AND the
14084
+ labels mark at least one gold incorrect step that is a real mid-trajectory
14085
+ action. Excluded cases and image pull failures are report rows, never
14086
+ skipped silently. --enumerate-only prints the enumeration without touching a
14087
+ sandbox.
14088
+
14089
+ --fix generate runs ONE LLM call per arm-A-reproduced case (cap
14090
+ --max-fix-cases, seeded sample beyond it) and executes the corrected command
14091
+ as arm B in a fresh sandbox. --fix loop feeds each failed attempt's REAL
14092
+ executed output into the next prompt, up to --fix-attempts attempts.
14093
+
14094
+ Outputs batch-report.json, batch-report.md, cases.jsonl (incremental), and one
14095
+ directory per case with the full replay-verify artifacts.
14096
+ See docs/replay-verify.md for orchestrator setup and honest limits.
14097
+ `;
14098
+ async function cmdReplayVerifyBatch(argv) {
14099
+ const parsed = parseFlags2("replay-verify-batch", argv, ["--enumerate-only"]);
14100
+ if (parsed === "help") {
14101
+ process.stdout.write(REPLAY_BATCH_USAGE);
14102
+ return;
14233
14103
  }
14234
- return { name, labelsPath: rest.slice(0, sep4), preparedDir: rest.slice(sep4 + 2) };
14235
- }
14236
- var SUBMIT_ACTION_SIGNATURE = "COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT";
14237
- function isSubmitAction(action) {
14238
- return action.includes(SUBMIT_ACTION_SIGNATURE);
14239
- }
14240
- function readLabelEntries(labelsPath) {
14241
- const parsed = JSON.parse(readFileSync4(labelsPath, "utf8"));
14242
- if (!Array.isArray(parsed)) throw new Error(`${labelsPath} is not a JSON array of label entries`);
14243
- for (const entry of parsed) {
14244
- if (typeof entry.traj_id !== "string") {
14245
- throw new Error(`${labelsPath} has an entry without a string traj_id`);
14246
- }
14104
+ const { values, corpora } = parsed;
14105
+ const flags = flagReader("replay-verify-batch", values);
14106
+ if (corpora.length === 0) {
14107
+ throw new Error("replay-verify-batch: at least one --corpus name=<labels>::<prepared> is required");
14247
14108
  }
14248
- return parsed;
14249
- }
14250
- function goldIncorrectSteps(entry) {
14251
- const ids = /* @__PURE__ */ new Set();
14252
- for (const stage of entry.incorrect_stages ?? []) {
14253
- for (const id of stage.incorrect_step_ids ?? []) ids.add(id);
14109
+ if (parsed.switches.has("--enumerate-only")) {
14110
+ const enumeration = enumerateReplayableCases(corpora);
14111
+ const replayable = enumeration.replayable.map((c) => ({
14112
+ corpus: c.corpus,
14113
+ trajId: c.trajId,
14114
+ image: c.image,
14115
+ cwd: c.cwd,
14116
+ cwdSource: c.cwdSource,
14117
+ k: c.k,
14118
+ submitGoldsSkipped: c.submitGoldsSkipped,
14119
+ recordedReturncodeAtK: c.recordedReturncodeAtK,
14120
+ goldIncorrectSteps: c.goldIncorrectSteps
14121
+ }));
14122
+ const report2 = { labelEntries: enumeration.labelEntryCount, replayable, excluded: enumeration.excluded };
14123
+ process.stdout.write(`${JSON.stringify(report2, null, 2)}
14124
+ `);
14125
+ return;
14254
14126
  }
14255
- return [...ids].sort((a, b) => a - b);
14256
- }
14257
- function findRawTrajFiles(sweRawDir) {
14258
- if (!existsSync2(sweRawDir)) return [];
14259
- const entries = readdirSync2(sweRawDir, { recursive: true, encoding: "utf8" });
14260
- return entries.filter((relative7) => relative7.endsWith(".traj.json")).map((relative7) => join25(sweRawDir, relative7)).sort();
14261
- }
14262
- function cwdFromPwdObservation(steps) {
14263
- for (const step of steps) {
14264
- if (step.action.trim() !== "pwd") continue;
14265
- const line = parseObservationOutput(step.observation).split("\n").map((l) => l.trim()).find((l) => l.startsWith("/"));
14266
- if (line) return line;
14127
+ const out = flags.required("--out");
14128
+ const fix = values.get("--fix") ?? "none";
14129
+ if (fix !== "none" && fix !== "generate" && fix !== "loop") {
14130
+ throw new Error(`replay-verify-batch: --fix must be none, generate, or loop, got ${fix}`);
14267
14131
  }
14268
- return null;
14269
- }
14270
- function taskStatementOf(preparedDir, trajId, raw) {
14271
- const taskPath = join25(preparedDir, "normalized", trajId, "task.md");
14272
- if (existsSync2(taskPath)) {
14273
- const text2 = readFileSync4(taskPath, "utf8").trim();
14274
- if (text2.length > 0) return text2;
14132
+ const fixAttempts = flags.number("--fix-attempts") ?? 3;
14133
+ if (!Number.isInteger(fixAttempts) || fixAttempts < 1) {
14134
+ throw new Error(`replay-verify-batch: --fix-attempts must be a positive integer, got ${fixAttempts}`);
14275
14135
  }
14276
- const userMessage = raw.messages?.find((m) => m.role === "user");
14277
- if (typeof userMessage?.content === "string" && userMessage.content.trim().length > 0) {
14278
- return userMessage.content.trim();
14136
+ const fixModel = values.get("--fix-model") ?? "glm-5.2";
14137
+ let fixCaller;
14138
+ if (fix !== "none") {
14139
+ const fixApiKeyEnv = values.get("--fix-api-key-env") ?? "ZAI_GLM_API_KEY";
14140
+ const fixApiKey = process.env[fixApiKeyEnv];
14141
+ if (!fixApiKey) {
14142
+ throw new Error(`replay-verify-batch: env var ${fixApiKeyEnv} is empty \u2014 pass --fix-api-key-env or export it`);
14143
+ }
14144
+ fixCaller = zaiChatCaller({
14145
+ baseUrl: values.get("--fix-base-url") ?? "https://api.z.ai/api/coding/paas/v4",
14146
+ apiKey: fixApiKey,
14147
+ model: fixModel
14148
+ });
14279
14149
  }
14280
- return null;
14150
+ const report = await runReplayBatch({
14151
+ corpora,
14152
+ out,
14153
+ fix,
14154
+ fixAttempts,
14155
+ fixCaller,
14156
+ fixModelLabel: fixModel,
14157
+ maxFixCases: flags.number("--max-fix-cases") ?? 30,
14158
+ seed: flags.number("--seed") ?? 17,
14159
+ stepTimeoutMs: flags.number("--step-timeout"),
14160
+ prefixLimit: flags.number("--prefix-limit"),
14161
+ caseFilter: values.get("--case-filter"),
14162
+ caseLimit: flags.number("--case-limit"),
14163
+ backendFactory: sandboxBackendFactory(sandboxOptions("replay-verify-batch", values, true)),
14164
+ onProgress: progress
14165
+ });
14166
+ const { replayabilityRate, fixFlipRate } = report.headline;
14167
+ progress(
14168
+ `replay-verify-batch: replayability ${replayabilityRate.numerator}/${replayabilityRate.denominator}` + (fixFlipRate ? `, fix-flip ${fixFlipRate.numerator}/${fixFlipRate.denominator}` : "") + ` \u2192 ${out}`
14169
+ );
14281
14170
  }
14282
- function resolveCaseResources(corpus, trajId) {
14283
- const sweRawDir = join25(corpus.preparedDir, "extracted", trajId, "swe_raw");
14284
- const rawFiles = findRawTrajFiles(sweRawDir);
14285
- if (rawFiles.length === 0) {
14286
- return { resolved: false, reason: "no-swe-raw-trajectory", detail: sweRawDir };
14287
- }
14288
- if (rawFiles.length > 1) {
14289
- return { resolved: false, reason: "ambiguous-swe-raw-trajectory", detail: rawFiles.join(", ") };
14290
- }
14291
- let raw;
14292
- try {
14293
- raw = JSON.parse(readFileSync4(rawFiles[0], "utf8"));
14294
- } catch (err) {
14295
- return {
14296
- resolved: false,
14297
- reason: "unreadable-raw-trajectory",
14298
- detail: `${rawFiles[0]}: ${err instanceof Error ? err.message : String(err)}`
14299
- };
14300
- }
14301
- const dockerConfig = raw.info?.docker_config;
14302
- const image = dockerConfig?.base_image;
14303
- if (typeof image !== "string" || image.length === 0) {
14304
- return { resolved: false, reason: "no-docker-image", detail: rawFiles[0] };
14305
- }
14306
- const stepsPath = join25(corpus.preparedDir, "normalized", trajId, "steps.json");
14307
- if (!existsSync2(stepsPath)) {
14308
- return { resolved: false, reason: "missing-steps-json", detail: stepsPath };
14309
- }
14310
- const steps = JSON.parse(readFileSync4(stepsPath, "utf8"));
14311
- const runConfigCwd = raw.info?.config?.environment?.cwd;
14312
- const dockerCwd = dockerConfig?.cwd;
14313
- let cwd;
14314
- let cwdSource;
14315
- if (typeof runConfigCwd === "string" && runConfigCwd.length > 0) {
14316
- cwd = runConfigCwd;
14317
- cwdSource = "run-config";
14318
- } else if (typeof dockerCwd === "string" && dockerCwd.length > 0) {
14319
- cwd = dockerCwd;
14320
- cwdSource = "docker-config";
14321
- } else {
14322
- const observed = cwdFromPwdObservation(steps);
14323
- if (!observed) return { resolved: false, reason: "cwd-underivable" };
14324
- cwd = observed;
14325
- cwdSource = "pwd-observation";
14326
- }
14327
- const recordedTimeout = raw.info?.config?.environment?.timeout;
14328
- return {
14329
- resolved: true,
14330
- resources: {
14331
- corpus: corpus.name,
14332
- trajId,
14333
- stepsPath,
14334
- steps,
14335
- taskStatement: taskStatementOf(corpus.preparedDir, trajId, raw),
14336
- image,
14337
- cwd,
14338
- cwdSource,
14339
- recordedStepTimeoutMs: typeof recordedTimeout === "number" && recordedTimeout > 0 ? recordedTimeout * 1e3 : null
14340
- }
14341
- };
14342
- }
14343
- function enumerateReplayableCases(corpora) {
14344
- const replayable = [];
14345
- const excluded = [];
14346
- let labelEntryCount = 0;
14347
- for (const corpus of corpora) {
14348
- for (const entry of readLabelEntries(corpus.labelsPath)) {
14349
- labelEntryCount += 1;
14350
- const trajId = entry.traj_id;
14351
- const resolution = resolveCaseResources(corpus, trajId);
14352
- if (!resolution.resolved) {
14353
- excluded.push({ corpus: corpus.name, trajId, reason: resolution.reason, detail: resolution.detail });
14354
- continue;
14355
- }
14356
- const gold = goldIncorrectSteps(entry);
14357
- if (gold.length === 0) {
14358
- excluded.push({ corpus: corpus.name, trajId, reason: "no-gold-incorrect-step" });
14359
- continue;
14360
- }
14361
- const resources = resolution.resources;
14362
- let submitGoldsSkipped = 0;
14363
- let target = null;
14364
- let missingGoldId = null;
14365
- for (const goldId of gold) {
14366
- const step = resources.steps.find((s) => s.step_id === goldId);
14367
- if (!step) {
14368
- missingGoldId = goldId;
14369
- break;
14370
- }
14371
- if (isSubmitAction(step.action)) {
14372
- submitGoldsSkipped += 1;
14373
- continue;
14374
- }
14375
- target = step;
14376
- break;
14377
- }
14378
- if (missingGoldId !== null) {
14379
- excluded.push({
14380
- corpus: corpus.name,
14381
- trajId,
14382
- reason: "gold-step-outside-steps",
14383
- detail: `k=${missingGoldId}, steps=${resources.steps.length}`
14384
- });
14385
- continue;
14386
- }
14387
- if (!target) {
14388
- excluded.push({
14389
- corpus: corpus.name,
14390
- trajId,
14391
- reason: "gold-only-submit-step",
14392
- detail: `${submitGoldsSkipped} gold step(s), all submit commands`
14393
- });
14394
- continue;
14395
- }
14396
- replayable.push({
14397
- ...resources,
14398
- goldIncorrectSteps: gold,
14399
- k: target.step_id,
14400
- submitGoldsSkipped,
14401
- recordedReturncodeAtK: parseRecordedReturncode(target.observation)
14402
- });
14403
- }
14404
- }
14405
- return { replayable, excluded, labelEntryCount };
14406
- }
14407
-
14408
- // src/replay-fix.ts
14409
- function zaiChatCaller(options) {
14410
- const fetchImpl = options.fetchImpl ?? fetch;
14411
- return {
14412
- async complete(system, user) {
14413
- const baseCap = options.maxTokens ?? 16384;
14414
- let lastError = "empty completion content";
14415
- for (const cap of [baseCap, baseCap * 2]) {
14416
- const outcome = await callOnce(cap);
14417
- if (outcome.succeeded) return outcome;
14418
- lastError = outcome.error;
14419
- if (!/finish_reason":"length/.test(outcome.error)) return outcome;
14420
- }
14421
- return { succeeded: false, error: lastError };
14422
- async function callOnce(maxTokens) {
14423
- const controller = new AbortController();
14424
- const timer = setTimeout(() => controller.abort(), options.timeoutMs ?? 18e4);
14425
- try {
14426
- const response = await fetchImpl(`${options.baseUrl.replace(/\/$/, "")}/chat/completions`, {
14427
- method: "POST",
14428
- headers: {
14429
- authorization: `Bearer ${options.apiKey}`,
14430
- "content-type": "application/json"
14431
- },
14432
- body: JSON.stringify({
14433
- model: options.model,
14434
- messages: [
14435
- { role: "system", content: system },
14436
- { role: "user", content: user }
14437
- ],
14438
- temperature: 0,
14439
- max_tokens: maxTokens
14440
- }),
14441
- signal: controller.signal
14442
- });
14443
- const bodyText = await response.text();
14444
- if (!response.ok) {
14445
- return { succeeded: false, error: `HTTP ${response.status}: ${bodyText.slice(0, 400)}` };
14446
- }
14447
- const body = JSON.parse(bodyText);
14448
- if (body.error) {
14449
- return { succeeded: false, error: String(body.error.message ?? "provider error") };
14450
- }
14451
- const content = body.choices?.[0]?.message?.content;
14452
- if (typeof content !== "string" || content.length === 0) {
14453
- return { succeeded: false, error: `empty completion content: ${bodyText.slice(0, 400)}` };
14454
- }
14455
- const promptTokens = body.usage?.prompt_tokens;
14456
- const completionTokens = body.usage?.completion_tokens;
14457
- const usage = typeof promptTokens === "number" && typeof completionTokens === "number" ? { promptTokens, completionTokens } : null;
14458
- return { succeeded: true, value: { content, usage } };
14459
- } catch (err) {
14460
- return { succeeded: false, error: err instanceof Error ? err.message : String(err) };
14461
- } finally {
14462
- clearTimeout(timer);
14463
- }
14464
- }
14465
- }
14466
- };
14467
- }
14468
- function clipText(text2, limit) {
14469
- if (text2.length <= limit) return text2;
14470
- const half = Math.floor(limit / 2);
14471
- return `${text2.slice(0, half)}
14472
- \u2026 [${text2.length - limit} chars elided] \u2026
14473
- ${text2.slice(-half)}`;
14474
- }
14475
- function renderStep(step, marker) {
14476
- const rc = parseRecordedReturncode(step.observation);
14477
- const output = clipText(parseObservationOutput(step.observation).trim(), 1600);
14478
- return [
14479
- `### step ${step.step_id}${marker}`,
14480
- "```sh",
14481
- clipText(step.action, 2e3),
14482
- "```",
14483
- `returncode: ${rc ?? "none recorded"}`,
14484
- output.length > 0 ? `output:
14485
- \`\`\`
14486
- ${output}
14487
- \`\`\`` : "output: (empty)"
14488
- ].join("\n");
14489
- }
14490
- function promptBody(input) {
14491
- const radius = input.contextRadius ?? 3;
14492
- const target = input.steps.find((s) => s.step_id === input.k);
14493
- if (!target) throw new Error(`replay-fix: no step with step_id ${input.k}`);
14494
- const context = input.steps.filter(
14495
- (s) => s.step_id !== input.k && Math.abs(s.step_id - input.k) <= radius
14496
- );
14497
- return [
14498
- "## Task the agent was solving",
14499
- input.taskStatement ? clipText(input.taskStatement, 4e3) : "(no task statement recorded)",
14500
- "",
14501
- "## Surrounding steps",
14502
- ...context.map((s) => renderStep(s, "")),
14503
- "",
14504
- "## Failing step to correct",
14505
- renderStep(target, " (INCORRECT \u2014 correct this one)")
14506
- ];
14507
- }
14508
- function buildFixPrompt(input) {
14509
- const system = [
14510
- "You repair one failed shell command from a recorded coding-agent trajectory.",
14511
- "The trajectory replays inside the original docker image; every command runs as a fresh /bin/sh subshell from a fixed working directory.",
14512
- "You are given the failing step, its recorded output, surrounding steps, and the task statement.",
14513
- "Reply with exactly ONE corrected shell command (compound commands with && or pipes are fine) inside a single ```sh fenced block.",
14514
- "The corrected command must accomplish the failing step's intent and exit 0. No prose outside the fenced block."
14515
- ].join("\n");
14516
- const user = [
14517
- ...promptBody(input),
14518
- "",
14519
- "Output the single corrected replacement for the failing step now."
14520
- ].join("\n");
14521
- return { system, user };
14522
- }
14523
- function renderFailedAttempt(prior) {
14524
- if (prior.command === null) {
14525
- return [
14526
- `### attempt ${prior.attempt}`,
14527
- `model call failed before producing a command: ${prior.llmError ?? "unknown error"}`
14528
- ].join("\n");
14529
- }
14530
- const stdout = (prior.stdoutTail ?? "").trim();
14531
- const stderr = (prior.stderrTail ?? "").trim();
14532
- return [
14533
- `### attempt ${prior.attempt}`,
14534
- "```sh",
14535
- clipText(prior.command, 2e3),
14536
- "```",
14537
- `exit code: ${prior.exitCode ?? "not executed"}`,
14538
- stdout.length > 0 ? `stdout:
14539
- \`\`\`
14540
- ${stdout}
14541
- \`\`\`` : "stdout: (empty)",
14542
- stderr.length > 0 ? `stderr:
14543
- \`\`\`
14544
- ${stderr}
14545
- \`\`\`` : "stderr: (empty)"
14546
- ].join("\n");
14547
- }
14548
- function buildRetryFixPrompt(input, priorAttempts, maxScriptCommands = 5) {
14549
- if (priorAttempts.length === 0) {
14550
- throw new Error("replay-fix: buildRetryFixPrompt requires at least one prior attempt");
14551
- }
14552
- const system = [
14553
- "You repair one failed shell command from a recorded coding-agent trajectory.",
14554
- "The trajectory replays inside the original docker image; every command runs as a fresh /bin/sh subshell from a fixed working directory.",
14555
- "Earlier corrected commands were executed for real and failed; their actual output is included below.",
14556
- `Reply with a corrected fix inside a single \`\`\`sh fenced block: either one command, or a short script of at most ${maxScriptCommands} commands (one per line).`,
14557
- "The whole block executes as ONE /bin/sh unit from the fixed working directory and must exit 0.",
14558
- "Do not repeat a command that already failed. Keep reasoning brief. No prose outside the fenced block."
14559
- ].join("\n");
14560
- const user = [
14561
- ...promptBody(input),
14562
- "",
14563
- "## Previous fix attempts (executed for real \u2014 all failed)",
14564
- ...priorAttempts.map((prior) => renderFailedAttempt(prior)),
14565
- "",
14566
- `Output a corrected fix now \u2014 one \`\`\`sh block, at most ${maxScriptCommands} commands.`
14567
- ].join("\n");
14568
- return { system, user };
14569
- }
14570
- function countScriptCommands(script) {
14571
- return script.split("\n").filter((line) => line.trim().length > 0 && !line.trim().startsWith("#")).length;
14572
- }
14573
- function extractFixCommand(content) {
14574
- const blocks = [...content.matchAll(/```(?:sh|bash|shell)?\n([\s\S]*?)```/g)];
14575
- const candidate = blocks.length > 0 ? blocks[blocks.length - 1][1] : content;
14576
- const command = candidate.trim();
14577
- if (command.length === 0 || command.includes("```")) return null;
14578
- return command;
14579
- }
14580
- async function generateFixCommand(caller, input) {
14581
- const { system, user } = buildFixPrompt(input);
14582
- const outcome = await caller.complete(system, user);
14583
- if (!outcome.succeeded) return outcome;
14584
- const command = extractFixCommand(outcome.value.content);
14585
- if (command === null) {
14586
- return {
14587
- succeeded: false,
14588
- error: `completion carried no usable command: ${clipText(outcome.value.content, 300)}`
14589
- };
14590
- }
14591
- return { succeeded: true, value: { command, usage: outcome.value.usage } };
14592
- }
14593
-
14594
- // src/replay-fix-loop.ts
14595
- function toFailedAttempt(record2) {
14596
- return {
14597
- attempt: record2.attempt,
14598
- command: record2.command,
14599
- exitCode: record2.exitCode,
14600
- stdoutTail: record2.stdoutTail,
14601
- stderrTail: record2.stderrTail,
14602
- llmError: record2.llmError ?? record2.armBError
14603
- };
14604
- }
14605
- async function runFixLoop(caller, input, executor, options) {
14606
- if (!Number.isInteger(options.maxAttempts) || options.maxAttempts < 1) {
14607
- throw new Error(
14608
- `replay-fix-loop: maxAttempts must be a positive integer, got ${options.maxAttempts}`
14609
- );
14610
- }
14611
- const scriptCap = options.maxScriptCommands ?? 5;
14612
- const tailChars = options.outputTailChars ?? 1600;
14613
- const attempts = [];
14614
- let llmCalls = 0;
14615
- let llmFailures = 0;
14616
- let promptTokens = 0;
14617
- let completionTokens = 0;
14618
- const unexecuted = (attempt, llmError, usage, command, started) => ({
14619
- attempt,
14620
- command,
14621
- llmError,
14622
- usage,
14623
- executed: false,
14624
- exitCode: null,
14625
- prefixExecuted: null,
14626
- prefixDivergences: null,
14627
- failureVanished: null,
14628
- stdoutTail: null,
14629
- stderrTail: null,
14630
- armBError: null,
14631
- wallMs: Date.now() - started
14632
- });
14633
- for (let attempt = 1; attempt <= options.maxAttempts; attempt++) {
14634
- const started = Date.now();
14635
- const prompt = attempt === 1 ? buildFixPrompt(input) : buildRetryFixPrompt(input, attempts.map(toFailedAttempt), scriptCap);
14636
- llmCalls += 1;
14637
- const outcome = await caller.complete(prompt.system, prompt.user);
14638
- if (!outcome.succeeded) {
14639
- llmFailures += 1;
14640
- attempts.push(unexecuted(attempt, outcome.error, null, null, started));
14641
- options.onProgress?.(
14642
- `fix-loop attempt ${attempt}: LLM failed \u2014 ${outcome.error.slice(0, 160)}`
14643
- );
14644
- continue;
14645
- }
14646
- const usage = outcome.value.usage;
14647
- promptTokens += usage?.promptTokens ?? 0;
14648
- completionTokens += usage?.completionTokens ?? 0;
14649
- const command = extractFixCommand(outcome.value.content);
14650
- if (command === null) {
14651
- llmFailures += 1;
14652
- attempts.push(
14653
- unexecuted(
14654
- attempt,
14655
- `completion carried no usable command: ${clipText(outcome.value.content, 300)}`,
14656
- usage,
14657
- null,
14658
- started
14659
- )
14660
- );
14661
- continue;
14662
- }
14663
- if (attempt > 1) {
14664
- const commandLines = countScriptCommands(command);
14665
- if (commandLines > scriptCap) {
14666
- llmFailures += 1;
14667
- attempts.push(
14668
- unexecuted(
14669
- attempt,
14670
- `script exceeds ${scriptCap} command lines (${commandLines})`,
14671
- usage,
14672
- null,
14673
- started
14674
- )
14675
- );
14676
- options.onProgress?.(
14677
- `fix-loop attempt ${attempt}: rejected script with ${commandLines} command lines`
14678
- );
14679
- continue;
14680
- }
14681
- }
14682
- let execution;
14683
- try {
14684
- execution = await executor(command, attempt);
14685
- } catch (err) {
14686
- const message = err instanceof Error ? err.message : String(err);
14687
- attempts.push({
14688
- ...unexecuted(attempt, "", usage, command, started),
14689
- llmError: null,
14690
- armBError: message.slice(0, 500)
14691
- });
14692
- options.onProgress?.(
14693
- `fix-loop attempt ${attempt}: sandbox error \u2014 ${message.slice(0, 160)}`
14694
- );
14695
- return {
14696
- flipped: false,
14697
- flippedAtAttempt: null,
14698
- attempts,
14699
- aborted: true,
14700
- llmCalls,
14701
- llmFailures,
14702
- promptTokens,
14703
- completionTokens
14704
- };
14705
- }
14706
- attempts.push({
14707
- attempt,
14708
- command,
14709
- llmError: null,
14710
- usage,
14711
- executed: true,
14712
- exitCode: execution.exitCode,
14713
- prefixExecuted: execution.prefixExecuted,
14714
- prefixDivergences: execution.prefixDivergences,
14715
- failureVanished: execution.failureVanished,
14716
- stdoutTail: clipText(execution.stdout, tailChars),
14717
- stderrTail: clipText(execution.stderr, tailChars),
14718
- armBError: null,
14719
- wallMs: Date.now() - started
14720
- });
14721
- options.onProgress?.(
14722
- `fix-loop attempt ${attempt}: exit=${execution.exitCode} failureVanished=${execution.failureVanished}`
14723
- );
14724
- if (execution.failureVanished) {
14725
- return {
14726
- flipped: true,
14727
- flippedAtAttempt: attempt,
14728
- attempts,
14729
- aborted: false,
14730
- llmCalls,
14731
- llmFailures,
14732
- promptTokens,
14733
- completionTokens
14734
- };
14735
- }
14736
- }
14737
- return {
14738
- flipped: false,
14739
- flippedAtAttempt: null,
14740
- attempts,
14741
- aborted: false,
14742
- llmCalls,
14743
- llmFailures,
14744
- promptTokens,
14745
- completionTokens
14746
- };
14747
- }
14748
-
14749
- // src/replay-batch.ts
14750
- import { execFile } from "child_process";
14751
- import { createHash as createHash12 } from "crypto";
14752
- import { appendFileSync, mkdirSync as mkdirSync2, mkdtempSync, writeFileSync as writeFileSync2 } from "fs";
14753
- import { tmpdir as tmpdir5 } from "os";
14754
- import { join as join26 } from "path";
14755
- import { promisify } from "util";
14756
- import {
14757
- InMemoryTraceStore as InMemoryTraceStore3,
14758
- runCounterfactual as runCounterfactual2
14759
- } from "@tangle-network/agent-eval";
14760
- var execFileAsync = promisify(execFile);
14761
- function derivedImageTag(image, cwd) {
14762
- const digest = createHash12("sha256").update(`${image}
14763
- ${cwd}`).digest("hex").slice(0, 12);
14764
- return `ctb-replay:${digest}-uid1000`;
14765
- }
14766
- function dockerImagePreparer(options = {}) {
14767
- const pullTimeoutMs = options.pullTimeoutMs ?? 12e5;
14768
- const buildTimeoutMs = options.buildTimeoutMs ?? 9e5;
14769
- const imageExists = async (tag) => {
14770
- try {
14771
- await execFileAsync("docker", ["image", "inspect", tag], { maxBuffer: 8 * 1024 * 1024 });
14772
- return true;
14773
- } catch {
14774
- return false;
14775
- }
14776
- };
14777
- const errorTail = (err) => {
14778
- const raw = err && typeof err === "object" && "stderr" in err && typeof err.stderr === "string" && err.stderr.length > 0 ? err.stderr : err instanceof Error ? err.message : String(err);
14779
- return raw.trim().split("\n").slice(-3).join(" | ").slice(0, 400);
14780
- };
14781
- return {
14782
- async ensure(image, cwd) {
14783
- if (cwd === "/") {
14784
- if (await imageExists(image)) {
14785
- return { succeeded: true, value: { derivedImage: image, pulled: false, built: false } };
14786
- }
14787
- try {
14788
- await execFileAsync("docker", ["pull", image], {
14789
- timeout: pullTimeoutMs,
14790
- maxBuffer: 32 * 1024 * 1024
14791
- });
14792
- } catch (err) {
14793
- return { succeeded: false, error: `pull ${image}: ${errorTail(err)}` };
14794
- }
14795
- return { succeeded: true, value: { derivedImage: image, pulled: true, built: false } };
14796
- }
14797
- const derived = derivedImageTag(image, cwd);
14798
- if (await imageExists(derived)) {
14799
- return { succeeded: true, value: { derivedImage: derived, pulled: false, built: false } };
14800
- }
14801
- let pulled = false;
14802
- if (!await imageExists(image)) {
14803
- try {
14804
- await execFileAsync("docker", ["pull", image], {
14805
- timeout: pullTimeoutMs,
14806
- maxBuffer: 32 * 1024 * 1024
14807
- });
14808
- pulled = true;
14809
- } catch (err) {
14810
- return { succeeded: false, error: `pull ${image}: ${errorTail(err)}` };
14811
- }
14812
- }
14813
- const contextDir = mkdtempSync(join26(tmpdir5(), "ctb-replay-image-"));
14814
- const quotedCwd = `'${cwd.replaceAll("'", `'\\''`)}'`;
14815
- writeFileSync2(
14816
- join26(contextDir, "Dockerfile"),
14817
- `FROM ${image}
14818
- RUN chown -R 1000:1000 ${quotedCwd}
14819
- `
14820
- );
14821
- try {
14822
- await execFileAsync("docker", ["build", "-t", derived, contextDir], {
14823
- timeout: buildTimeoutMs,
14824
- maxBuffer: 32 * 1024 * 1024
14825
- });
14826
- } catch (err) {
14827
- return { succeeded: false, error: `build ${derived} from ${image}: ${errorTail(err)}` };
14828
- }
14829
- return { succeeded: true, value: { derivedImage: derived, pulled, built: true } };
14830
- }
14831
- };
14832
- }
14833
- function mulberry32(seed) {
14834
- let state = seed >>> 0;
14835
- return () => {
14836
- state = state + 1831565813 >>> 0;
14837
- let t = state;
14838
- t = Math.imul(t ^ t >>> 15, t | 1);
14839
- t ^= t + Math.imul(t ^ t >>> 7, t | 61);
14840
- return ((t ^ t >>> 14) >>> 0) / 4294967296;
14841
- };
14842
- }
14843
- function seededSample(items, size, seed) {
14844
- const pool = [...items];
14845
- const random = mulberry32(seed);
14846
- for (let i = pool.length - 1; i > 0; i--) {
14847
- const j = Math.floor(random() * (i + 1));
14848
- [pool[i], pool[j]] = [pool[j], pool[i]];
14849
- }
14850
- return new Set(pool.slice(0, size));
14851
- }
14852
- function caseOutDirName(row) {
14853
- return `${row.corpus}--${row.trajId}`.replaceAll(/[^A-Za-z0-9._-]/g, "_").slice(0, 180);
14854
- }
14855
- async function executeArmB(replayCase, fixCommand, backend, signature, stepTimeoutMs, prefixLimit, onProgress) {
14856
- const store = new InMemoryTraceStore3();
14857
- const { runId } = await ingestCodeTraceBenchSteps(store, replayCase.steps, replayCase.trajId);
14858
- const runner = new SandboxCounterfactualRunner(backend, {
14859
- cwd: replayCase.cwd,
14860
- stepTimeoutMs,
14861
- prefixLimit,
14862
- onProgress
14863
- });
14864
- await runCounterfactual2(
14865
- store,
14866
- runId,
14867
- {
14868
- kind: "custom",
14869
- at: replayCase.k - 1,
14870
- describe: "arm-B corrected step",
14871
- apply: (step) => ({
14872
- ...step,
14873
- span: { ...step.span, args: { command: fixCommand } }
14874
- })
14875
- },
14876
- runner
14877
- );
14878
- const exec = runner.lastArm;
14879
- if (!exec) throw new Error("arm B finished without executing the corrected step");
14880
- const output = `${exec.stdout}
14881
- ${exec.stderr}`;
14882
- return {
14883
- exitCode: exec.exitCode,
14884
- prefixExecuted: runner.lastPrefix?.prefixExecuted ?? 0,
14885
- prefixDivergences: runner.lastPrefix?.prefixDivergences.length ?? 0,
14886
- failureVanished: exec.exitCode === 0 && (signature ? !output.includes(signature) : true),
14887
- stdout: exec.stdout,
14888
- stderr: exec.stderr
14889
- };
14890
- }
14891
- function rate(numerator, denominator) {
14892
- return denominator > 0 ? numerator / denominator : null;
14893
- }
14894
- async function runReplayBatch(options) {
14895
- const onProgress = options.onProgress ?? (() => {
14896
- });
14897
- const enumeration = enumerateReplayableCases(options.corpora);
14898
- let selected = enumeration.replayable;
14899
- if (options.caseFilter) {
14900
- selected = selected.filter((c) => c.trajId.includes(options.caseFilter));
14901
- }
14902
- if (options.caseLimit !== void 0) selected = selected.slice(0, options.caseLimit);
14903
- onProgress(
14904
- `enumerated ${enumeration.labelEntryCount} label entries \u2192 ${enumeration.replayable.length} replayable, ${enumeration.excluded.length} excluded; executing ${selected.length}`
14905
- );
14906
- const preparer = options.preparer ?? dockerImagePreparer();
14907
- const backendFactory = options.backendFactory ?? ((derivedImage) => sandboxReplayBackend({
14908
- image: derivedImage,
14909
- apiKey: requireOption2(options.apiKey, "apiKey"),
14910
- baseUrl: requireOption2(options.baseUrl, "baseUrl"),
14911
- maxLifetimeSeconds: options.maxLifetimeSeconds ?? 3600
14912
- }));
14913
- mkdirSync2(options.out, { recursive: true });
14914
- const progressPath = join26(options.out, "cases.jsonl");
14915
- const rows = [];
14916
- const pullFailures = [];
14917
- const verdictByTraj = /* @__PURE__ */ new Map();
14918
- for (const [index, replayCase] of selected.entries()) {
14919
- const caseStart = Date.now();
14920
- const label = `[${index + 1}/${selected.length}] ${replayCase.corpus}/${replayCase.trajId}`;
14921
- onProgress(`${label}: preparing image ${replayCase.image}`);
14922
- const preparation = await preparer.ensure(replayCase.image, replayCase.cwd);
14923
- const base = {
14924
- corpus: replayCase.corpus,
14925
- trajId: replayCase.trajId,
14926
- image: replayCase.image,
14927
- cwd: replayCase.cwd,
14928
- cwdSource: replayCase.cwdSource,
14929
- k: replayCase.k,
14930
- stepCount: replayCase.steps.length,
14931
- goldIncorrectSteps: replayCase.goldIncorrectSteps,
14932
- submitGoldsSkipped: replayCase.submitGoldsSkipped,
14933
- recordedReturncodeAtK: replayCase.recordedReturncodeAtK
14934
- };
14935
- if (!preparation.succeeded) {
14936
- pullFailures.push({
14937
- corpus: replayCase.corpus,
14938
- trajId: replayCase.trajId,
14939
- image: replayCase.image,
14940
- error: preparation.error
14941
- });
14942
- const row = {
14943
- ...base,
14944
- derivedImage: null,
14945
- signature: null,
14946
- status: "image-unavailable",
14947
- error: preparation.error,
14948
- imagePulled: false,
14949
- imageBuilt: false,
14950
- prefixExecuted: null,
14951
- prefixDivergences: null,
14952
- prefixDivergencePct: null,
14953
- armAExit: null,
14954
- armAReturncodeMatch: false,
14955
- armASignatureMatch: false,
14956
- replayed: false,
14957
- fix: null,
14958
- wallMs: Date.now() - caseStart
14959
- };
14960
- rows.push(row);
14961
- appendFileSync(progressPath, `${JSON.stringify(row)}
14962
- `);
14963
- onProgress(`${label}: image unavailable \u2014 ${preparation.error}`);
14964
- continue;
14965
- }
14966
- const { derivedImage, pulled, built } = preparation.value;
14967
- const stepTimeoutMs = options.stepTimeoutMs ?? replayCase.recordedStepTimeoutMs ?? 12e4;
14968
- const caseOut = join26(options.out, caseOutDirName(replayCase));
14969
- onProgress(`${label}: arm A on ${derivedImage} (k=${replayCase.k}, timeout ${stepTimeoutMs}ms)`);
14970
- try {
14971
- const verdict = await replayVerify({
14972
- stepsPath: replayCase.stepsPath,
14973
- image: derivedImage,
14974
- at: replayCase.k,
14975
- cwd: replayCase.cwd,
14976
- out: caseOut,
14977
- caseId: replayCase.trajId,
14978
- stepTimeoutMs,
14979
- prefixLimit: options.prefixLimit,
14980
- backend: backendFactory(derivedImage),
14981
- onProgress: (message) => onProgress(`${label}: ${message}`)
14982
- });
14983
- verdictByTraj.set(replayCase.trajId, verdict);
14984
- const divergencePct = verdict.prefixExecuted > 0 ? verdict.prefixDivergences.length / verdict.prefixExecuted * 100 : 0;
14985
- const returncodeMatch = verdict.recordedReturncode !== null && verdict.armA.exitCode === verdict.recordedReturncode;
14986
- const row = {
14987
- ...base,
14988
- derivedImage,
14989
- signature: verdict.signature,
14990
- status: "ok",
14991
- error: null,
14992
- imagePulled: pulled,
14993
- imageBuilt: built,
14994
- prefixExecuted: verdict.prefixExecuted,
14995
- prefixDivergences: verdict.prefixDivergences.length,
14996
- prefixDivergencePct: Number(divergencePct.toFixed(1)),
14997
- armAExit: verdict.armA.exitCode,
14998
- armAReturncodeMatch: returncodeMatch,
14999
- armASignatureMatch: verdict.armA.failureSignatureMatch,
15000
- replayed: divergencePct <= 10 && returncodeMatch,
15001
- fix: null,
15002
- wallMs: Date.now() - caseStart
15003
- };
15004
- rows.push(row);
15005
- appendFileSync(progressPath, `${JSON.stringify(row)}
15006
- `);
15007
- onProgress(
15008
- `${label}: armA exit=${verdict.armA.exitCode} rcMatch=${returncodeMatch} divergences=${verdict.prefixDivergences.length}/${verdict.prefixExecuted}`
15009
- );
15010
- } catch (err) {
15011
- const message = err instanceof Error ? err.message : String(err);
15012
- const row = {
15013
- ...base,
15014
- derivedImage,
15015
- signature: null,
15016
- status: "replay-error",
15017
- error: message.slice(0, 500),
15018
- imagePulled: pulled,
15019
- imageBuilt: built,
15020
- prefixExecuted: null,
15021
- prefixDivergences: null,
15022
- prefixDivergencePct: null,
15023
- armAExit: null,
15024
- armAReturncodeMatch: false,
15025
- armASignatureMatch: false,
15026
- replayed: false,
15027
- fix: null,
15028
- wallMs: Date.now() - caseStart
15029
- };
15030
- rows.push(row);
15031
- appendFileSync(progressPath, `${JSON.stringify(row)}
15032
- `);
15033
- onProgress(`${label}: replay error \u2014 ${message.slice(0, 200)}`);
15034
- }
15035
- }
15036
- let llm = null;
15037
- if (options.fix !== "none") {
15038
- const caller = options.fixCaller;
15039
- if (!caller) throw new Error(`replay-verify-batch: fix=${options.fix} requires a fixCaller`);
15040
- const fixAttempts = options.fixAttempts ?? 3;
15041
- const maxFixCases = options.maxFixCases ?? 30;
15042
- const seed = options.seed ?? 17;
15043
- const eligible = rows.filter((r) => r.status === "ok" && r.replayed);
15044
- const sampled = eligible.length > maxFixCases ? seededSample(eligible, maxFixCases, seed) : new Set(eligible);
15045
- if (eligible.length > maxFixCases) {
15046
- onProgress(
15047
- `fix phase: ${eligible.length} eligible > cap ${maxFixCases}; seeded sample (seed=${seed})`
15048
- );
15049
- }
15050
- let calls = 0;
15051
- let failures = 0;
15052
- let promptTokens = 0;
15053
- let completionTokens = 0;
15054
- for (const row of eligible) {
15055
- const index = rows.indexOf(row);
15056
- if (!sampled.has(row)) {
15057
- rows[index] = {
15058
- ...row,
15059
- fix: {
15060
- attempted: false,
15061
- sampledOut: true,
15062
- command: null,
15063
- llmError: null,
15064
- usage: null,
15065
- armBExit: null,
15066
- armBPrefixExecuted: null,
15067
- armBPrefixDivergences: null,
15068
- failureVanished: null,
15069
- armBError: null,
15070
- attempts: null,
15071
- flippedAtAttempt: null
15072
- }
15073
- };
15074
- continue;
15075
- }
15076
- const replayCase = selected.find((c) => c.trajId === row.trajId && c.corpus === row.corpus);
15077
- const label = `fix ${row.corpus}/${row.trajId}`;
15078
- const signature = verdictByTraj.get(row.trajId)?.signature ?? null;
15079
- const stepTimeoutMs = options.stepTimeoutMs ?? replayCase.recordedStepTimeoutMs ?? 12e4;
15080
- const promptInput = {
15081
- taskStatement: replayCase.taskStatement,
15082
- steps: replayCase.steps,
15083
- k: replayCase.k
15084
- };
15085
- const caseOut = join26(options.out, caseOutDirName(row));
15086
- if (options.fix === "loop") {
15087
- onProgress(`${label}: fix loop (budget ${fixAttempts} attempts)`);
15088
- const result = await runFixLoop(
15089
- caller,
15090
- promptInput,
15091
- async (command, attempt) => {
15092
- onProgress(
15093
- `${label}: attempt ${attempt} arm B \u2014 ${command.split("\n")[0].slice(0, 120)}`
15094
- );
15095
- const armB = await executeArmB(
15096
- replayCase,
15097
- command,
15098
- backendFactory(row.derivedImage),
15099
- signature,
15100
- stepTimeoutMs,
15101
- options.prefixLimit,
15102
- (message) => onProgress(`${label}: attempt ${attempt}: ${message}`)
15103
- );
15104
- mkdirSync2(caseOut, { recursive: true });
15105
- writeFileSync2(
15106
- join26(caseOut, `armB-attempt${attempt}-result.json`),
15107
- `${JSON.stringify({ command, ...armB }, null, 2)}
15108
- `
15109
- );
15110
- writeFileSync2(
15111
- join26(caseOut, "armB-result.json"),
15112
- `${JSON.stringify({ command, attempt, ...armB }, null, 2)}
15113
- `
15114
- );
15115
- return armB;
15116
- },
15117
- { maxAttempts: fixAttempts, onProgress: (message) => onProgress(`${label}: ${message}`) }
15118
- );
15119
- calls += result.llmCalls;
15120
- failures += result.llmFailures;
15121
- promptTokens += result.promptTokens;
15122
- completionTokens += result.completionTokens;
15123
- const usage = result.promptTokens + result.completionTokens > 0 ? { promptTokens: result.promptTokens, completionTokens: result.completionTokens } : null;
15124
- const flippedRecord = result.flippedAtAttempt !== null ? result.attempts.find((a) => a.attempt === result.flippedAtAttempt) : null;
15125
- const summary = flippedRecord ?? [...result.attempts].reverse().find((a) => a.executed) ?? null;
15126
- const lastRecord = result.attempts.at(-1) ?? null;
15127
- rows[index] = {
15128
- ...row,
15129
- fix: summary ? {
15130
- attempted: true,
15131
- sampledOut: false,
15132
- command: summary.command,
15133
- llmError: null,
15134
- usage,
15135
- armBExit: summary.exitCode,
15136
- armBPrefixExecuted: summary.prefixExecuted,
15137
- armBPrefixDivergences: summary.prefixDivergences,
15138
- failureVanished: summary.failureVanished,
15139
- armBError: null,
15140
- attempts: result.attempts,
15141
- flippedAtAttempt: result.flippedAtAttempt
15142
- } : result.aborted ? {
15143
- attempted: true,
15144
- sampledOut: false,
15145
- command: lastRecord?.command ?? null,
15146
- llmError: null,
15147
- usage,
15148
- armBExit: null,
15149
- armBPrefixExecuted: null,
15150
- armBPrefixDivergences: null,
15151
- failureVanished: null,
15152
- armBError: lastRecord?.armBError ?? "sandbox error",
15153
- attempts: result.attempts,
15154
- flippedAtAttempt: null
15155
- } : {
15156
- attempted: true,
15157
- sampledOut: false,
15158
- command: null,
15159
- llmError: lastRecord?.llmError ?? "no attempt produced a runnable fix",
15160
- usage,
15161
- armBExit: null,
15162
- armBPrefixExecuted: null,
15163
- armBPrefixDivergences: null,
15164
- failureVanished: null,
15165
- armBError: null,
15166
- attempts: result.attempts,
15167
- flippedAtAttempt: null
15168
- }
15169
- };
15170
- onProgress(
15171
- `${label}: loop done \u2014 flipped=${result.flipped}` + (result.flippedAtAttempt !== null ? ` at attempt ${result.flippedAtAttempt}` : "") + ` (${result.llmCalls} calls, ${result.attempts.filter((a) => a.executed).length} arms)`
15172
- );
15173
- continue;
15174
- }
15175
- onProgress(`${label}: generating corrected command`);
15176
- calls += 1;
15177
- const generated = await generateFixCommand(caller, promptInput);
15178
- if (!generated.succeeded) {
15179
- failures += 1;
15180
- rows[index] = {
15181
- ...row,
15182
- fix: {
15183
- attempted: true,
15184
- sampledOut: false,
15185
- command: null,
15186
- llmError: generated.error,
15187
- usage: null,
15188
- armBExit: null,
15189
- armBPrefixExecuted: null,
15190
- armBPrefixDivergences: null,
15191
- failureVanished: null,
15192
- armBError: null,
15193
- attempts: null,
15194
- flippedAtAttempt: null
15195
- }
15196
- };
15197
- onProgress(`${label}: LLM failed \u2014 ${generated.error.slice(0, 200)}`);
15198
- continue;
15199
- }
15200
- promptTokens += generated.value.usage?.promptTokens ?? 0;
15201
- completionTokens += generated.value.usage?.completionTokens ?? 0;
15202
- onProgress(`${label}: arm B \u2014 ${generated.value.command.split("\n")[0].slice(0, 120)}`);
15203
- try {
15204
- const armB = await executeArmB(
15205
- replayCase,
15206
- generated.value.command,
15207
- backendFactory(row.derivedImage),
15208
- signature,
15209
- stepTimeoutMs,
15210
- options.prefixLimit,
15211
- (message) => onProgress(`${label}: ${message}`)
15212
- );
15213
- rows[index] = {
15214
- ...row,
15215
- fix: {
15216
- attempted: true,
15217
- sampledOut: false,
15218
- command: generated.value.command,
15219
- llmError: null,
15220
- usage: generated.value.usage,
15221
- armBExit: armB.exitCode,
15222
- armBPrefixExecuted: armB.prefixExecuted,
15223
- armBPrefixDivergences: armB.prefixDivergences,
15224
- failureVanished: armB.failureVanished,
15225
- armBError: null,
15226
- attempts: null,
15227
- flippedAtAttempt: armB.failureVanished ? 1 : null
15228
- }
15229
- };
15230
- mkdirSync2(caseOut, { recursive: true });
15231
- writeFileSync2(
15232
- join26(caseOut, "armB-result.json"),
15233
- `${JSON.stringify({ command: generated.value.command, ...armB }, null, 2)}
15234
- `
15235
- );
15236
- onProgress(`${label}: armB exit=${armB.exitCode} failureVanished=${armB.failureVanished}`);
15237
- } catch (err) {
15238
- const message = err instanceof Error ? err.message : String(err);
15239
- rows[index] = {
15240
- ...row,
15241
- fix: {
15242
- attempted: true,
15243
- sampledOut: false,
15244
- command: generated.value.command,
15245
- llmError: null,
15246
- usage: generated.value.usage,
15247
- armBExit: null,
15248
- armBPrefixExecuted: null,
15249
- armBPrefixDivergences: null,
15250
- failureVanished: null,
15251
- armBError: message.slice(0, 500),
15252
- attempts: null,
15253
- flippedAtAttempt: null
15254
- }
15255
- };
15256
- onProgress(`${label}: arm B error \u2014 ${message.slice(0, 200)}`);
15257
- }
15258
- }
15259
- llm = {
15260
- model: options.fixModelLabel ?? "unknown",
15261
- calls,
15262
- failures,
15263
- promptTokens,
15264
- completionTokens
15265
- };
15266
- }
15267
- const executedRows = rows.filter((r) => r.status === "ok");
15268
- const replayed = executedRows.filter((r) => r.replayed);
15269
- const signatureStrict = executedRows.filter(
15270
- (r) => r.replayed && r.armASignatureMatch
15271
- );
15272
- const armBExecuted = rows.filter((r) => r.fix?.attempted && r.fix.failureVanished !== null);
15273
- const flipped = armBExecuted.filter((r) => r.fix.failureVanished === true);
15274
- const armBNonzeroRc = armBExecuted.filter(
15275
- (r) => r.recordedReturncodeAtK !== null && r.recordedReturncodeAtK !== 0
15276
- );
15277
- const flippedNonzeroRc = armBNonzeroRc.filter((r) => r.fix.failureVanished === true);
15278
- const attempt1Executed = rows.filter(
15279
- (r) => r.fix?.attempts?.find((a) => a.attempt === 1)?.executed === true
15280
- );
15281
- const flippedAt1 = rows.filter((r) => r.fix?.flippedAtAttempt === 1);
15282
- const flipsByAttempt = {};
15283
- for (const row of rows) {
15284
- const at = row.fix?.flippedAtAttempt;
15285
- if (typeof at === "number") flipsByAttempt[String(at)] = (flipsByAttempt[String(at)] ?? 0) + 1;
15286
- }
15287
- const excludedByReason = {};
15288
- for (const excluded of enumeration.excluded) {
15289
- excludedByReason[excluded.reason] = (excludedByReason[excluded.reason] ?? 0) + 1;
15290
- }
15291
- const submitGoldsByCorpus = {};
15292
- const submitEntry = (corpus) => submitGoldsByCorpus[corpus] ??= { submitOnlyCases: 0, goldsSkippedWithinReplayable: 0 };
15293
- for (const excluded of enumeration.excluded) {
15294
- if (excluded.reason === "gold-only-submit-step") submitEntry(excluded.corpus).submitOnlyCases += 1;
15295
- }
15296
- for (const replayCase of enumeration.replayable) {
15297
- if (replayCase.submitGoldsSkipped > 0) {
15298
- submitEntry(replayCase.corpus).goldsSkippedWithinReplayable += replayCase.submitGoldsSkipped;
15299
- }
15300
- }
15301
- const report = {
15302
- generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
15303
- corpora: options.corpora.map((c) => ({
15304
- name: c.name,
15305
- labelsPath: c.labelsPath,
15306
- preparedDir: c.preparedDir
15307
- })),
15308
- totals: {
15309
- labelEntries: enumeration.labelEntryCount,
15310
- replayable: enumeration.replayable.length,
15311
- executed: selected.length,
15312
- excludedByReason,
15313
- submitGoldsByCorpus
15314
- },
15315
- headline: {
15316
- replayabilityRate: {
15317
- numerator: replayed.length,
15318
- denominator: selected.length,
15319
- value: rate(replayed.length, selected.length)
15320
- },
15321
- signatureStrictRate: {
15322
- numerator: signatureStrict.length,
15323
- denominator: selected.length,
15324
- value: rate(signatureStrict.length, selected.length)
15325
- },
15326
- fixFlipRate: options.fix !== "none" ? {
15327
- numerator: flipped.length,
15328
- denominator: armBExecuted.length,
15329
- value: rate(flipped.length, armBExecuted.length)
15330
- } : null,
15331
- fixFlipRateNonzeroRc: options.fix !== "none" ? {
15332
- numerator: flippedNonzeroRc.length,
15333
- denominator: armBNonzeroRc.length,
15334
- value: rate(flippedNonzeroRc.length, armBNonzeroRc.length)
15335
- } : null,
15336
- fixFlipAttempt1: options.fix === "loop" ? {
15337
- numerator: flippedAt1.length,
15338
- denominator: attempt1Executed.length,
15339
- value: rate(flippedAt1.length, attempt1Executed.length)
15340
- } : null,
15341
- flipsByAttempt: options.fix === "loop" ? flipsByAttempt : null
15342
- },
15343
- llm,
15344
- excluded: enumeration.excluded,
15345
- pullFailures,
15346
- cases: rows
15347
- };
15348
- writeFileSync2(join26(options.out, "batch-report.json"), `${JSON.stringify(report, null, 2)}
15349
- `);
15350
- writeFileSync2(join26(options.out, "batch-report.md"), renderBatchReport(report));
15351
- return report;
15352
- }
15353
- function requireOption2(value, name) {
15354
- if (value === void 0) throw new Error(`replay-verify-batch: ${name} is required`);
15355
- return value;
15356
- }
15357
- function pct(value) {
15358
- return value === null ? "\u2014" : `${(value * 100).toFixed(1)}%`;
15359
- }
15360
- function renderBatchReport(report) {
15361
- const lines = [];
15362
- const { headline, totals } = report;
15363
- lines.push("# Replay-verify batch report");
15364
- lines.push("");
15365
- lines.push(`Generated ${report.generatedAt}.`);
15366
- lines.push("");
15367
- lines.push("## Headline");
15368
- lines.push("");
15369
- lines.push(
15370
- `- **Replayability rate: ${pct(headline.replayabilityRate.value)}** (${headline.replayabilityRate.numerator}/${headline.replayabilityRate.denominator} replayable cases where the prefix replayed with \u226410% returncode divergence AND arm A reproduced the recorded returncode at the gold step k).`
15371
- );
15372
- lines.push(
15373
- `- Signature-strict rate: ${pct(headline.signatureStrictRate.value)} (${headline.signatureStrictRate.numerator}/${headline.signatureStrictRate.denominator}; additionally requires the recorded error substring in arm A output).`
15374
- );
15375
- if (headline.fixFlipRate) {
15376
- lines.push(
15377
- `- **Fix-flip rate: ${pct(headline.fixFlipRate.value)}** (${headline.fixFlipRate.numerator}/${headline.fixFlipRate.denominator} arm-B-executed cases where the generated fix made the failure vanish).`
15378
- );
15379
- }
15380
- if (headline.fixFlipRateNonzeroRc) {
15381
- lines.push(
15382
- `- Fix-flip rate on recorded-rc\u22600 cases: ${pct(headline.fixFlipRateNonzeroRc.value)} (${headline.fixFlipRateNonzeroRc.numerator}/${headline.fixFlipRateNonzeroRc.denominator}; real recorded failures \u2014 a gold step recorded with rc 0 flips vacuously).`
15383
- );
15384
- }
15385
- if (headline.fixFlipAttempt1) {
15386
- lines.push(
15387
- `- Fix-flip@1: ${pct(headline.fixFlipAttempt1.value)} (${headline.fixFlipAttempt1.numerator}/${headline.fixFlipAttempt1.denominator} cases whose attempt 1 executed \u2014 the one-shot-comparable number).`
15388
- );
15389
- }
15390
- if (headline.flipsByAttempt && Object.keys(headline.flipsByAttempt).length > 0) {
15391
- const parts = Object.entries(headline.flipsByAttempt).sort((a, b) => Number(a[0]) - Number(b[0])).map(([attempt, count2]) => `attempt ${attempt}: ${count2}`);
15392
- lines.push(`- Flips by attempt: ${parts.join(", ")}.`);
15393
- }
15394
- lines.push("");
15395
- lines.push("## Enumeration");
15396
- lines.push("");
15397
- lines.push(
15398
- `${totals.labelEntries} label entries across ${report.corpora.length} corpora \u2192 ${totals.replayable} replayable (SWE-style docker image + \u22651 gold incorrect step), ${totals.executed} executed.`
15399
- );
15400
- lines.push("");
15401
- lines.push("| exclusion reason | count |");
15402
- lines.push("| --- | --- |");
15403
- for (const [reason, count2] of Object.entries(totals.excludedByReason).sort((a, b) => b[1] - a[1])) {
15404
- lines.push(`| ${reason} | ${count2} |`);
15405
- }
15406
- const submitEntries = Object.entries(totals.submitGoldsByCorpus);
15407
- if (submitEntries.length > 0) {
15408
- lines.push("");
15409
- lines.push(
15410
- "Submit-command golds are never counterfactual targets (a gold on the submit step marks a bad submit decision, not a failed command):"
15411
- );
15412
- lines.push("");
15413
- lines.push("| corpus | cases excluded (all golds = submit) | golds skipped within replayable cases |");
15414
- lines.push("| --- | --- | --- |");
15415
- for (const [corpus, stats] of submitEntries.sort((a, b) => a[0].localeCompare(b[0]))) {
15416
- lines.push(`| ${corpus} | ${stats.submitOnlyCases} | ${stats.goldsSkippedWithinReplayable} |`);
15417
- }
15418
- }
15419
- if (report.pullFailures.length > 0) {
15420
- lines.push("");
15421
- lines.push("## Image pull/build failures");
15422
- lines.push("");
15423
- lines.push("| corpus | trajectory | image | error |");
15424
- lines.push("| --- | --- | --- | --- |");
15425
- for (const failure2 of report.pullFailures) {
15426
- lines.push(
15427
- `| ${failure2.corpus} | ${failure2.trajId} | \`${failure2.image}\` | ${failure2.error.replaceAll("|", "\\|")} |`
15428
- );
15429
- }
15430
- }
15431
- lines.push("");
15432
- lines.push("## Per-case results");
15433
- lines.push("");
15434
- lines.push(
15435
- "| corpus | trajectory | k | rc@k | prefix | div | div% | armA exit | rc match | sig match | replayed | fix | armB exit | vanished | wall s |"
15436
- );
15437
- lines.push(
15438
- "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |"
15439
- );
15440
- for (const row of report.cases) {
15441
- const fix = row.fix;
15442
- const fixCell = !fix ? "\u2014" : fix.sampledOut ? "sampled-out" : fix.llmError ? "llm-failed" : fix.armBError ? "armB-error" : fix.attempts ? fix.flippedAtAttempt !== null ? `flip@${fix.flippedAtAttempt}` : `exhausted(${fix.attempts.length})` : "generated";
15443
- lines.push(
15444
- [
15445
- row.corpus,
15446
- row.trajId.length > 48 ? `${row.trajId.slice(0, 45)}\u2026` : row.trajId,
15447
- row.k,
15448
- row.recordedReturncodeAtK ?? "null",
15449
- row.prefixExecuted ?? "\u2014",
15450
- row.prefixDivergences ?? "\u2014",
15451
- row.prefixDivergencePct ?? "\u2014",
15452
- row.status === "ok" ? row.armAExit : row.status,
15453
- row.armAReturncodeMatch ? "yes" : "no",
15454
- row.armASignatureMatch ? "yes" : "no",
15455
- row.replayed ? "**yes**" : "no",
15456
- fixCell,
15457
- fix?.armBExit ?? "\u2014",
15458
- fix?.failureVanished === null || fix === null ? "\u2014" : fix.failureVanished ? "**yes**" : "no",
15459
- (row.wallMs / 1e3).toFixed(1)
15460
- ].join(" | ")
15461
- );
15462
- }
15463
- if (report.llm) {
15464
- lines.push("");
15465
- lines.push("## LLM fix generation");
15466
- lines.push("");
15467
- lines.push(
15468
- `Model ${report.llm.model}: ${report.llm.calls} calls (${report.llm.failures} failed), ${report.llm.promptTokens} prompt + ${report.llm.completionTokens} completion tokens.`
15469
- );
15470
- }
15471
- lines.push("");
15472
- return lines.join("\n");
15473
- }
15474
- function parseReplayBatchArgs(argv) {
15475
- const corpora = [];
15476
- const values = /* @__PURE__ */ new Map();
15477
- let enumerateOnly = false;
15478
- for (let i = 0; i < argv.length; i++) {
15479
- const arg = argv[i];
15480
- if (arg === "--help" || arg === "-h") return "help";
15481
- if (arg === "--enumerate-only") {
15482
- enumerateOnly = true;
15483
- continue;
15484
- }
15485
- if (!arg.startsWith("--")) throw new Error(`unexpected positional argument: ${arg}`);
15486
- const value = argv[++i];
15487
- if (value === void 0) throw new Error(`missing value for ${arg}`);
15488
- if (arg === "--corpus") corpora.push(parseCorpusFlag(value));
15489
- else values.set(arg, value);
15490
- }
15491
- if (corpora.length === 0) {
15492
- throw new Error("replay-verify-batch: at least one --corpus name=<labels>::<prepared> is required");
15493
- }
15494
- const out = values.get("--out");
15495
- if (!out && !enumerateOnly) throw new Error("replay-verify-batch: --out is required");
15496
- const optionalNumber = (flag) => {
15497
- const v = values.get(flag);
15498
- if (v === void 0) return void 0;
15499
- const n = Number(v);
15500
- if (!Number.isFinite(n)) throw new Error(`replay-verify-batch: ${flag} must be a number, got ${v}`);
15501
- return n;
15502
- };
15503
- const fix = values.get("--fix") ?? "none";
15504
- if (fix !== "none" && fix !== "generate" && fix !== "loop") {
15505
- throw new Error(`replay-verify-batch: --fix must be none, generate, or loop, got ${fix}`);
15506
- }
15507
- const fixAttempts = optionalNumber("--fix-attempts") ?? 3;
15508
- if (!Number.isInteger(fixAttempts) || fixAttempts < 1) {
15509
- throw new Error(
15510
- `replay-verify-batch: --fix-attempts must be a positive integer, got ${values.get("--fix-attempts")}`
15511
- );
15512
- }
15513
- return {
15514
- corpora,
15515
- out: out ?? ".",
15516
- fix,
15517
- fixAttempts,
15518
- fixModel: values.get("--fix-model") ?? "glm-5.2",
15519
- fixBaseUrl: values.get("--fix-base-url") ?? "https://api.z.ai/api/coding/paas/v4",
15520
- fixApiKeyEnv: values.get("--fix-api-key-env") ?? "ZAI_GLM_API_KEY",
15521
- maxFixCases: optionalNumber("--max-fix-cases") ?? 30,
15522
- seed: optionalNumber("--seed") ?? 17,
15523
- stepTimeoutMs: optionalNumber("--step-timeout"),
15524
- prefixLimit: optionalNumber("--prefix-limit"),
15525
- caseFilter: values.get("--case-filter"),
15526
- caseLimit: optionalNumber("--case-limit"),
15527
- baseUrl: values.get("--base-url") ?? process.env.SANDBOX_API_URL ?? "http://127.0.0.1:4097",
15528
- apiKeyEnv: values.get("--api-key-env") ?? "SANDBOX_API_KEY",
15529
- maxLifetimeSeconds: optionalNumber("--max-lifetime"),
15530
- enumerateOnly
15531
- };
15532
- }
15533
- function replayBatchUsage() {
15534
- return `traces replay-verify-batch \u2014 measure replayability and fix-flip rates across gold-labeled corpora
15535
-
15536
- Usage:
15537
- traces replay-verify-batch \\
15538
- --corpus NAME=<labels.json>::<prepared-dir> [--corpus ...] \\
15539
- --out DIR [--fix none|generate|loop] [--fix-attempts 3] [--enumerate-only] \\
15540
- [--fix-model glm-5.2] [--fix-base-url URL] [--fix-api-key-env ZAI_GLM_API_KEY] \\
15541
- [--max-fix-cases 30] [--seed 17] [--step-timeout MS] [--prefix-limit N] \\
15542
- [--case-filter SUBSTRING] [--case-limit N] \\
15543
- [--base-url URL] [--api-key-env SANDBOX_API_KEY] [--max-lifetime SECONDS]
15544
-
15545
- Replayable = the raw trajectory carries info.docker_config.base_image AND the
15546
- labels mark at least one gold incorrect step that is a real mid-trajectory
15547
- action. k = the first such gold step; golds whose action is the agent's
15548
- submit command (COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT) are skipped and
15549
- counted, and cases whose golds are all submit steps are excluded as
15550
- gold-only-submit-step. Excluded cases and image pull failures are report
15551
- rows, never skipped silently. --enumerate-only prints the enumeration
15552
- without touching a sandbox.
15553
-
15554
- --fix generate runs ONE LLM call per arm-A-reproduced case (cap
15555
- --max-fix-cases, seeded sample beyond it) and executes the corrected command
15556
- as arm B in a fresh sandbox.
15557
-
15558
- --fix loop iterates: attempt 1 is the one-shot prompt; when an arm fails
15559
- (nonzero exit or the failure signature persists) or the model call fails,
15560
- the next prompt carries every prior command with its REAL executed
15561
- stdout/stderr, up to --fix-attempts attempts (default 3). Retries may
15562
- answer with a short script (<=5 commands) executed as one /bin/sh unit.
15563
- Every attempt runs in its own fresh sandbox with the same replayed prefix.
15564
-
15565
- Outputs batch-report.json, batch-report.md, cases.jsonl (incremental), and one
15566
- directory per case with the full replay-verify artifacts.
15567
- See docs/replay-verify.md for orchestrator setup and honest limits.
15568
- `;
15569
- }
15570
- async function cmdReplayVerifyBatch(argv) {
15571
- const parsed = parseReplayBatchArgs(argv);
15572
- if (parsed === "help") {
15573
- process.stdout.write(replayBatchUsage());
15574
- return;
15575
- }
15576
- if (parsed.enumerateOnly) {
15577
- const enumeration = enumerateReplayableCases(parsed.corpora);
15578
- process.stdout.write(
15579
- `${JSON.stringify(
15580
- {
15581
- labelEntries: enumeration.labelEntryCount,
15582
- replayable: enumeration.replayable.map((c) => ({
15583
- corpus: c.corpus,
15584
- trajId: c.trajId,
15585
- image: c.image,
15586
- cwd: c.cwd,
15587
- cwdSource: c.cwdSource,
15588
- k: c.k,
15589
- submitGoldsSkipped: c.submitGoldsSkipped,
15590
- recordedReturncodeAtK: c.recordedReturncodeAtK,
15591
- goldIncorrectSteps: c.goldIncorrectSteps
15592
- })),
15593
- excluded: enumeration.excluded
15594
- },
15595
- null,
15596
- 2
15597
- )}
15598
- `
15599
- );
15600
- return;
15601
- }
15602
- const apiKey = process.env[parsed.apiKeyEnv];
15603
- if (!apiKey) {
15604
- throw new Error(
15605
- `replay-verify-batch: env var ${parsed.apiKeyEnv} is empty \u2014 pass --api-key-env or export it`
15606
- );
15607
- }
15608
- let fixCaller;
15609
- if (parsed.fix !== "none") {
15610
- const fixApiKey = process.env[parsed.fixApiKeyEnv];
15611
- if (!fixApiKey) {
15612
- throw new Error(
15613
- `replay-verify-batch: env var ${parsed.fixApiKeyEnv} is empty \u2014 pass --fix-api-key-env or export it`
15614
- );
15615
- }
15616
- fixCaller = zaiChatCaller({
15617
- baseUrl: parsed.fixBaseUrl,
15618
- apiKey: fixApiKey,
15619
- model: parsed.fixModel
15620
- });
15621
- }
15622
- const report = await runReplayBatch({
15623
- corpora: parsed.corpora,
15624
- out: parsed.out,
15625
- baseUrl: parsed.baseUrl,
15626
- apiKey,
15627
- fix: parsed.fix,
15628
- fixAttempts: parsed.fixAttempts,
15629
- fixCaller,
15630
- fixModelLabel: parsed.fixModel,
15631
- maxFixCases: parsed.maxFixCases,
15632
- seed: parsed.seed,
15633
- stepTimeoutMs: parsed.stepTimeoutMs,
15634
- prefixLimit: parsed.prefixLimit,
15635
- caseFilter: parsed.caseFilter,
15636
- caseLimit: parsed.caseLimit,
15637
- maxLifetimeSeconds: parsed.maxLifetimeSeconds,
15638
- onProgress: (message) => process.stderr.write(`${message}
15639
- `)
15640
- });
15641
- const { replayabilityRate, fixFlipRate } = report.headline;
15642
- process.stderr.write(
15643
- `replay-verify-batch: replayability ${replayabilityRate.numerator}/${replayabilityRate.denominator}` + (fixFlipRate ? `, fix-flip ${fixFlipRate.numerator}/${fixFlipRate.denominator}` : "") + ` \u2192 ${parsed.out}
15644
- `
15645
- );
15646
- }
15647
-
15648
- // src/replay-wire.ts
15649
- var SUBJECT_PATTERN = /^incorrect-steps-(\d+)-(\d+)-(escaped|unescaped)-consequence-(\d+)$/;
15650
- function parseIncorrectStepsSubject(subject) {
15651
- const match = SUBJECT_PATTERN.exec(subject);
15652
- if (!match) return null;
15653
- return {
15654
- firstStep: Number(match[1]),
15655
- lastStep: Number(match[2]),
15656
- escapeStatus: match[3],
15657
- consequenceStep: Number(match[4])
15658
- };
15659
- }
15660
- function resolveFindingInvocation(finding2, corpora) {
15661
- const subject = parseIncorrectStepsSubject(finding2.subject);
15662
- if (!subject) {
15663
- throw new Error(
15664
- `replay-wire: subject '${finding2.subject}' is not incorrect-steps-<first>-<last>-<escaped|unescaped>-consequence-<step>`
15665
- );
15666
- }
15667
- const failures = [];
15668
- for (const corpus of corpora) {
15669
- const resolution = resolveCaseResources(corpus, finding2.trajId);
15670
- if (resolution.resolved) {
15671
- const at = subject.firstStep;
15672
- const step = resolution.resources.steps.find((s) => s.step_id === at);
15673
- if (!step) {
15674
- throw new Error(
15675
- `replay-wire: finding step ${at} is outside ${finding2.trajId} (${resolution.resources.steps.length} steps)`
15676
- );
15677
- }
15678
- return { resources: resolution.resources, subject, at };
15679
- }
15680
- failures.push(`${corpus.name}: ${resolution.reason}`);
15681
- }
15682
- throw new Error(
15683
- `replay-wire: trajectory ${finding2.trajId} is not replayable in any corpus \u2014 ${failures.join("; ")}`
15684
- );
15685
- }
15686
- async function replayVerifyFinding(finding2, options) {
15687
- if (options.fixCaller && options.fixCommand !== void 0) {
15688
- throw new Error("replay-wire: pass fixCaller or fixCommand, not both");
15689
- }
15690
- const invocation = resolveFindingInvocation(finding2, options.corpora);
15691
- let fixCommand = options.fixCommand ?? null;
15692
- if (options.fixCaller) {
15693
- const generated = await generateFixCommand(options.fixCaller, {
15694
- taskStatement: invocation.resources.taskStatement,
15695
- steps: invocation.resources.steps,
15696
- k: invocation.at
15697
- });
15698
- if (!generated.succeeded) {
15699
- throw new Error(`replay-wire: fix generation failed \u2014 ${generated.error}`);
15700
- }
15701
- fixCommand = generated.value.command;
15702
- }
15703
- const verdict = await replayVerify({
15704
- stepsPath: invocation.resources.stepsPath,
15705
- image: invocation.resources.image,
15706
- at: invocation.at,
15707
- fixCommand: fixCommand ?? void 0,
15708
- cwd: invocation.resources.cwd,
15709
- out: options.out,
15710
- caseId: invocation.resources.trajId,
15711
- stepTimeoutMs: options.stepTimeoutMs ?? invocation.resources.recordedStepTimeoutMs ?? void 0,
15712
- prefixLimit: options.prefixLimit,
15713
- backend: options.backend,
15714
- apiKey: options.apiKey,
15715
- baseUrl: options.baseUrl,
15716
- onProgress: options.onProgress
15717
- });
15718
- return { invocation, fixCommand, verdict };
15719
- }
15720
-
15721
- // src/analyze-verify.ts
15722
- import { mkdirSync as mkdirSync3, readFileSync as readFileSync5, writeFileSync as writeFileSync3 } from "fs";
15723
- import { join as join27 } from "path";
15724
- var SINGLE_STEP_SUBJECT = /^incorrect-step-(\d+)$/;
15725
- function findingReplayStep(finding2) {
15726
- const fromMetadata = finding2.metadata?.block_first_step;
15727
- if (typeof fromMetadata === "number" && Number.isInteger(fromMetadata) && fromMetadata >= 1) {
15728
- return fromMetadata;
15729
- }
15730
- const subject = finding2.subject ?? "";
15731
- const wire = parseIncorrectStepsSubject(subject);
15732
- if (wire) return wire.firstStep;
15733
- const single = SINGLE_STEP_SUBJECT.exec(subject);
15734
- if (single) return Number(single[1]);
15735
- return null;
15736
- }
15737
- var TRACE_EVIDENCE_URI = /^trace:\/\/([^/]+)\//;
15738
- function findingTrajectoryId(finding2) {
15739
- for (const ref of finding2.evidence_refs ?? []) {
15740
- if (typeof ref.uri !== "string") continue;
15741
- const match = TRACE_EVIDENCE_URI.exec(ref.uri);
15742
- if (match) return match[1];
15743
- }
15744
- return null;
15745
- }
15746
- function checkStep(steps, at) {
15747
- const step = steps.find((s) => s.step_id === at);
15748
- if (!step) {
15749
- return { ok: false, reason: `step ${at} is outside the trajectory (${steps.length} steps)` };
15750
- }
15751
- if (isSubmitAction(step.action)) {
15752
- return {
15753
- ok: false,
15754
- reason: `step ${at} is the submit action \u2014 a submit decision has no executable failure to replay`
15755
- };
15756
- }
15757
- const recordedReturncode = parseRecordedReturncode(step.observation);
15758
- if (recordedReturncode === null) {
15759
- return {
15760
- ok: false,
15761
- reason: `step ${at} recorded no returncode \u2014 there is no executable failure signature to reproduce`
15762
- };
15763
- }
15764
- return { ok: true, recordedReturncode };
15765
- }
15766
- function resolveFindingReplayability(finding2, source) {
15767
- const at = findingReplayStep(finding2);
15768
- if (at === null) {
15769
- return {
15770
- replayable: false,
15771
- reason: `subject '${finding2.subject ?? "(none)"}' names no trajectory step (expected incorrect-step-<n>, incorrect-steps-<f>-<l>-\u2026, or metadata.block_first_step)`
15772
- };
15773
- }
15774
- if (source.kind === "direct") {
15775
- const trajId2 = findingTrajectoryId(finding2);
15776
- if (trajId2 && source.caseId && trajId2 !== source.caseId) {
15777
- return {
15778
- replayable: false,
15779
- reason: `finding cites trajectory '${trajId2}' but the supplied steps are case '${source.caseId}'`
15780
- };
15781
- }
15782
- let steps;
15783
- try {
15784
- steps = JSON.parse(readFileSync5(source.stepsPath, "utf8"));
15785
- } catch (err) {
15786
- throw new Error(
15787
- `verify-findings: cannot read steps file ${source.stepsPath} \u2014 ${err instanceof Error ? err.message : String(err)}`
15788
- );
15789
- }
15790
- if (!Array.isArray(steps) || steps.length === 0) {
15791
- throw new Error(`verify-findings: ${source.stepsPath} is not a non-empty steps array`);
15792
- }
15793
- const step = checkStep(steps, at);
15794
- if (!step.ok) return { replayable: false, reason: step.reason };
15795
- return {
15796
- replayable: true,
15797
- resolved: {
15798
- caseId: source.caseId ?? trajId2 ?? source.stepsPath,
15799
- stepsPath: source.stepsPath,
15800
- image: source.image,
15801
- cwd: source.cwd,
15802
- at,
15803
- recordedReturncode: step.recordedReturncode,
15804
- recordedStepTimeoutMs: null
15805
- }
15806
- };
15807
- }
15808
- const trajId = findingTrajectoryId(finding2);
15809
- if (!trajId) {
15810
- return {
15811
- replayable: false,
15812
- reason: "finding carries no trace://<trajectory>/ evidence ref naming its trajectory"
15813
- };
15814
- }
15815
- const failures = [];
15816
- for (const corpus of source.corpora) {
15817
- const resolution = resolveCaseResources(corpus, trajId);
15818
- if (!resolution.resolved) {
15819
- failures.push(`${corpus.name}: ${resolution.reason}${resolution.detail ? ` (${resolution.detail})` : ""}`);
15820
- continue;
15821
- }
15822
- const step = checkStep(resolution.resources.steps, at);
15823
- if (!step.ok) return { replayable: false, reason: step.reason };
15824
- return {
15825
- replayable: true,
15826
- resolved: {
15827
- caseId: trajId,
15828
- stepsPath: resolution.resources.stepsPath,
15829
- image: resolution.resources.image,
15830
- cwd: resolution.resources.cwd,
15831
- at,
15832
- recordedReturncode: step.recordedReturncode,
15833
- recordedStepTimeoutMs: resolution.resources.recordedStepTimeoutMs
15834
- }
15835
- };
15836
- }
15837
- return {
15838
- replayable: false,
15839
- reason: `trajectory ${trajId} is not replayable in any corpus \u2014 ${failures.join("; ")}`
15840
- };
15841
- }
15842
- function classifyVerdict(verdict) {
15843
- if (!verdict.armA.failureSignatureMatch) return "divergent";
15844
- if (verdict.armB?.failureVanished) return "fix-flipped";
15845
- return "reproduced";
15846
- }
15847
- var DEFAULT_SANDBOX_BASE_URL = "http://127.0.0.1:4097";
15848
- async function assertSandboxReachable(baseUrl) {
15849
- let response;
15850
- try {
15851
- response = await fetch(new URL("/health", baseUrl), { signal: AbortSignal.timeout(5e3) });
15852
- } catch (err) {
15853
- throw new Error(
15854
- `verify-findings: sandbox API unreachable at ${baseUrl} \u2014 ${err instanceof Error ? err.message : String(err)}. Executing findings requires a running sandbox orchestrator + SDK adapter (see docs/replay-verify.md); start them or pass --base-url.`
15855
- );
15856
- }
15857
- if (!response.ok) {
15858
- throw new Error(
15859
- `verify-findings: sandbox API at ${baseUrl} answered /health with HTTP ${response.status} \u2014 refusing to run proofs against degraded infrastructure`
15860
- );
15861
- }
15862
- }
15863
- function receiptExecution(verdict) {
15864
- return {
15865
- image: verdict.image,
15866
- cwd: verdict.cwd,
15867
- recordedReturncode: verdict.recordedReturncode,
15868
- signature: verdict.signature,
15869
- signatureBasis: verdict.signatureBasis,
15870
- armA: {
15871
- command: verdict.armA.command,
15872
- exitCode: verdict.armA.exitCode,
15873
- wallMs: verdict.armA.wallMs,
15874
- failureSignatureMatch: verdict.armA.failureSignatureMatch
15875
- },
15876
- armB: verdict.armB ? {
15877
- command: verdict.armB.command,
15878
- exitCode: verdict.armB.exitCode,
15879
- wallMs: verdict.armB.wallMs,
15880
- failureVanished: verdict.armB.failureVanished
15881
- } : null,
15882
- prefixExecuted: verdict.prefixExecuted,
15883
- prefixDivergences: verdict.prefixDivergences.length,
15884
- totalMs: verdict.timings.totalMs
15885
- };
15886
- }
15887
- function writeReceipt(receiptDir, finding2, verification, execution) {
15888
- const receipt = {
15889
- schema_version: "1.0.0",
15890
- produced_at: (/* @__PURE__ */ new Date()).toISOString(),
15891
- finding_id: verification.finding_id,
15892
- analyst_id: finding2.analyst_id ?? null,
15893
- subject: verification.subject,
15894
- claim: typeof finding2.claim === "string" ? finding2.claim.slice(0, 600) : null,
15895
- trajectory_id: verification.trajectory_id,
15896
- step: verification.step,
15897
- verified: verification.verified,
15898
- reason: verification.reason,
15899
- execution,
15900
- verdict_path: verification.verdict_path,
15901
- deduplicated_with: verification.deduplicated_with
15902
- };
15903
- writeFileSync3(join27(receiptDir, "receipt.json"), `${JSON.stringify(receipt, null, 2)}
15904
- `);
15905
- }
15906
- function receiptDirName(index, finding2) {
15907
- const id = typeof finding2.finding_id === "string" && finding2.finding_id.length > 0 ? finding2.finding_id.replace(/[^A-Za-z0-9_-]/g, "_") : "finding";
15908
- return `${String(index + 1).padStart(3, "0")}-${id}`;
15909
- }
15910
- async function verifyFindings(findings, options) {
15911
- if (findings.length === 0) throw new Error("verify-findings: no findings to verify");
15912
- mkdirSync3(options.out, { recursive: true });
15913
- const resolutions = findings.map((finding2) => resolveFindingReplayability(finding2, options.source));
15914
- const baseUrl = options.baseUrl ?? DEFAULT_SANDBOX_BASE_URL;
15915
- if (resolutions.some((resolution) => resolution.replayable) && !options.backend) {
15916
- if (!options.apiKey) {
15917
- throw new Error(
15918
- "verify-findings: replayable findings need a sandbox API key \u2014 export SANDBOX_API_KEY or pass --api-key-env"
15919
- );
15920
- }
15921
- await assertSandboxReachable(baseUrl);
15922
- }
15923
- const preparer = options.source.kind === "corpus" && !options.backend ? options.source.preparer ?? dockerImagePreparer() : null;
15924
- const preparedImages = /* @__PURE__ */ new Map();
15925
- const executedByKey = /* @__PURE__ */ new Map();
15926
- const verifications = [];
15927
- let executions = 0;
15928
- for (let index = 0; index < findings.length; index++) {
15929
- const finding2 = findings[index];
15930
- const resolution = resolutions[index];
15931
- const receiptDir = join27(options.out, receiptDirName(index, finding2));
15932
- mkdirSync3(receiptDir, { recursive: true });
15933
- const identity2 = {
15934
- finding_id: finding2.finding_id ?? null,
15935
- subject: finding2.subject ?? null,
15936
- trajectory_id: findingTrajectoryId(finding2)
15937
- };
15938
- if (!resolution.replayable) {
15939
- const verification2 = {
15940
- ...identity2,
15941
- step: findingReplayStep(finding2),
15942
- verified: "not-replayable",
15943
- reason: resolution.reason,
15944
- receipt: receiptDir,
15945
- verdict_path: null,
15946
- deduplicated_with: null
15947
- };
15948
- writeReceipt(receiptDir, finding2, verification2, null);
15949
- verifications.push(verification2);
15950
- options.onProgress?.(`${identity2.finding_id ?? `finding ${index + 1}`}: not-replayable \u2014 ${resolution.reason}`);
15951
- continue;
15952
- }
15953
- const resolved = resolution.resolved;
15954
- const dedupeKey = `${resolved.caseId}::${resolved.at}::${options.fixCommand ?? ""}`;
15955
- const prior = executedByKey.get(dedupeKey);
15956
- if (prior) {
15957
- const verification2 = {
15958
- ...identity2,
15959
- step: resolved.at,
15960
- verified: prior.verified,
15961
- reason: prior.reason,
15962
- receipt: receiptDir,
15963
- verdict_path: prior.verdict_path,
15964
- deduplicated_with: prior.receipt
15965
- };
15966
- const priorVerdict = prior.verdict_path ? JSON.parse(readFileSync5(prior.verdict_path, "utf8")) : null;
15967
- writeReceipt(receiptDir, finding2, verification2, priorVerdict ? receiptExecution(priorVerdict) : null);
15968
- verifications.push(verification2);
15969
- options.onProgress?.(
15970
- `${identity2.finding_id ?? `finding ${index + 1}`}: ${prior.verified} (shares proof with ${prior.finding_id ?? prior.receipt})`
15971
- );
15972
- continue;
15973
- }
15974
- let image = resolved.image;
15975
- if (preparer) {
15976
- const preparationKey = `${resolved.image}::${resolved.cwd}`;
15977
- let preparation = preparedImages.get(preparationKey);
15978
- if (!preparation) {
15979
- const ensured = await preparer.ensure(resolved.image, resolved.cwd);
15980
- preparation = ensured.succeeded ? { succeeded: true, image: ensured.value.derivedImage } : { succeeded: false, error: ensured.error };
15981
- preparedImages.set(preparationKey, preparation);
15982
- }
15983
- if (!preparation.succeeded) {
15984
- const verification2 = {
15985
- ...identity2,
15986
- step: resolved.at,
15987
- verified: "not-replayable",
15988
- reason: `replay image could not be prepared \u2014 ${preparation.error}`,
15989
- receipt: receiptDir,
15990
- verdict_path: null,
15991
- deduplicated_with: null
15992
- };
15993
- writeReceipt(receiptDir, finding2, verification2, null);
15994
- verifications.push(verification2);
15995
- options.onProgress?.(`${identity2.finding_id ?? `finding ${index + 1}`}: not-replayable \u2014 image preparation failed`);
15996
- continue;
15997
- }
15998
- image = preparation.image;
15999
- }
16000
- options.onProgress?.(
16001
- `${identity2.finding_id ?? `finding ${index + 1}`}: executing arm A at step ${resolved.at} of ${resolved.caseId} on ${image}`
16002
- );
16003
- const verdict = await replayVerify({
16004
- stepsPath: resolved.stepsPath,
16005
- image,
16006
- at: resolved.at,
16007
- fixCommand: options.fixCommand,
16008
- cwd: resolved.cwd,
16009
- out: receiptDir,
16010
- caseId: resolved.caseId,
16011
- stepTimeoutMs: options.stepTimeoutMs ?? resolved.recordedStepTimeoutMs ?? void 0,
16012
- prefixLimit: options.prefixLimit,
16013
- backend: options.backend,
16014
- apiKey: options.apiKey,
16015
- baseUrl,
16016
- maxLifetimeSeconds: options.maxLifetimeSeconds,
16017
- onProgress: options.onProgress
16018
- });
16019
- executions += 1;
16020
- const verification = {
16021
- ...identity2,
16022
- step: resolved.at,
16023
- verified: classifyVerdict(verdict),
16024
- reason: null,
16025
- receipt: receiptDir,
16026
- verdict_path: join27(receiptDir, "replay-verdict.json"),
16027
- deduplicated_with: null
16028
- };
16029
- executedByKey.set(dedupeKey, verification);
16030
- writeReceipt(receiptDir, finding2, verification, receiptExecution(verdict));
16031
- verifications.push(verification);
16032
- options.onProgress?.(`${identity2.finding_id ?? `finding ${index + 1}`}: ${verification.verified}`);
16033
- }
16034
- const counts = {
16035
- reproduced: 0,
16036
- "fix-flipped": 0,
16037
- divergent: 0,
16038
- "not-replayable": 0
16039
- };
16040
- for (const verification of verifications) counts[verification.verified] += 1;
16041
- const run = { out: options.out, verifications, counts, executions };
16042
- writeFileSync3(
16043
- join27(options.out, "verifications.json"),
16044
- `${JSON.stringify({ schema_version: "1.0.0", ...run }, null, 2)}
16045
- `
16046
- );
16047
- return run;
16048
- }
16049
- function verdictCell(verification) {
16050
- switch (verification.verified) {
16051
- case "reproduced":
16052
- return "**VERIFIED** \u2014 reproduced";
16053
- case "fix-flipped":
16054
- return "**VERIFIED** \u2014 fix-flipped";
16055
- case "divergent":
16056
- return "DIVERGENT \u2014 recorded failure did not reproduce";
16057
- case "not-replayable":
16058
- return `UNVERIFIABLE \u2014 ${verification.reason ?? "no reason recorded"}`;
16059
- }
16060
- }
16061
- function renderVerifiedFindingsSection(run) {
16062
- const lines = ["## Verified findings (executed replay)", ""];
16063
- const total = run.verifications.length;
16064
- lines.push(
16065
- `${total} finding(s) \u2192 ${run.counts.reproduced} reproduced, ${run.counts["fix-flipped"]} fix-flipped, ${run.counts.divergent} divergent, ${run.counts["not-replayable"]} not replayable (${run.executions} sandbox execution(s); findings accusing the same step share one proof).`
16066
- );
16067
- lines.push("");
16068
- lines.push("| Finding | Subject | Step | Verdict | Receipt |");
16069
- lines.push("|---|---|---:|---|---|");
16070
- for (const verification of run.verifications) {
16071
- const shared = verification.deduplicated_with ? " (shared proof)" : "";
16072
- lines.push(
16073
- `| \`${verification.finding_id ?? "\u2014"}\` | \`${verification.subject ?? "\u2014"}\` | ${verification.step ?? "\u2014"} | ${verdictCell(verification)} | \`${verification.receipt}\`${shared} |`
16074
- );
16075
- }
16076
- lines.push("");
16077
- lines.push(
16078
- "VERIFIED = the accused step was re-executed in a sandbox after replaying the trajectory prefix, and the recorded failure signature reproduced (fix-flipped: a corrected command additionally made it vanish). Each receipt directory carries receipt.json and, when executed, replay-verdict.json + report.md with real stdout/stderr."
16079
- );
16080
- lines.push("");
16081
- return lines.join("\n");
16082
- }
16083
- function readFindingsFile(path) {
16084
- const parsed = JSON.parse(readFileSync5(path, "utf8"));
16085
- const array = Array.isArray(parsed) ? parsed : parsed && typeof parsed === "object" && Array.isArray(parsed.findings) ? parsed.findings : null;
16086
- if (!array) {
16087
- throw new Error(`${path} is neither a findings array nor an object with a findings array`);
16088
- }
16089
- for (const entry of array) {
16090
- if (!entry || typeof entry !== "object") {
16091
- throw new Error(`${path}: every finding must be an object, got ${JSON.stringify(entry)}`);
16092
- }
16093
- }
16094
- return array;
16095
- }
16096
- function verifyFindingsUsage() {
16097
- return `traces verify-findings \u2014 execute analyst findings as sandbox replay proofs
14171
+ var VERIFY_FINDINGS_USAGE = `traces verify-findings \u2014 execute analyst findings as sandbox replay proofs
16098
14172
 
16099
14173
  Usage:
16100
14174
  traces verify-findings --findings FINDINGS.json --out DIR \\
@@ -16111,7 +14185,7 @@ Usage:
16111
14185
  verify against one trajectory; the image must be replay-ready
16112
14186
  (uid-1000 derived), exactly like traces replay-verify
16113
14187
  --case trajectory id of --steps; findings citing other trajectories
16114
- are honestly marked not-replayable
14188
+ are marked not-replayable
16115
14189
  --corpus resolve each finding's trajectory in CodeTraceBench corpora;
16116
14190
  images are derived via the batch uid-1000 preparer (docker)
16117
14191
  --fix-command corrected step for arm B; a reproduced finding whose fix
@@ -16124,110 +14198,83 @@ with a receipt directory carrying the executed evidence or the precise reason.
16124
14198
  Requires a running sandbox orchestrator when any finding is replayable \u2014 infra
16125
14199
  absence is an error, never a silent skip. See docs/replay-verify.md for setup.
16126
14200
  `;
14201
+ function verifyFindingsInSandbox(findings, options) {
14202
+ if (findings.length === 0) throw new Error("verify-findings: no findings to verify");
14203
+ const { sandbox, ...rest } = options;
14204
+ return verifyFindings(findings, {
14205
+ ...rest,
14206
+ backendFactory: sandboxBackendFactory(sandbox),
14207
+ preflight: async () => {
14208
+ if (!sandbox.apiKey) {
14209
+ throw new Error(
14210
+ "verify-findings: replayable findings need a sandbox API key \u2014 export SANDBOX_API_KEY or pass --api-key-env"
14211
+ );
14212
+ }
14213
+ await assertSandboxReachable(sandbox.baseUrl);
14214
+ },
14215
+ onProgress: progress
14216
+ });
16127
14217
  }
16128
- function parseVerifyFindingsArgs(argv) {
16129
- const values = /* @__PURE__ */ new Map();
16130
- const corpora = [];
16131
- for (let i = 0; i < argv.length; i++) {
16132
- const arg = argv[i];
16133
- if (arg === "--help" || arg === "-h") return "help";
16134
- if (!arg.startsWith("--")) throw new Error(`unexpected positional argument: ${arg}`);
16135
- const value = argv[++i];
16136
- if (value === void 0) throw new Error(`missing value for ${arg}`);
16137
- if (arg === "--corpus") {
16138
- corpora.push(parseCorpusFlag(value));
16139
- continue;
16140
- }
16141
- values.set(arg, value);
14218
+ async function cmdVerifyFindings(argv) {
14219
+ const parsed = parseFlags2("verify-findings", argv);
14220
+ if (parsed === "help") {
14221
+ process.stdout.write(VERIFY_FINDINGS_USAGE);
14222
+ return;
16142
14223
  }
16143
- const required = (flag) => {
16144
- const v = values.get(flag);
16145
- if (v === void 0) throw new Error(`verify-findings: ${flag} is required`);
16146
- return v;
16147
- };
16148
- const optionalNumber = (flag) => {
16149
- const v = values.get(flag);
16150
- if (v === void 0) return void 0;
16151
- const n = Number(v);
16152
- if (!Number.isFinite(n)) throw new Error(`verify-findings: ${flag} must be a number, got ${v}`);
16153
- return n;
16154
- };
16155
- const findingsPath = required("--findings");
16156
- const out = required("--out");
14224
+ const { values, corpora } = parsed;
14225
+ const flags = flagReader("verify-findings", values);
14226
+ const findingsPath = flags.required("--findings");
14227
+ const out = flags.required("--out");
16157
14228
  const direct = values.has("--steps") || values.has("--image") || values.has("--cwd");
16158
14229
  if (direct && corpora.length > 0) {
16159
14230
  throw new Error("verify-findings: pass --steps/--image/--cwd or --corpus, not both");
16160
14231
  }
16161
14232
  let source;
16162
14233
  if (direct) {
14234
+ const caseId = values.get("--case");
16163
14235
  source = {
16164
14236
  kind: "direct",
16165
- stepsPath: required("--steps"),
16166
- image: required("--image"),
16167
- cwd: required("--cwd"),
16168
- ...values.has("--case") ? { caseId: values.get("--case") } : {}
14237
+ stepsPath: flags.required("--steps"),
14238
+ image: flags.required("--image"),
14239
+ cwd: flags.required("--cwd"),
14240
+ ...caseId === void 0 ? {} : { caseId }
16169
14241
  };
16170
14242
  } else if (corpora.length > 0) {
16171
14243
  source = { kind: "corpus", corpora };
16172
14244
  } else {
16173
14245
  throw new Error("verify-findings: a replay source is required \u2014 --steps/--image/--cwd or --corpus");
16174
14246
  }
16175
- return {
16176
- findingsPath,
14247
+ const run = await verifyFindingsInSandbox(readFindingsFile(findingsPath), {
16177
14248
  source,
16178
14249
  out,
14250
+ sandbox: sandboxOptions("verify-findings", values, false),
16179
14251
  fixCommand: values.get("--fix-command"),
16180
- stepTimeoutMs: optionalNumber("--step-timeout"),
16181
- prefixLimit: optionalNumber("--prefix-limit"),
16182
- baseUrl: values.get("--base-url") ?? process.env.SANDBOX_API_URL ?? DEFAULT_SANDBOX_BASE_URL,
16183
- apiKeyEnv: values.get("--api-key-env") ?? "SANDBOX_API_KEY",
16184
- maxLifetimeSeconds: optionalNumber("--max-lifetime")
16185
- };
16186
- }
16187
- async function cmdVerifyFindings(argv) {
16188
- const parsed = parseVerifyFindingsArgs(argv);
16189
- if (parsed === "help") {
16190
- process.stdout.write(verifyFindingsUsage());
16191
- return;
16192
- }
16193
- const findings = readFindingsFile(parsed.findingsPath);
16194
- const apiKey = process.env[parsed.apiKeyEnv];
16195
- const run = await verifyFindings(findings, {
16196
- source: parsed.source,
16197
- out: parsed.out,
16198
- fixCommand: parsed.fixCommand,
16199
- stepTimeoutMs: parsed.stepTimeoutMs,
16200
- prefixLimit: parsed.prefixLimit,
16201
- apiKey,
16202
- baseUrl: parsed.baseUrl,
16203
- maxLifetimeSeconds: parsed.maxLifetimeSeconds,
16204
- onProgress: (message) => process.stderr.write(`${message}
16205
- `)
14252
+ stepTimeoutMs: flags.number("--step-timeout"),
14253
+ prefixLimit: flags.number("--prefix-limit")
16206
14254
  });
16207
14255
  process.stdout.write(`${JSON.stringify(run, null, 2)}
16208
14256
  `);
16209
- process.stderr.write(
16210
- `verify-findings: ${run.verifications.length} finding(s) \u2192 ${run.counts.reproduced} reproduced, ${run.counts["fix-flipped"]} fix-flipped, ${run.counts.divergent} divergent, ${run.counts["not-replayable"]} not-replayable (${run.executions} execution(s)) \u2192 ${run.out}
16211
- `
14257
+ progress(
14258
+ `verify-findings: ${run.verifications.length} finding(s) \u2192 ${run.counts.reproduced} reproduced, ${run.counts["fix-flipped"]} fix-flipped, ${run.counts.divergent} divergent, ${run.counts["not-replayable"]} not-replayable (${run.executions} execution(s)) \u2192 ${run.out}`
16212
14259
  );
16213
14260
  }
16214
14261
 
16215
14262
  // src/hodoscope.ts
16216
- import { createHash as createHash13 } from "crypto";
14263
+ import { createHash as createHash12 } from "crypto";
16217
14264
  import { mkdir as mkdir5, mkdtemp as mkdtemp5, readFile as readFile11, readdir as readdir17, writeFile as writeFile7 } from "fs/promises";
16218
- import { endianness, tmpdir as tmpdir6 } from "os";
16219
- import { join as join28, resolve as resolve10 } from "path";
14265
+ import { endianness, tmpdir as tmpdir5 } from "os";
14266
+ import { join as join24, resolve as resolve10 } from "path";
16220
14267
  var HODOSCOPE_VERSION = "0.2.4";
16221
14268
  async function writeHodoscopeInput(otlpPath, options = {}) {
16222
14269
  const explicitDirectory = options.directory ? resolve10(options.directory) : void 0;
16223
- const directory = explicitDirectory ?? await mkdtemp5(join28(tmpdir6(), "traces-hodoscope-input-"));
14270
+ const directory = explicitDirectory ?? await mkdtemp5(join24(tmpdir5(), "traces-hodoscope-input-"));
16224
14271
  if (explicitDirectory) {
16225
14272
  await mkdir5(directory, { recursive: true });
16226
14273
  if ((await readdir17(directory)).length > 0) {
16227
14274
  throw new Error(`Hodoscope explicit directory must be empty: ${directory}`);
16228
14275
  }
16229
14276
  }
16230
- const samples = join28(directory, "samples");
14277
+ const samples = join24(directory, "samples");
16231
14278
  await mkdir5(samples);
16232
14279
  const rows = [];
16233
14280
  for await (const value of readJsonl(otlpPath)) rows.push(parseOpenInferenceRow(value));
@@ -16252,8 +14299,8 @@ async function writeHodoscopeInput(otlpPath, options = {}) {
16252
14299
  throw new Error(`Hodoscope trace '${traceId}' contains no assistant actions`);
16253
14300
  }
16254
14301
  actionCount += actions;
16255
- const file = `${createHash13("sha256").update(traceId).digest("hex").slice(0, 24)}.json`;
16256
- await writeFile7(join28(samples, file), `${JSON.stringify(trajectory, null, 2)}
14302
+ const file = `${createHash12("sha256").update(traceId).digest("hex").slice(0, 24)}.json`;
14303
+ await writeFile7(join24(samples, file), `${JSON.stringify(trajectory, null, 2)}
16257
14304
  `, "utf8");
16258
14305
  }
16259
14306
  return {
@@ -16275,12 +14322,12 @@ function hodoscopeAnalyzer(options = {}) {
16275
14322
  return {
16276
14323
  name: "hodoscope",
16277
14324
  async analyze(otlpPath, runOptions2 = {}) {
16278
- const outputRoot = options.outputDir ? resolve10(options.outputDir) : tmpdir6();
14325
+ const outputRoot = options.outputDir ? resolve10(options.outputDir) : tmpdir5();
16279
14326
  await mkdir5(outputRoot, { recursive: true });
16280
- const directory = await mkdtemp5(join28(outputRoot, "traces-hodoscope-"));
16281
- const input = join28(directory, "input");
16282
- const analysisFile = join28(directory, "analysis.hodoscope.json");
16283
- const samplesFile = join28(directory, "samples.json");
14327
+ const directory = await mkdtemp5(join24(outputRoot, "traces-hodoscope-"));
14328
+ const input = join24(directory, "input");
14329
+ const analysisFile = join24(directory, "analysis.hodoscope.json");
14330
+ const samplesFile = join24(directory, "samples.json");
16284
14331
  try {
16285
14332
  const prepared = await writeHodoscopeInput(otlpPath, {
16286
14333
  directory: input,
@@ -16900,7 +14947,7 @@ function assertPositiveInteger(name, value) {
16900
14947
  }
16901
14948
 
16902
14949
  // src/analyst-engine-prime.ts
16903
- import { createHash as createHash14 } from "crypto";
14950
+ import { createHash as createHash13 } from "crypto";
16904
14951
  import { request as httpRequest } from "http";
16905
14952
  import { request as httpsRequest } from "https";
16906
14953
  var DEFAULT_BRIDGE_URL = "http://localhost:4181";
@@ -17124,7 +15171,7 @@ function toAnalystFinding(row, index, spanTraces, model, producedAt) {
17124
15171
  const traces = spanTraces.get(spanId);
17125
15172
  return spanEvidenceUri([...traces][0], spanId);
17126
15173
  });
17127
- const findingId = `prime-${createHash14("sha256").update(`${index}
15174
+ const findingId = `prime-${createHash13("sha256").update(`${index}
17128
15175
  ${row.claim}
17129
15176
  ${uris.join("\n")}`).digest("hex").slice(0, 12)}`;
17130
15177
  return {
@@ -17408,10 +15455,10 @@ async function applyRedactor(spans, redactor) {
17408
15455
  }
17409
15456
 
17410
15457
  // src/file-export.ts
17411
- import { createHash as createHash15 } from "crypto";
15458
+ import { createHash as createHash14 } from "crypto";
17412
15459
  import { mkdtemp as mkdtemp6, readFile as readFile12, writeFile as writeFile8 } from "fs/promises";
17413
- import { tmpdir as tmpdir7 } from "os";
17414
- import { join as join29 } from "path";
15460
+ import { tmpdir as tmpdir6 } from "os";
15461
+ import { join as join25 } from "path";
17415
15462
  import {
17416
15463
  firstNumberAttr as firstNumberAttr2,
17417
15464
  LLM_CACHED_TOKEN_ATTR_KEYS,
@@ -17446,7 +15493,7 @@ function stableJson(value) {
17446
15493
  });
17447
15494
  }
17448
15495
  function hashId(value, chars = 16) {
17449
- return createHash15("sha256").update(stableJson(value)).digest("hex").slice(0, chars);
15496
+ return createHash14("sha256").update(stableJson(value)).digest("hex").slice(0, chars);
17450
15497
  }
17451
15498
  function isoTime2(value) {
17452
15499
  if (typeof value === "string") {
@@ -17757,6 +15804,10 @@ function llmEventUsage(row) {
17757
15804
  ...costUsd !== void 0 ? { costUsd } : {}
17758
15805
  };
17759
15806
  }
15807
+ function toolPartId(row) {
15808
+ const part = objectValue2(objectValue2(row.data)?.part) ?? objectValue2(row.part);
15809
+ return part && part.type === "tool" ? stringValue3(part.id) : void 0;
15810
+ }
17760
15811
  function sandboxEventsToSpans(rows, wrapper) {
17761
15812
  const context = wrapper ?? {};
17762
15813
  const sessionId = findStringKey(context, ["session_id", "sessionId"], 3) ?? findStringKey(rows, ["session_id", "sessionId"], 4);
@@ -17788,12 +15839,32 @@ function sandboxEventsToSpans(rows, wrapper) {
17788
15839
  }
17789
15840
  })
17790
15841
  ];
15842
+ const lastEventOfPart = /* @__PURE__ */ new Map();
15843
+ const firstTimeOfPart = /* @__PURE__ */ new Map();
15844
+ const errorOfPart = /* @__PURE__ */ new Map();
15845
+ rows.forEach((row, index) => {
15846
+ const part = toolPartId(row);
15847
+ if (part === void 0) return;
15848
+ lastEventOfPart.set(part, index);
15849
+ const time = eventTimestamp(row);
15850
+ const first = firstTimeOfPart.get(part);
15851
+ if (time && (first === void 0 || time < first)) firstTimeOfPart.set(part, time);
15852
+ const status = eventStatus(row, eventType(row) ?? "event");
15853
+ if (status.status === "ERROR") errorOfPart.set(part, status);
15854
+ });
15855
+ const mergedToolEvents = rows.filter((row, index) => {
15856
+ const part = toolPartId(row);
15857
+ return part !== void 0 && lastEventOfPart.get(part) !== index;
15858
+ }).length;
15859
+ if (mergedToolEvents > 0) spans[0].attributes["traces.event.tool_part_updates_merged"] = mergedToolEvents;
17791
15860
  let previousTime = rootStart;
17792
15861
  rows.forEach((row, index) => {
15862
+ const part = toolPartId(row);
15863
+ if (part !== void 0 && lastEventOfPart.get(part) !== index) return;
17793
15864
  const type = eventType(row) ?? "event";
17794
- const time = eventTimestamp(row) ?? previousTime;
15865
+ const time = (part !== void 0 ? firstTimeOfPart.get(part) : void 0) ?? eventTimestamp(row) ?? previousTime;
17795
15866
  previousTime = time;
17796
- const { status, message } = eventStatus(row, type);
15867
+ const { status, message } = (part !== void 0 ? errorOfPart.get(part) : void 0) ?? eventStatus(row, type);
17797
15868
  const { kind, tool } = eventKind(row, type);
17798
15869
  const name = kind === "TOOL" && tool ? `tool.${tool}` : `event.${type}`;
17799
15870
  const toolInput2 = kind === "TOOL" ? eventValue(row, ["toolInput", "tool_input", "input", "args", "arguments", "full_command"]) : void 0;
@@ -17808,6 +15879,7 @@ function sandboxEventsToSpans(rows, wrapper) {
17808
15879
  }) : {}
17809
15880
  };
17810
15881
  copyDefined(extra, {
15882
+ "traces.event.part_id": part,
17811
15883
  "traces.event.id": stringValue3(row.id),
17812
15884
  "traces.event.raw_type": stringValue3(row.type),
17813
15885
  "traces.event.raw_event": stringValue3(row.event)
@@ -18057,15 +16129,15 @@ async function exportTraceEvidenceFile(inputPath, opts = {}) {
18057
16129
  }
18058
16130
  async function writeTraceEvidenceExportFile(inputPath, outPath, opts = {}) {
18059
16131
  const result = await exportTraceEvidenceFile(inputPath, opts);
18060
- const path = outPath ?? join29(await mkdtemp6(join29(tmpdir7(), "traces-export-")), "spans.openinference.jsonl");
16132
+ const path = outPath ?? join25(await mkdtemp6(join25(tmpdir6(), "traces-export-")), "spans.openinference.jsonl");
18061
16133
  await writeFile8(path, serializeSpans(result.spans), "utf8");
18062
16134
  return { ...result, path };
18063
16135
  }
18064
16136
 
18065
16137
  // src/inspect.ts
18066
16138
  import { mkdtemp as mkdtemp7, readFile as readFile13, writeFile as writeFile9 } from "fs/promises";
18067
- import { tmpdir as tmpdir8 } from "os";
18068
- import { basename as basename15, join as join30 } from "path";
16139
+ import { tmpdir as tmpdir7 } from "os";
16140
+ import { basename as basename15, join as join26 } from "path";
18069
16141
  var severityRank2 = {
18070
16142
  high: 0,
18071
16143
  medium: 1,
@@ -18081,7 +16153,7 @@ function sessionLabel(row) {
18081
16153
  const id = row.session.sessionId.length > 12 ? row.session.sessionId.slice(0, 12) : row.session.sessionId;
18082
16154
  return `${row.session.harness}:${id} (${repoLabel(row)})`;
18083
16155
  }
18084
- function pct2(value) {
16156
+ function pct(value) {
18085
16157
  return `${(value * 100).toFixed(value >= 0.1 ? 0 : 1)}%`;
18086
16158
  }
18087
16159
  function topToolErrors(row) {
@@ -18160,7 +16232,7 @@ function addSessionFindings(index, findings) {
18160
16232
  area: "session",
18161
16233
  title: `High tool-error count/rate in ${errorSessions.length}/${index.totals.sessions} session(s)`,
18162
16234
  impact: errors,
18163
- evidence: errorSessions.slice(0, 5).map((row) => `${sessionLabel(row)}: ${row.metrics.erroredToolCallCount}/${row.metrics.toolCallCount} tool errors (${pct2(row.metrics.toolErrorRate)}); ${topToolErrors(row)}`),
16235
+ evidence: errorSessions.slice(0, 5).map((row) => `${sessionLabel(row)}: ${row.metrics.erroredToolCallCount}/${row.metrics.toolCallCount} tool errors (${pct(row.metrics.toolErrorRate)}); ${topToolErrors(row)}`),
18164
16236
  next: "Fix the most common failing command/tool first, then rerun `traces index` to confirm the error count drops.",
18165
16237
  refs: errorSessions.slice(0, 5).map((row) => row.session.path)
18166
16238
  });
@@ -18354,13 +16426,13 @@ async function readSessionIndexFile(path) {
18354
16426
  return parsed;
18355
16427
  }
18356
16428
  async function writeInspectionReportFile(report, outPath) {
18357
- const path = outPath ?? join30(await mkdtemp7(join30(tmpdir8(), "traces-inspect-")), "inspection-report.md");
16429
+ const path = outPath ?? join26(await mkdtemp7(join26(tmpdir7(), "traces-inspect-")), "inspection-report.md");
18358
16430
  await writeFile9(path, renderInspectionReport(report), "utf8");
18359
16431
  return path;
18360
16432
  }
18361
16433
 
18362
16434
  // src/live.ts
18363
- import { createHash as createHash16 } from "crypto";
16435
+ import { createHash as createHash15 } from "crypto";
18364
16436
  var VERIFY_RE = /\b(test|tests|vitest|jest|pytest|go test|cargo test|tsc|typecheck|lint|biome|eslint|build|check:invariants|preflight)\b/i;
18365
16437
  var CHANGE_RE = /\b(apply_patch|edit|write|create|delete|rename|replace|update|git apply|sed -i|perl -pi|tee\s|cat\s+>)/i;
18366
16438
  var CLAIM_DONE_RE = /\b(done|fixed|complete|completed|implemented|merged|verified|all green|tests pass|ready|works now)\b/i;
@@ -18380,7 +16452,7 @@ var sleep = (ms2, signal) => new Promise((resolve13) => {
18380
16452
  signal?.addEventListener("abort", onAbort, { once: true });
18381
16453
  });
18382
16454
  function hash(parts) {
18383
- return createHash16("sha256").update(parts.join("\0")).digest("hex").slice(0, 16);
16455
+ return createHash15("sha256").update(parts.join("\0")).digest("hex").slice(0, 16);
18384
16456
  }
18385
16457
  function asString(value) {
18386
16458
  if (typeof value === "string" && value.trim()) return value;
@@ -18788,7 +16860,7 @@ async function streamSessions(opts) {
18788
16860
 
18789
16861
  // src/supervisor-run-context.ts
18790
16862
  import { readFile as readFile14, stat as stat16 } from "fs/promises";
18791
- import { basename as basename16, join as join31 } from "path";
16863
+ import { basename as basename16, join as join27 } from "path";
18792
16864
  import { materializeTreeView } from "@tangle-network/agent-runtime/kernel";
18793
16865
  import {
18794
16866
  readRuntimeSupervisorRun
@@ -18950,14 +17022,14 @@ function parseInstant(v) {
18950
17022
  }
18951
17023
  async function readFileRunContext(runDir, opts = {}) {
18952
17024
  const sources = await readRuntimeSupervisorRun(runDir);
18953
- const coordination = readCoordinationLog(await readMaybe(join31(runDir, COORDINATION_LOG_FILE)));
17025
+ const coordination = readCoordinationLog(await readMaybe(join27(runDir, COORDINATION_LOG_FILE)));
18954
17026
  return {
18955
17027
  ...sources,
18956
17028
  instanceId: opts.instanceId !== void 0 ? opts.instanceId : sources.instanceId,
18957
17029
  arm: opts.arm !== void 0 ? opts.arm : basename16(runDir),
18958
17030
  workers: sources.workers !== null && coordination.present ? workerSources(sources.workers, coordination) : sources.workers,
18959
17031
  ...sources.journal !== null ? {
18960
- rootTranscriptRef: join31(runDir, SPAWN_JOURNAL_FILE),
17032
+ rootTranscriptRef: join27(runDir, SPAWN_JOURNAL_FILE),
18961
17033
  traceCommand: `traces watch ${runDir}`
18962
17034
  } : {}
18963
17035
  };
@@ -18966,7 +17038,7 @@ function fileRunContextSupervisorRunReader(runDir, opts = {}) {
18966
17038
  return { runRef: runDir, read: () => readFileRunContext(runDir, opts) };
18967
17039
  }
18968
17040
  async function isFileRunContextDir(runDir) {
18969
- return stat16(join31(runDir, SPAWN_JOURNAL_FILE)).then((s) => s.isFile()).catch(() => false);
17041
+ return stat16(join27(runDir, SPAWN_JOURNAL_FILE)).then((s) => s.isFile()).catch(() => false);
18970
17042
  }
18971
17043
  function parseJsonObject(text2) {
18972
17044
  if (text2 === null) return null;
@@ -19032,9 +17104,9 @@ function mergeRolls(a, b) {
19032
17104
  };
19033
17105
  }
19034
17106
  async function readRunContextSnapshot(runDir) {
19035
- const journal = readRunContextJournal(await readMaybe(join31(runDir, SPAWN_JOURNAL_FILE)));
19036
- const coordination = readCoordinationLog(await readMaybe(join31(runDir, COORDINATION_LOG_FILE)));
19037
- const resultText = await readMaybe(join31(runDir, RESULT_FILE));
17107
+ const journal = readRunContextJournal(await readMaybe(join27(runDir, SPAWN_JOURNAL_FILE)));
17108
+ const coordination = readCoordinationLog(await readMaybe(join27(runDir, COORDINATION_LOG_FILE)));
17109
+ const resultText = await readMaybe(join27(runDir, RESULT_FILE));
19038
17110
  const result = parseResult(resultText);
19039
17111
  const view = safeTreeView(journal.events);
19040
17112
  const nodes = view === null ? [] : orderNodes(view, journal);
@@ -19217,7 +17289,7 @@ function wrap(text2, indent) {
19217
17289
 
19218
17290
  // src/run-span-tree.ts
19219
17291
  import { readdir as readdir18 } from "fs/promises";
19220
- import { join as join32 } from "path";
17292
+ import { join as join28 } from "path";
19221
17293
  import {
19222
17294
  firstNumberAttr as firstNumberAttr3,
19223
17295
  LLM_COST_ATTR_KEYS as LLM_COST_ATTR_KEYS2,
@@ -19492,7 +17564,7 @@ async function findSpanSourceFiles(dir) {
19492
17564
  const found = [];
19493
17565
  const entries = await readdir18(dir, { withFileTypes: true }).catch(() => []);
19494
17566
  for (const entry of entries) {
19495
- const child = join32(dir, entry.name);
17567
+ const child = join28(dir, entry.name);
19496
17568
  if (entry.isFile()) {
19497
17569
  if (SPAN_FILE_PATTERN.test(entry.name)) found.push(child);
19498
17570
  continue;
@@ -19500,7 +17572,7 @@ async function findSpanSourceFiles(dir) {
19500
17572
  if (!entry.isDirectory()) continue;
19501
17573
  const nested = await readdir18(child, { withFileTypes: true }).catch(() => []);
19502
17574
  for (const inner of nested) {
19503
- if (inner.isFile() && SPAN_FILE_PATTERN.test(inner.name)) found.push(join32(child, inner.name));
17575
+ if (inner.isFile() && SPAN_FILE_PATTERN.test(inner.name)) found.push(join28(child, inner.name));
19504
17576
  }
19505
17577
  }
19506
17578
  return found.sort();
@@ -19508,7 +17580,7 @@ async function findSpanSourceFiles(dir) {
19508
17580
 
19509
17581
  // src/supervisor-run-watch.ts
19510
17582
  import { readdir as readdir19 } from "fs/promises";
19511
- import { join as join33 } from "path";
17583
+ import { join as join29 } from "path";
19512
17584
  function budget(node) {
19513
17585
  const parts = [`${int(node.budget.maxIterations)}it`, `${int(node.budget.maxTokens)}tok`];
19514
17586
  if (node.budget.maxUsd !== void 0) parts.push(`$${node.budget.maxUsd.toFixed(2)}`);
@@ -19652,7 +17724,7 @@ async function resolveRunContextDir(dir) {
19652
17724
  const candidates = [];
19653
17725
  for (const entry of entries) {
19654
17726
  if (!entry.isDirectory()) continue;
19655
- const child = join33(dir, entry.name);
17727
+ const child = join29(dir, entry.name);
19656
17728
  if (await isFileRunContextDir(child)) candidates.push(child);
19657
17729
  }
19658
17730
  if (candidates.length === 1) return candidates[0];
@@ -19988,7 +18060,7 @@ function parseJsonAnswer(text2) {
19988
18060
 
19989
18061
  // src/ask.ts
19990
18062
  import { mkdir as mkdir6, readFile as readFile15, writeFile as writeFile10 } from "fs/promises";
19991
- import { join as join34, resolve as resolve11 } from "path";
18063
+ import { join as join30, resolve as resolve11 } from "path";
19992
18064
  import { performance } from "perf_hooks";
19993
18065
  import { CostCeilingReachedError, CostLedger } from "@tangle-network/agent-eval";
19994
18066
  import {
@@ -20543,8 +18615,8 @@ async function writeTraceQuestionsArtifacts(result, outDir) {
20543
18615
  await mkdir6(directory, { recursive: true });
20544
18616
  const paths = {
20545
18617
  directory,
20546
- result: join34(directory, "answers.json"),
20547
- report: join34(directory, "report.md"),
18618
+ result: join30(directory, "answers.json"),
18619
+ report: join30(directory, "report.md"),
20548
18620
  traces: result.otlpPath
20549
18621
  };
20550
18622
  const { report, ...machine } = result;
@@ -20606,9 +18678,9 @@ async function watchSessions(opts) {
20606
18678
  }
20607
18679
 
20608
18680
  // src/bundle-view.ts
20609
- import { createHash as createHash17 } from "crypto";
18681
+ import { createHash as createHash16 } from "crypto";
20610
18682
  import { mkdir as mkdir7, readdir as readdir20, readFile as readFile16, writeFile as writeFile11 } from "fs/promises";
20611
- import { dirname as dirname11, join as join35, resolve as resolve12 } from "path";
18683
+ import { dirname as dirname11, join as join31, resolve as resolve12 } from "path";
20612
18684
  var EVIDENCE_ONLY_ALLOW_LIST = [
20613
18685
  {
20614
18686
  path: "derived/session-index.json",
@@ -20732,7 +18804,7 @@ async function findSessionTextLeaks(opts) {
20732
18804
  const index = /* @__PURE__ */ new Map();
20733
18805
  const carried = [];
20734
18806
  for (const path of opts.carriedPaths) {
20735
- const signatures = sessionTextSignatures(await readFile16(join35(opts.viewDir, path), "utf8"));
18807
+ const signatures = sessionTextSignatures(await readFile16(join31(opts.viewDir, path), "utf8"));
20736
18808
  carried.push({ path, signatures: signatures.size });
20737
18809
  for (const signature of signatures) if (!index.has(signature)) index.set(signature, path);
20738
18810
  }
@@ -20740,7 +18812,7 @@ async function findSessionTextLeaks(opts) {
20740
18812
  const comparedSources = [];
20741
18813
  let sourceSignatures = 0;
20742
18814
  for (const sourcePath of opts.excludedPaths) {
20743
- const signatures = sessionTextSignatures(await readFile16(join35(opts.sourceDir, sourcePath), "utf8"));
18815
+ const signatures = sessionTextSignatures(await readFile16(join31(opts.sourceDir, sourcePath), "utf8"));
20744
18816
  comparedSources.push(sourcePath);
20745
18817
  sourceSignatures += signatures.size;
20746
18818
  for (const signature of signatures) {
@@ -20799,17 +18871,17 @@ async function projectSessionBundle(opts) {
20799
18871
  const candidates = [];
20800
18872
  for (const path of await listSessionBundleFiles(sourceDir)) {
20801
18873
  if (path === "manifest.json") continue;
20802
- const bytes = await readFile16(join35(sourceDir, path));
20803
- const sha2562 = createHash17("sha256").update(bytes).digest("hex");
18874
+ const bytes = await readFile16(join31(sourceDir, path));
18875
+ const sha2562 = createHash16("sha256").update(bytes).digest("hex");
20804
18876
  const recorded = sourceHashes.get(path);
20805
18877
  if (!recorded) {
20806
18878
  throw new Error(
20807
- `${join35(sourceDir, path)} is not listed in the bundle manifest \u2014 the bundle changed after assembly and cannot be projected; re-run \`traces bundle\``
18879
+ `${join31(sourceDir, path)} is not listed in the bundle manifest \u2014 the bundle changed after assembly and cannot be projected; re-run \`traces bundle\``
20808
18880
  );
20809
18881
  }
20810
18882
  if (recorded.sha256 !== sha2562) {
20811
18883
  throw new Error(
20812
- `${join35(sourceDir, path)} does not match its manifest hash (${recorded.sha256} recorded, ${sha2562} on disk) \u2014 the bundle changed after assembly and cannot be projected; re-run \`traces bundle\``
18884
+ `${join31(sourceDir, path)} does not match its manifest hash (${recorded.sha256} recorded, ${sha2562} on disk) \u2014 the bundle changed after assembly and cannot be projected; re-run \`traces bundle\``
20813
18885
  );
20814
18886
  }
20815
18887
  const verdict = classifyEvidenceOnlyPath(path);
@@ -20845,7 +18917,7 @@ async function projectSessionBundle(opts) {
20845
18917
  });
20846
18918
  continue;
20847
18919
  }
20848
- const target = join35(outDir, candidate.path);
18920
+ const target = join31(outDir, candidate.path);
20849
18921
  await mkdir7(dirname11(target), { recursive: true });
20850
18922
  await writeFile11(target, candidate.bytes);
20851
18923
  carriedPaths.push(candidate.path);
@@ -20862,7 +18934,7 @@ async function projectSessionBundle(opts) {
20862
18934
  })),
20863
18935
  matches: 0
20864
18936
  };
20865
- const sourceManifestBytes = await readFile16(join35(sourceDir, "manifest.json"));
18937
+ const sourceManifestBytes = await readFile16(join31(sourceDir, "manifest.json"));
20866
18938
  const manifest = {
20867
18939
  schemaVersion: 2,
20868
18940
  kind: "traces.session_bundle",
@@ -20876,7 +18948,7 @@ async function projectSessionBundle(opts) {
20876
18948
  knownLimits: [...source.knownLimits, ...EVIDENCE_ONLY_LIMITS],
20877
18949
  projection: {
20878
18950
  sourceDirectory: sourceDir,
20879
- sourceManifestSha256: createHash17("sha256").update(sourceManifestBytes).digest("hex"),
18951
+ sourceManifestSha256: createHash16("sha256").update(sourceManifestBytes).digest("hex"),
20880
18952
  leakCheck
20881
18953
  }
20882
18954
  };
@@ -20887,11 +18959,11 @@ async function projectSessionBundle(opts) {
20887
18959
  // src/upload-state.ts
20888
18960
  import { mkdir as mkdir8, readFile as readFile17, writeFile as writeFile12 } from "fs/promises";
20889
18961
  import { homedir as homedir12 } from "os";
20890
- import { dirname as dirname12, join as join36 } from "path";
18962
+ import { dirname as dirname12, join as join32 } from "path";
20891
18963
  import { contentHash } from "@tangle-network/agent-eval";
20892
18964
  function stateFile() {
20893
- const base = process.env.XDG_STATE_HOME ?? join36(homedir12(), ".local", "state");
20894
- return join36(base, "tangle-traces", "uploaded.json");
18965
+ const base = process.env.XDG_STATE_HOME ?? join32(homedir12(), ".local", "state");
18966
+ return join32(base, "tangle-traces", "uploaded.json");
20895
18967
  }
20896
18968
  function uploadKey(harness, sessionId) {
20897
18969
  return `${harness}:${sessionId}`;
@@ -20923,7 +18995,7 @@ function alreadyUploaded(state, harness, sessionId, hash2) {
20923
18995
  import { readFile as readFile18 } from "fs/promises";
20924
18996
  import { createRequire as createRequire2 } from "module";
20925
18997
  import { hostname as hostname2 } from "os";
20926
- import { basename as basename17, join as join37 } from "path";
18998
+ import { basename as basename17, join as join33 } from "path";
20927
18999
  import { hostedClientFromEnv } from "@tangle-network/agent-eval/hosted";
20928
19000
  import { REDACTION_VERSION } from "@tangle-network/agent-eval/traces";
20929
19001
  var require2 = createRequire2(import.meta.url);
@@ -20947,7 +19019,7 @@ async function planUpload(opts = {}) {
20947
19019
  async function gitBranch(cwd) {
20948
19020
  if (!cwd) return void 0;
20949
19021
  try {
20950
- const head = await readFile18(join37(cwd, ".git", "HEAD"), "utf8");
19022
+ const head = await readFile18(join33(cwd, ".git", "HEAD"), "utf8");
20951
19023
  const m = head.match(/^ref:\s*refs\/heads\/(.+)$/m);
20952
19024
  return m?.[1]?.trim();
20953
19025
  } catch {
@@ -21275,60 +19347,20 @@ export {
21275
19347
  hashSessionBundleFiles,
21276
19348
  writeSessionBundleManifest,
21277
19349
  readSessionBundleManifest,
19350
+ verifySessionBundle,
21278
19351
  createBundleSourceReader,
21279
19352
  assertOutsideSourceBundle,
21280
19353
  isChatTrajectoryMessage,
21281
19354
  isChatTrajectory,
21282
19355
  chatTrajectoryToSpans,
21283
19356
  importCodeTraceBench,
21284
- parseRecordedReturncode,
21285
- parseObservationOutput,
21286
- deriveFailureSignature,
21287
- ingestCodeTraceBenchSteps,
19357
+ DEFAULT_SANDBOX_BASE_URL,
21288
19358
  sandboxReplayBackend,
21289
- wrapActionForExec,
21290
- SandboxCounterfactualRunner,
21291
- replayVerify,
21292
- parseReplayVerifyArgs,
21293
- replayVerifyUsage,
21294
- cmdReplayVerify,
21295
- parseCorpusFlag,
21296
- SUBMIT_ACTION_SIGNATURE,
21297
- isSubmitAction,
21298
- readLabelEntries,
21299
- goldIncorrectSteps,
21300
- resolveCaseResources,
21301
- enumerateReplayableCases,
19359
+ assertSandboxReachable,
21302
19360
  zaiChatCaller,
21303
- clipText,
21304
- buildFixPrompt,
21305
- buildRetryFixPrompt,
21306
- countScriptCommands,
21307
- extractFixCommand,
21308
- generateFixCommand,
21309
- runFixLoop,
21310
- derivedImageTag,
21311
- dockerImagePreparer,
21312
- seededSample,
21313
- runReplayBatch,
21314
- renderBatchReport,
21315
- parseReplayBatchArgs,
21316
- replayBatchUsage,
19361
+ cmdReplayVerify,
21317
19362
  cmdReplayVerifyBatch,
21318
- parseIncorrectStepsSubject,
21319
- resolveFindingInvocation,
21320
- replayVerifyFinding,
21321
- findingReplayStep,
21322
- findingTrajectoryId,
21323
- resolveFindingReplayability,
21324
- classifyVerdict,
21325
- DEFAULT_SANDBOX_BASE_URL,
21326
- assertSandboxReachable,
21327
- verifyFindings,
21328
- renderVerifiedFindingsSection,
21329
- readFindingsFile,
21330
- verifyFindingsUsage,
21331
- parseVerifyFindingsArgs,
19363
+ verifyFindingsInSandbox,
21332
19364
  cmdVerifyFindings,
21333
19365
  writeHodoscopeInput,
21334
19366
  hodoscopeAnalyzer,