@tangle-network/traces 0.15.1 → 0.15.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1017,9 +1017,9 @@ function serializeSpans(spans) {
1017
1017
  }
1018
1018
  async function writeOtlpFile(spans, outPath) {
1019
1019
  const { mkdtemp: mkdtemp8, writeFile: writeFile13 } = await import("fs/promises");
1020
- const { tmpdir: tmpdir9 } = await import("os");
1021
- const { join: join38 } = await import("path");
1022
- const path = outPath ?? join38(await mkdtemp8(join38(tmpdir9(), "traces-")), "spans.otlp.jsonl");
1020
+ const { tmpdir: tmpdir8 } = await import("os");
1021
+ const { join: join34 } = await import("path");
1022
+ const path = outPath ?? join34(await mkdtemp8(join34(tmpdir8(), "traces-")), "spans.otlp.jsonl");
1023
1023
  await writeFile13(path, serializeSpans(spans), "utf8");
1024
1024
  return path;
1025
1025
  }
@@ -1130,6 +1130,10 @@ var KIND = {
1130
1130
  LLM: "llm",
1131
1131
  TOOL: "tool",
1132
1132
  CHAIN: "custom",
1133
+ EMBEDDING: "custom",
1134
+ RERANKER: "custom",
1135
+ GUARDRAIL: "custom",
1136
+ PROMPT: "custom",
1133
1137
  EVALUATOR: "custom",
1134
1138
  RETRIEVER: "custom",
1135
1139
  UNKNOWN: "custom"
@@ -1818,7 +1822,7 @@ import {
1818
1822
  LLM_INPUT_TOKENS as LLM_INPUT_TOKENS2,
1819
1823
  LLM_OUTPUT_TOKENS as LLM_OUTPUT_TOKENS2
1820
1824
  } from "@tangle-network/agent-eval/trace-attributes";
1821
- import { deriveHexId as deriveHexId2 } from "@tangle-network/agent-trace-contract";
1825
+ import { ATTR as ATTR3, deriveHexId as deriveHexId2 } from "@tangle-network/agent-trace-contract";
1822
1826
 
1823
1827
  // src/adapters/claude-subagent-sources.ts
1824
1828
  import { readdir as readdir4, realpath } from "fs/promises";
@@ -1984,7 +1988,10 @@ function selectWorkflowBinding(file, bindings, childStartedAt, explicitToolUseId
1984
1988
  `Cannot link Claude Workflow subagent ${file}: parent call is ambiguous`
1985
1989
  );
1986
1990
  }
1987
- return latest[0];
1991
+ return {
1992
+ binding: latest[0],
1993
+ confidence: explicitToolUseId || bindings.length === 1 ? "correlated" : "heuristic"
1994
+ };
1988
1995
  }
1989
1996
 
1990
1997
  // src/adapters/claude-subagent-sources.ts
@@ -2092,6 +2099,21 @@ async function collectClaudeSubagentSources(ref, bindings, signal) {
2092
2099
 
2093
2100
  // src/adapters/claude.ts
2094
2101
  var SERVICE2 = "claude-code";
2102
+ var AGENT_TOOL_RENAMED_IN = [2, 1, 63];
2103
+ function modelFacingToolNames(tools, version) {
2104
+ if (typeof version !== "string" || !atLeast(version, AGENT_TOOL_RENAMED_IN)) return [...tools];
2105
+ return tools.map((name) => name === "Task" ? "Agent" : name);
2106
+ }
2107
+ function atLeast(version, min) {
2108
+ const parts = version.split(".").map((p) => Number.parseInt(p, 10));
2109
+ if (parts.length < min.length || parts.some((p) => !Number.isFinite(p))) return false;
2110
+ for (let i = 0; i < min.length; i++) {
2111
+ if (parts[i] !== min[i]) return parts[i] > min[i];
2112
+ }
2113
+ return true;
2114
+ }
2115
+ var TOOL_DEFINITIONS_ATTR = "gen_ai.tool.definitions";
2116
+ var RUN_STATUS_ATTR = "run.status";
2095
2117
  var EPOCH = (/* @__PURE__ */ new Date(0)).toISOString();
2096
2118
  var HUMAN_ORIGIN = "human";
2097
2119
  var ORIGIN_EVIDENCE_ATTR = "traces.claude.actor_evidence";
@@ -2203,6 +2225,7 @@ function projectClaudeEvent(event) {
2203
2225
  cacheWriteInputTokens: event.message.usage?.cache_creation_input_tokens ?? null,
2204
2226
  content: textOf2(event.message.content) || null,
2205
2227
  contentSource: textSources(event.message, "content"),
2228
+ parentToolUseId: event.parent_tool_use_id ?? null,
2206
2229
  tools
2207
2230
  };
2208
2231
  }
@@ -2233,6 +2256,7 @@ function projectClaudeEvent(event) {
2233
2256
  timestamp,
2234
2257
  prompt: prompt || null,
2235
2258
  contentSource: textSources(event.message, "content"),
2259
+ parentToolUseId: event.parent_tool_use_id ?? null,
2236
2260
  ...prompt ? {
2237
2261
  isSidechain: event.isSidechain === true,
2238
2262
  isMeta: event.isMeta === true,
@@ -2352,15 +2376,21 @@ function consumeDistinctClaudeEvent(event, ctx, state, seen, sourcePath) {
2352
2376
  consumeClaudeEvent(accepted.projection, accepted.uid, ctx, state);
2353
2377
  return true;
2354
2378
  }
2379
+ function inStreamParent(parentToolUseId, state) {
2380
+ if (!parentToolUseId) return void 0;
2381
+ const parentTool = state.toolSpanByUseId.get(parentToolUseId);
2382
+ return parentTool ? { span: parentTool, confidence: "explicit" } : void 0;
2383
+ }
2355
2384
  function consumeClaudeEvent(event, uid, ctx, state) {
2356
2385
  if (event.kind === "assistant") {
2357
2386
  const messageId = event.messageId ?? uid;
2358
2387
  let llmSpan = state.llmSpanByMessageId.get(messageId);
2359
2388
  if (!llmSpan) {
2389
+ const parent = inStreamParent(event.parentToolUseId, state);
2360
2390
  llmSpan = span({
2361
2391
  traceId: ctx.traceId,
2362
2392
  spanId: `${ctx.idPrefix}${uid}`,
2363
- parentSpanId: ctx.rootParent,
2393
+ parentSpanId: parent ? parent.span.span_id : ctx.rootParent,
2364
2394
  name: "llm.turn",
2365
2395
  kind: "LLM",
2366
2396
  startTime: event.timestamp,
@@ -2373,6 +2403,7 @@ function consumeClaudeEvent(event, uid, ctx, state) {
2373
2403
  cacheWriteInputTokens: event.cacheWriteInputTokens,
2374
2404
  step: state.step
2375
2405
  });
2406
+ if (parent) llmSpan.attributes[ATTR3.parentConfidence] = parent.confidence;
2376
2407
  state.spans.push(llmSpan);
2377
2408
  state.llmSpanByMessageId.set(messageId, llmSpan);
2378
2409
  state.step += 1;
@@ -2444,10 +2475,11 @@ function consumeClaudeEvent(event, uid, ctx, state) {
2444
2475
  }));
2445
2476
  state.sawUserTurn = true;
2446
2477
  if (event.originKind) state.sawRecordedOrigin = true;
2478
+ const userParent = inStreamParent(event.parentToolUseId, state);
2447
2479
  const prompt = userPromptSpan({
2448
2480
  traceId: ctx.traceId,
2449
2481
  spanId: `${ctx.idPrefix}${uid}:user`,
2450
- parentSpanId: ctx.rootParent,
2482
+ parentSpanId: userParent ? userParent.span.span_id : ctx.rootParent,
2451
2483
  startTime: event.timestamp,
2452
2484
  service: SERVICE2,
2453
2485
  agent: ctx.agent,
@@ -2460,6 +2492,7 @@ function consumeClaudeEvent(event, uid, ctx, state) {
2460
2492
  prompt.attributes[ORIGIN_RECORDED_ATTR] = true;
2461
2493
  prompt.attributes[ORIGIN_KIND_ATTR] = event.originKind;
2462
2494
  }
2495
+ if (userParent) prompt.attributes[ATTR3.parentConfidence] = userParent.confidence;
2463
2496
  state.spans.push(prompt);
2464
2497
  state.promptSpans.push(prompt);
2465
2498
  state.step += 1;
@@ -2788,9 +2821,21 @@ var ClaudeAdapter = class {
2788
2821
  let selectedTurnId;
2789
2822
  let selectedTurnFound = false;
2790
2823
  let active = taskScope === "all";
2824
+ let offeredTools;
2825
+ let runResult;
2791
2826
  for await (const event of readJsonl(ref.path, sessionJsonlOptions(ref, options))) {
2792
2827
  options.signal?.throwIfAborted();
2793
- if (!discoveredTraceId && event.sessionId) discoveredTraceId = event.sessionId;
2828
+ const eventSessionId = event.sessionId ?? event.session_id;
2829
+ if (!discoveredTraceId && eventSessionId) discoveredTraceId = eventSessionId;
2830
+ if (event.type === "system" && event.subtype === "init" && Array.isArray(event.tools)) {
2831
+ offeredTools = modelFacingToolNames(
2832
+ event.tools.filter((t) => typeof t === "string"),
2833
+ event.claude_code_version
2834
+ );
2835
+ }
2836
+ if (event.type === "result" && typeof event.is_error === "boolean") {
2837
+ runResult = { failed: event.is_error, subtype: event.subtype ?? "unknown" };
2838
+ }
2794
2839
  const accepted = distinctClaudeEvent(event, seen, ref.path, `step${state.step}`);
2795
2840
  if (!accepted) continue;
2796
2841
  indexWorkflowProjection(accepted.projection, workflowIndex);
@@ -2845,6 +2890,13 @@ var ClaudeAdapter = class {
2845
2890
  service: SERVICE2,
2846
2891
  agent: SERVICE2
2847
2892
  });
2893
+ if (offeredTools) {
2894
+ root.attributes[TOOL_DEFINITIONS_ATTR] = JSON.stringify(offeredTools.map((name) => ({ type: "function", name })));
2895
+ }
2896
+ if (runResult) {
2897
+ root.attributes[RUN_STATUS_ATTR] = runResult.failed ? "failed" : "completed";
2898
+ root.status = runResult.failed ? { code: "ERROR", message: `the run ended with ${runResult.subtype}` } : { code: "OK" };
2899
+ }
2848
2900
  const spans = [root];
2849
2901
  appendAll(spans, main.spans);
2850
2902
  await this.foldSubagents(
@@ -2905,13 +2957,14 @@ var ClaudeAdapter = class {
2905
2957
  workflowBindings
2906
2958
  );
2907
2959
  const parsed = await parseClaudeSubagent(ref, traceId, agent, options);
2908
- const binding = selectWorkflowBinding(
2960
+ const { binding, confidence } = selectWorkflowBinding(
2909
2961
  basename5(file),
2910
2962
  runBindings,
2911
2963
  firstSubagentTimestamp(agent, parsed),
2912
2964
  meta.toolUseId
2913
2965
  );
2914
2966
  agent.parentToolUseId = binding.toolUseId;
2967
+ agent.parentConfidence = confidence;
2915
2968
  }
2916
2969
  agents.push(agent);
2917
2970
  }
@@ -2968,10 +3021,14 @@ var ClaudeAdapter = class {
2968
3021
  const parentAgentMissing = Boolean(
2969
3022
  agent.meta.parentAgentId && !parsedByAgentId.has(agent.meta.parentAgentId)
2970
3023
  );
3024
+ const confidence = parentSpan ? agent.parentConfidence ?? "correlated" : "unknown";
2971
3025
  const task = subagentTaskName(agent);
2972
3026
  const lifecycle = subagentLifecycleSpan(agent, parsed, traceId, parent, task, subDir);
2973
3027
  for (const item of [...parsed.spans, ...lifecycle ? [lifecycle] : []]) {
2974
- if (item.parent_span_id === `root:${traceId}`) item.parent_span_id = parent;
3028
+ if (item.parent_span_id === `root:${traceId}` || item === lifecycle) {
3029
+ item.parent_span_id = parent;
3030
+ item.attributes[ATTR3.parentConfidence] = confidence;
3031
+ }
2975
3032
  item.attributes[SUBAGENT_SPAN_ATTR] = true;
2976
3033
  if (task) item.attributes[SUBAGENT_TASK_ATTR] = task;
2977
3034
  if (agent.parentToolUseId && !parentSpan) {
@@ -3787,9 +3844,9 @@ function itemCountsJson(counts) {
3787
3844
  }
3788
3845
  function placeInnerSpans(pending, windows) {
3789
3846
  return pending.map((entry) => {
3790
- const { parent, join: join38 } = joinItem(windows, entry.startMs, entry.endMs);
3847
+ const { parent, join: join34 } = joinItem(windows, entry.startMs, entry.endMs);
3791
3848
  if (parent) entry.span.parent_span_id = parent.span_id;
3792
- entry.span.attributes["traces.codex.item_join"] = join38;
3849
+ entry.span.attributes["traces.codex.item_join"] = join34;
3793
3850
  return entry.span;
3794
3851
  });
3795
3852
  }
@@ -6434,9 +6491,9 @@ function normalizeRemote(url) {
6434
6491
  }
6435
6492
  async function readGit(cwd) {
6436
6493
  await stat12(`${cwd}/.git`);
6437
- const { execFile: execFile2 } = await import("child_process");
6438
- const { promisify: promisify2 } = await import("util");
6439
- const run = promisify2(execFile2);
6494
+ const { execFile } = await import("child_process");
6495
+ const { promisify } = await import("util");
6496
+ const run = promisify(execFile);
6440
6497
  const git = async (args) => {
6441
6498
  try {
6442
6499
  const { stdout } = await run("git", ["-C", cwd, ...args], { timeout: 5e3 });
@@ -6462,9 +6519,9 @@ async function readGit(cwd) {
6462
6519
  };
6463
6520
  }
6464
6521
  async function gitOutput(cwd, args) {
6465
- const { execFile: execFile2 } = await import("child_process");
6466
- const { promisify: promisify2 } = await import("util");
6467
- const run = promisify2(execFile2);
6522
+ const { execFile } = await import("child_process");
6523
+ const { promisify } = await import("util");
6524
+ const run = promisify(execFile);
6468
6525
  try {
6469
6526
  const { stdout } = await run("git", ["-C", cwd, ...args], { timeout: 5e3 });
6470
6527
  const v = stdout.trim();
@@ -7530,11 +7587,11 @@ function decodeEvidenceRef(value, label, knownSpanUris) {
7530
7587
  }
7531
7588
  const uri = nonEmptyString3(ref.uri, `${label}.uri`);
7532
7589
  if (kind === "span") validateSpanEvidence(uri, knownSpanUris, label);
7533
- const excerpt2 = optionalString(ref.excerpt, `${label}.excerpt`);
7590
+ const excerpt = optionalString(ref.excerpt, `${label}.excerpt`);
7534
7591
  return {
7535
7592
  kind,
7536
7593
  uri,
7537
- ...excerpt2 === void 0 ? {} : { excerpt: excerpt2 }
7594
+ ...excerpt === void 0 ? {} : { excerpt }
7538
7595
  };
7539
7596
  }
7540
7597
  function decodeFinding(value, label, knownSpanUris) {
@@ -7741,14 +7798,14 @@ function normalizeFinding(row, analystId, index, context) {
7741
7798
  to: uri
7742
7799
  });
7743
7800
  }
7744
- const excerpt2 = citation.excerpt === void 0 ? void 0 : normalizeExcerpt(citation.excerpt, span2);
7745
- if (excerpt2 !== citation.excerpt) {
7801
+ const excerpt = citation.excerpt === void 0 ? void 0 : normalizeExcerpt(citation.excerpt, span2);
7802
+ if (excerpt !== citation.excerpt) {
7746
7803
  context.log?.("finding citation normalized: json-escaped excerpt", {
7747
7804
  analyst_id: analystId,
7748
7805
  uri
7749
7806
  });
7750
7807
  }
7751
- return excerpt2 === void 0 ? { uri } : { uri, excerpt: excerpt2 };
7808
+ return excerpt === void 0 ? { uri } : { uri, excerpt };
7752
7809
  });
7753
7810
  const normalized = { ...row, evidence: evidence3 };
7754
7811
  const allowed = KIND_EXPECTED_SUBJECTS[analystId];
@@ -7797,10 +7854,10 @@ function decodeComponent(value) {
7797
7854
  return value;
7798
7855
  }
7799
7856
  }
7800
- function normalizeExcerpt(excerpt2, span2) {
7801
- if (containsText(span2.content, excerpt2)) return excerpt2;
7802
- const decoded = decodeJsonEscapes(excerpt2);
7803
- return decoded !== excerpt2 && containsText(span2.content, decoded) ? decoded : excerpt2;
7857
+ function normalizeExcerpt(excerpt, span2) {
7858
+ if (containsText(span2.content, excerpt)) return excerpt;
7859
+ const decoded = decodeJsonEscapes(excerpt);
7860
+ return decoded !== excerpt && containsText(span2.content, decoded) ? decoded : excerpt;
7804
7861
  }
7805
7862
  function decodeJsonEscapes(text2) {
7806
7863
  return text2.replace(
@@ -9118,13 +9175,13 @@ import { OtlpFileTraceStore } from "@tangle-network/agent-eval/traces";
9118
9175
  import { isUtf8 as isUtf82 } from "buffer";
9119
9176
  import { createHash as createHash9 } from "crypto";
9120
9177
  import { constants } from "fs";
9121
- import { lstat, open, realpath as realpath3 } from "fs/promises";
9178
+ import { lstat as lstat2, open, realpath as realpath3 } from "fs/promises";
9122
9179
  import { dirname as dirname8, isAbsolute as isAbsolute4, join as join20, relative as relative5, resolve as resolve7, sep as sep2 } from "path";
9123
9180
 
9124
9181
  // src/bundle.ts
9125
9182
  import { createHash as createHash8 } from "crypto";
9126
9183
  import { readFileSync as readFileSync2 } from "fs";
9127
- import { copyFile, cp, mkdir as mkdir2, readdir as readdir16, readFile as readFile7, stat as stat14, writeFile as writeFile5 } from "fs/promises";
9184
+ import { copyFile, cp, lstat, mkdir as mkdir2, readdir as readdir16, readFile as readFile7, stat as stat14, writeFile as writeFile5 } from "fs/promises";
9128
9185
  import { dirname as dirname7, join as join19, relative as relative4, resolve as resolve6, sep } from "path";
9129
9186
 
9130
9187
  // src/improvement.ts
@@ -9444,7 +9501,7 @@ function validationExitCode(validation) {
9444
9501
 
9445
9502
  // src/loop-analysis.ts
9446
9503
  import {
9447
- ATTR as ATTR3,
9504
+ ATTR as ATTR4,
9448
9505
  firstNumberAttr,
9449
9506
  firstStringAttr,
9450
9507
  ITERATION_ATTR_KEYS,
@@ -9458,11 +9515,11 @@ function attributes(span2) {
9458
9515
  return span2.attributes;
9459
9516
  }
9460
9517
  function outcomeOf(span2) {
9461
- const raw = firstStringAttr(attributes(span2), [ATTR3.outcome]);
9518
+ const raw = firstStringAttr(attributes(span2), [ATTR4.outcome]);
9462
9519
  return raw !== void 0 && OUTCOMES.includes(raw) ? raw : null;
9463
9520
  }
9464
9521
  function scoreOf(span2) {
9465
- return firstNumberAttr(attributes(span2), [ATTR3.score]) ?? null;
9522
+ return firstNumberAttr(attributes(span2), [ATTR4.score]) ?? null;
9466
9523
  }
9467
9524
  function iterationOf(span2) {
9468
9525
  const value = firstNumberAttr(attributes(span2), ITERATION_ATTR_KEYS);
@@ -9535,7 +9592,7 @@ function trendOf(iterations) {
9535
9592
  return {
9536
9593
  trend: "unscored",
9537
9594
  basis: "none",
9538
- detail: `${iterations.length} round(s) recorded, ${scored.length + graded.length} graded: the trace carries neither ${ATTR3.outcome} nor ${ATTR3.score} on enough rounds to compare them`
9595
+ detail: `${iterations.length} round(s) recorded, ${scored.length + graded.length} graded: the trace carries neither ${ATTR4.outcome} nor ${ATTR4.score} on enough rounds to compare them`
9539
9596
  };
9540
9597
  }
9541
9598
  function analyzeLoopConvergence(spans) {
@@ -9546,7 +9603,7 @@ function analyzeLoopConvergence(spans) {
9546
9603
  const iteration = iterationOf(span2);
9547
9604
  if (iteration === null) continue;
9548
9605
  iterationSpans += 1;
9549
- const loopId = firstStringAttr(attributes(span2), [ATTR3.loopId]) ?? `(unnamed loop in ${span2.trace_id})`;
9606
+ const loopId = firstStringAttr(attributes(span2), [ATTR4.loopId]) ?? `(unnamed loop in ${span2.trace_id})`;
9550
9607
  const key = `${span2.trace_id}\0${loopId}`;
9551
9608
  const verdict = verdictFor(span2, children);
9552
9609
  const bucket = byLoop.get(key) ?? { traceId: span2.trace_id, iterations: [] };
@@ -10641,8 +10698,8 @@ function renderReport(result, meta) {
10641
10698
  if (f.recommended_action) lines.push(`- **Fix:** ${f.recommended_action}`);
10642
10699
  if (f.validation_plan) lines.push(`- **Check:** ${f.validation_plan}`);
10643
10700
  for (const ev of f.evidence_refs.slice(0, 3)) {
10644
- const excerpt2 = ev.excerpt ? ` \u2014 \`${ev.excerpt.slice(0, 200)}\`` : "";
10645
- lines.push(`- **Evidence:** ${ev.kind} ${ev.uri}${excerpt2}`);
10701
+ const excerpt = ev.excerpt ? ` \u2014 \`${ev.excerpt.slice(0, 200)}\`` : "";
10702
+ lines.push(`- **Evidence:** ${ev.kind} ${ev.uri}${excerpt}`);
10646
10703
  }
10647
10704
  lines.push("");
10648
10705
  }
@@ -10966,8 +11023,8 @@ function jsonl(rows) {
10966
11023
  return rows.length === 0 ? "" : `${rows.map((row) => JSON.stringify(row)).join("\n")}
10967
11024
  `;
10968
11025
  }
10969
- function evidence(kind, uri, excerpt2) {
10970
- return excerpt2 ? { kind, uri, excerpt: excerpt2 } : { kind, uri };
11026
+ function evidence(kind, uri, excerpt) {
11027
+ return excerpt ? { kind, uri, excerpt } : { kind, uri };
10971
11028
  }
10972
11029
  function totalReactionSignals(reactions) {
10973
11030
  return Object.values(reactions.signals).reduce((sum, count2) => sum + count2, 0);
@@ -11679,8 +11736,8 @@ async function collectContextRoot(root) {
11679
11736
  if (summary) files.push(summary);
11680
11737
  }
11681
11738
  }
11682
- const progress = await summarizeFile(join18(evolve, "progress.md"), "other");
11683
- if (progress) files.push(progress);
11739
+ const progress2 = await summarizeFile(join18(evolve, "progress.md"), "other");
11740
+ if (progress2) files.push(progress2);
11684
11741
  return { root, files: files.sort((a, b) => a.path.localeCompare(b.path)) };
11685
11742
  }
11686
11743
  async function listEvolveRootFiles(evolve) {
@@ -12209,6 +12266,9 @@ var KNOWN_LIMITS = [
12209
12266
  "progress.md is copied whole; no session-dated section extraction is performed",
12210
12267
  "open pull-request state is not captured: the bundler makes no network calls"
12211
12268
  ];
12269
+ function sha256Hex(bytes) {
12270
+ return createHash8("sha256").update(bytes).digest("hex");
12271
+ }
12212
12272
  function bundlePath(...parts) {
12213
12273
  return parts.join("/");
12214
12274
  }
@@ -12244,9 +12304,9 @@ function sliceJsonlByWindow(text2, windowStartMs, windowEndMs) {
12244
12304
  return { kept, totalRows, unparseableTsRows };
12245
12305
  }
12246
12306
  async function runGitLog(cwd, sinceIso, untilIso) {
12247
- const { execFile: execFile2 } = await import("child_process");
12248
- const { promisify: promisify2 } = await import("util");
12249
- const run = promisify2(execFile2);
12307
+ const { execFile } = await import("child_process");
12308
+ const { promisify } = await import("util");
12309
+ const run = promisify(execFile);
12250
12310
  try {
12251
12311
  const { stdout } = await run(
12252
12312
  "git",
@@ -12549,6 +12609,90 @@ async function readSessionBundleManifest(bundleDir) {
12549
12609
  }
12550
12610
  return manifest;
12551
12611
  }
12612
+ function unsafeBundlePath(path) {
12613
+ if (path.length === 0) return "empty path";
12614
+ if (path.includes("\\")) return "contains a backslash";
12615
+ if (path.startsWith("/") || /^[A-Za-z]:/.test(path)) return "is absolute";
12616
+ if (path.split("/").some((part) => part === ".." || part === "." || part === "")) {
12617
+ return 'has an empty, "." or ".." segment';
12618
+ }
12619
+ return null;
12620
+ }
12621
+ async function nonRegularEntries(root) {
12622
+ const out = [];
12623
+ const pending = [resolve6(root)];
12624
+ while (pending.length > 0) {
12625
+ const dir = pending.pop();
12626
+ for (const entry of await readdir16(dir, { withFileTypes: true })) {
12627
+ const path = join19(dir, entry.name);
12628
+ if (entry.isDirectory()) pending.push(path);
12629
+ else if (!entry.isFile()) out.push(relative4(resolve6(root), path).split(sep).join("/"));
12630
+ }
12631
+ }
12632
+ return out.sort();
12633
+ }
12634
+ async function verifySessionBundle(bundleDir) {
12635
+ const directory = resolve6(bundleDir);
12636
+ const manifest = await readSessionBundleManifest(directory);
12637
+ const issues = [];
12638
+ const listed = /* @__PURE__ */ new Set();
12639
+ let matched = 0;
12640
+ for (const file of manifest.files) {
12641
+ const unsafe = unsafeBundlePath(file.path);
12642
+ if (unsafe) {
12643
+ issues.push({ kind: "unsafe-path", path: file.path, detail: `manifest path ${unsafe}` });
12644
+ continue;
12645
+ }
12646
+ listed.add(file.path);
12647
+ const info = await lstat(join19(directory, file.path)).catch((error) => {
12648
+ if (isMissingPathError(error)) return null;
12649
+ throw error;
12650
+ });
12651
+ if (!info) {
12652
+ issues.push({ kind: "missing", path: file.path, detail: "listed in the manifest, not on disk" });
12653
+ continue;
12654
+ }
12655
+ if (!info.isFile()) continue;
12656
+ const sha2562 = await fileSha256(join19(directory, file.path));
12657
+ if (info.size !== file.bytes || sha2562 !== file.sha256) {
12658
+ issues.push({
12659
+ kind: "mismatch",
12660
+ path: file.path,
12661
+ detail: `recorded ${file.bytes} bytes sha256 ${file.sha256}, found ${info.size} bytes sha256 ${sha2562}`
12662
+ });
12663
+ continue;
12664
+ }
12665
+ matched += 1;
12666
+ }
12667
+ for (const path of await listSessionBundleFiles(directory)) {
12668
+ if (path === "manifest.json" || listed.has(path)) continue;
12669
+ issues.push({ kind: "unexpected", path, detail: "on disk, not listed in the manifest" });
12670
+ }
12671
+ for (const path of await nonRegularEntries(directory)) {
12672
+ issues.push({ kind: "not-a-file", path, detail: "a symbolic link or special file; a bundle holds regular files only" });
12673
+ }
12674
+ let source;
12675
+ if (manifest.projection) {
12676
+ const sourceManifest = join19(manifest.projection.sourceDirectory, "manifest.json");
12677
+ const bytes = await readFile7(sourceManifest).catch((error) => {
12678
+ if (isMissingPathError(error)) return null;
12679
+ throw error;
12680
+ });
12681
+ source = {
12682
+ directory: manifest.projection.sourceDirectory,
12683
+ status: bytes === null ? "unavailable" : sha256Hex(bytes) === manifest.projection.sourceManifestSha256 ? "match" : "mismatch"
12684
+ };
12685
+ }
12686
+ return {
12687
+ ok: issues.length === 0 && source?.status !== "mismatch",
12688
+ directory,
12689
+ view: manifest.view,
12690
+ listed: manifest.files.length,
12691
+ matched,
12692
+ issues,
12693
+ ...source ? { source } : {}
12694
+ };
12695
+ }
12552
12696
  var cachedVersion;
12553
12697
  function tracesPackageVersion() {
12554
12698
  if (cachedVersion) return cachedVersion;
@@ -12578,7 +12722,7 @@ async function openBundleFile(root, file) {
12578
12722
  let current = root;
12579
12723
  for (const part of relative5(root, path).split(sep2)) {
12580
12724
  current = join20(current, part);
12581
- if ((await lstat(current)).isSymbolicLink()) throw new Error("bundle source symlinks are not permitted");
12725
+ if ((await lstat2(current)).isSymbolicLink()) throw new Error("bundle source symlinks are not permitted");
12582
12726
  }
12583
12727
  if (await realpath3(path) !== path) throw new Error("bundle source resolved outside its declared path");
12584
12728
  const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
@@ -12614,9 +12758,9 @@ async function createBundleSourceReader(directory, spans, options = {}) {
12614
12758
  const maxRecordBytes = options.maxRecordBytes ?? 16 * 1024 * 1024;
12615
12759
  if (!Number.isSafeInteger(maxRecordBytes) || maxRecordBytes < 1) throw new Error("maxRecordBytes must be a positive safe integer");
12616
12760
  options.signal?.throwIfAborted();
12617
- if ((await lstat(resolve7(directory))).isSymbolicLink()) throw new Error("bundle directory must not be a symlink");
12761
+ if ((await lstat2(resolve7(directory))).isSymbolicLink()) throw new Error("bundle directory must not be a symlink");
12618
12762
  const root = await realpath3(resolve7(directory));
12619
- if ((await lstat(join20(root, "manifest.json"))).isSymbolicLink()) throw new Error("bundle manifest must not be a symlink");
12763
+ if ((await lstat2(join20(root, "manifest.json"))).isSymbolicLink()) throw new Error("bundle manifest must not be a symlink");
12620
12764
  const manifest = await readSessionBundleManifest(root);
12621
12765
  if (manifest.view !== "full") throw new Error("original source reads require an explicitly authorized full bundle");
12622
12766
  if (!Array.isArray(manifest.files)) throw new Error("bundle manifest has no file receipts");
@@ -13042,7 +13186,7 @@ import { basename as basename14, dirname as dirname10, join as join23, resolve a
13042
13186
  import { createHash as createHash11, randomUUID } from "crypto";
13043
13187
  import {
13044
13188
  link,
13045
- lstat as lstat2,
13189
+ lstat as lstat3,
13046
13190
  mkdir as mkdir3,
13047
13191
  readFile as readFile8,
13048
13192
  realpath as realpath4,
@@ -13211,7 +13355,7 @@ async function optionalContainedFile(root, candidate) {
13211
13355
  }
13212
13356
  async function assertPathMissing(path) {
13213
13357
  try {
13214
- await lstat2(path);
13358
+ await lstat3(path);
13215
13359
  } catch (error) {
13216
13360
  if (isErrno(error, "ENOENT")) return;
13217
13361
  throw error;
@@ -13725,54 +13869,18 @@ function validateRevision(value) {
13725
13869
  return value;
13726
13870
  }
13727
13871
 
13728
- // src/replay-verify.ts
13729
- import { mkdirSync, readFileSync as readFileSync3, writeFileSync } from "fs";
13730
- import { join as join24 } from "path";
13872
+ // src/replay.ts
13873
+ import { readFileSync as readFileSync3 } from "fs";
13731
13874
  import {
13732
- InMemoryTraceStore as InMemoryTraceStore2,
13733
- runCounterfactual,
13734
- TraceEmitter
13735
- } from "@tangle-network/agent-eval";
13875
+ enumerateReplayableCases,
13876
+ parseCorpusFlag,
13877
+ readFindingsFile,
13878
+ replayVerify,
13879
+ runReplayBatch,
13880
+ verifyFindings
13881
+ } from "@tangle-network/agent-eval/trajectory-replay";
13736
13882
  import { Sandbox } from "@tangle-network/sandbox";
13737
- function parseRecordedReturncode(observation) {
13738
- if (!observation) return null;
13739
- const m = /<returncode>(-?\d+)<\/returncode>/.exec(observation);
13740
- return m ? Number(m[1]) : null;
13741
- }
13742
- function parseObservationOutput(observation) {
13743
- if (!observation) return "";
13744
- const m = /<output>\n?([\s\S]*?)\n?<\/output>/.exec(observation);
13745
- return m ? m[1] : observation;
13746
- }
13747
- function deriveFailureSignature(observation) {
13748
- const line = parseObservationOutput(observation).split("\n").find((l) => /\berror\b/i.test(l));
13749
- return line ? line.trim().slice(0, 200) : null;
13750
- }
13751
- async function ingestCodeTraceBenchSteps(store, steps, caseId) {
13752
- let tick = 0;
13753
- const emitter = new TraceEmitter(store, { now: () => ++tick });
13754
- await emitter.startRun({
13755
- scenarioId: caseId,
13756
- tags: { source: "codetracebench-replay", caseId }
13757
- });
13758
- for (const step of steps) {
13759
- if (typeof step.action !== "string" || step.action.length === 0) {
13760
- throw new Error(`replay-verify: step ${step.step_id} has no action`);
13761
- }
13762
- const handle = await emitter.tool({
13763
- name: `step:${step.step_id}`,
13764
- toolName: "shell",
13765
- args: { command: step.action },
13766
- result: {
13767
- returncode: parseRecordedReturncode(step.observation),
13768
- observation: step.observation
13769
- }
13770
- });
13771
- await handle.end();
13772
- }
13773
- await emitter.endRun({ pass: true, notes: "recorded trajectory (ingested for replay)" });
13774
- return { runId: emitter.runId, stepCount: steps.length };
13775
- }
13883
+ var DEFAULT_SANDBOX_BASE_URL = "http://127.0.0.1:4097";
13776
13884
  function sandboxReplayBackend(options) {
13777
13885
  const client = new Sandbox({ apiKey: options.apiKey, baseUrl: options.baseUrl });
13778
13886
  return {
@@ -13797,365 +13905,117 @@ function sandboxReplayBackend(options) {
13797
13905
  }
13798
13906
  };
13799
13907
  }
13800
- function wrapActionForExec(action, cwd) {
13801
- const b64 = Buffer.from(action, "utf8").toString("base64");
13802
- const quotedCwd = `'${cwd.replaceAll("'", `'\\''`)}'`;
13803
- return `cd ${quotedCwd} && printf %s ${b64} | base64 -d | sh`;
13804
- }
13805
- function toolCommand(span2) {
13806
- const args = span2.args;
13807
- if (!args || typeof args.command !== "string") {
13808
- throw new Error(`replay-verify: span ${span2.name} carries no command`);
13809
- }
13810
- return args.command;
13811
- }
13812
- function recordedReturncodeOf(span2) {
13813
- const result = span2.result;
13814
- return typeof result?.returncode === "number" ? result.returncode : null;
13815
- }
13816
- var SandboxCounterfactualRunner = class {
13817
- constructor(backend, options) {
13818
- this.backend = backend;
13819
- this.options = options;
13820
- }
13821
- backend;
13822
- options;
13823
- lastPrefix = null;
13824
- lastArm = null;
13825
- async executeFrom(ctx, emitter) {
13826
- const { cwd, stepTimeoutMs, prefixLimit, onProgress } = this.options;
13827
- const session = await this.backend.open();
13828
- try {
13829
- const toolSteps = ctx.prefix.filter(
13830
- (s) => s.span.kind === "tool"
13831
- );
13832
- const toExecute = prefixLimit !== void 0 ? toolSteps.slice(0, prefixLimit) : toolSteps;
13833
- const divergences = [];
13834
- const prefixStart = Date.now();
13835
- for (const step of toExecute) {
13836
- const command = toolCommand(step.span);
13837
- const expected = recordedReturncodeOf(step.span);
13838
- const started = Date.now();
13839
- const result = await session.exec(wrapActionForExec(command, cwd), stepTimeoutMs);
13840
- const handle = await emitter.sandbox({
13841
- name: step.span.name,
13842
- command,
13843
- exitCode: result.exitCode,
13844
- wallMs: Date.now() - started
13845
- });
13846
- await handle.end();
13847
- if (expected !== null && expected !== result.exitCode) {
13848
- divergences.push({
13849
- step: step.index + 1,
13850
- expectedReturncode: expected,
13851
- actualExit: result.exitCode
13852
- });
13853
- }
13854
- onProgress?.(
13855
- `prefix ${step.span.name}: exit=${result.exitCode}` + (expected !== null && expected !== result.exitCode ? ` (recorded ${expected})` : "")
13856
- );
13857
- }
13858
- this.lastPrefix = {
13859
- prefixExecuted: toExecute.length,
13860
- prefixDivergences: divergences,
13861
- wallMs: Date.now() - prefixStart
13862
- };
13863
- if (ctx.mutatedStep.span.kind !== "tool") {
13864
- throw new Error("replay-verify: mutation target is not a tool span");
13865
- }
13866
- const armCommand = toolCommand(ctx.mutatedStep.span);
13867
- const armStart = Date.now();
13868
- const armResult = await session.exec(wrapActionForExec(armCommand, cwd), stepTimeoutMs);
13869
- const armWallMs = Date.now() - armStart;
13870
- const armHandle = await emitter.sandbox({
13871
- name: `candidate:${ctx.mutatedStep.span.name}`,
13872
- command: armCommand,
13873
- exitCode: armResult.exitCode,
13874
- wallMs: armWallMs
13875
- });
13876
- await armHandle.end();
13877
- this.lastArm = {
13878
- command: armCommand,
13879
- exitCode: armResult.exitCode,
13880
- stdout: armResult.stdout,
13881
- stderr: armResult.stderr,
13882
- wallMs: armWallMs
13883
- };
13884
- onProgress?.(`candidate step: exit=${armResult.exitCode} in ${armWallMs}ms`);
13885
- await emitter.endRun({
13886
- pass: armResult.exitCode === 0,
13887
- notes: `prefix ${toExecute.length} steps, ${divergences.length} divergences`
13888
- });
13889
- } finally {
13890
- await session.close();
13891
- }
13892
- }
13893
- };
13894
- function excerpt(text2, limit = 2400) {
13895
- if (text2.length <= limit) return text2;
13896
- const head = text2.slice(0, limit / 2);
13897
- const tail = text2.slice(-limit / 2);
13898
- return `${head}
13899
- \u2026 [${text2.length - limit} chars elided] \u2026
13900
- ${tail}`;
13901
- }
13902
- async function replayVerify(options) {
13903
- const totalStart = Date.now();
13904
- const steps = JSON.parse(readFileSync3(options.stepsPath, "utf8"));
13905
- if (!Array.isArray(steps) || steps.length === 0) {
13906
- throw new Error(`replay-verify: ${options.stepsPath} is not a non-empty steps array`);
13907
- }
13908
- const index = options.at - 1;
13909
- if (index < 0 || index >= steps.length) {
13910
- throw new Error(`replay-verify: --at ${options.at} out of range [1, ${steps.length}]`);
13911
- }
13912
- const target = steps[index];
13913
- if (target.step_id !== options.at) {
13908
+ function sandboxBackendFactory(options) {
13909
+ return (image) => sandboxReplayBackend({ ...options, image });
13910
+ }
13911
+ async function assertSandboxReachable(baseUrl) {
13912
+ let response;
13913
+ try {
13914
+ response = await fetch(new URL("/health", baseUrl), { signal: AbortSignal.timeout(5e3) });
13915
+ } catch (err) {
13914
13916
  throw new Error(
13915
- `replay-verify: steps[${index}].step_id=${target.step_id} != --at ${options.at}; steps.json must be ordered with 1-based contiguous step_ids`
13917
+ `verify-findings: sandbox API unreachable at ${baseUrl} \u2014 ${err instanceof Error ? err.message : String(err)}. Executing findings requires a running sandbox orchestrator + SDK adapter (see docs/replay-verify.md); start them or pass --base-url.`
13916
13918
  );
13917
13919
  }
13918
- const caseId = options.caseId ?? options.stepsPath;
13919
- const recordedReturncode = parseRecordedReturncode(target.observation);
13920
- const signature = options.signature ?? deriveFailureSignature(target.observation);
13921
- const signatureBasis = signature ? "returncode+output-substring" : "returncode-only";
13922
- const backend = options.backend ?? sandboxReplayBackend({
13923
- image: options.image,
13924
- apiKey: requireOption(options.apiKey, "apiKey"),
13925
- baseUrl: requireOption(options.baseUrl, "baseUrl"),
13926
- maxLifetimeSeconds: options.maxLifetimeSeconds
13927
- });
13928
- const runnerOptions = {
13929
- cwd: options.cwd,
13930
- stepTimeoutMs: options.stepTimeoutMs ?? 3e5,
13931
- prefixLimit: options.prefixLimit,
13932
- onProgress: options.onProgress
13933
- };
13934
- const store = new InMemoryTraceStore2();
13935
- const { runId } = await ingestCodeTraceBenchSteps(store, steps, caseId);
13936
- options.onProgress?.(`arm A: replaying ${index} prefix steps then recorded step ${options.at}`);
13937
- const armARunner = new SandboxCounterfactualRunner(backend, runnerOptions);
13938
- const armAStart = Date.now();
13939
- const armAResult = await runCounterfactual(
13940
- store,
13941
- runId,
13942
- { kind: "custom", at: index, describe: "arm-A identity replay", apply: (s) => s },
13943
- armARunner
13944
- );
13945
- const armAMs = Date.now() - armAStart;
13946
- const armAExec = requireExec(armARunner, "arm A");
13947
- const armAOutput = `${armAExec.stdout}
13948
- ${armAExec.stderr}`;
13949
- const failureSignatureMatch = recordedReturncode !== null && armAExec.exitCode === recordedReturncode && (signature ? armAOutput.includes(signature) : true);
13950
- let armBRunner = null;
13951
- let armBExec = null;
13952
- let armBRunId = null;
13953
- let armBMs = null;
13954
- if (options.fixCommand) {
13955
- const fixCommand = options.fixCommand;
13956
- options.onProgress?.(`arm B: replaying prefix then corrected step`);
13957
- armBRunner = new SandboxCounterfactualRunner(backend, runnerOptions);
13958
- const armBStart = Date.now();
13959
- const armBResult = await runCounterfactual(
13960
- store,
13961
- runId,
13962
- {
13963
- kind: "custom",
13964
- at: index,
13965
- describe: "arm-B corrected step",
13966
- apply: (step) => ({
13967
- ...step,
13968
- span: { ...step.span, args: { command: fixCommand } }
13969
- })
13970
- },
13971
- armBRunner
13920
+ if (!response.ok) {
13921
+ throw new Error(
13922
+ `verify-findings: sandbox API at ${baseUrl} answered /health with HTTP ${response.status} \u2014 refusing to run proofs against degraded infrastructure`
13972
13923
  );
13973
- armBMs = Date.now() - armBStart;
13974
- armBRunId = armBResult.counterfactualRunId;
13975
- armBExec = requireExec(armBRunner, "arm B");
13976
- }
13977
- const armBOutput = armBExec ? `${armBExec.stdout}
13978
- ${armBExec.stderr}` : null;
13979
- const verdict = {
13980
- case: caseId,
13981
- image: options.image,
13982
- driver: options.driverLabel ?? "docker",
13983
- k: options.at,
13984
- cwd: options.cwd,
13985
- recordedReturncode,
13986
- signature,
13987
- signatureBasis,
13988
- prefixExecuted: armARunner.lastPrefix?.prefixExecuted ?? 0,
13989
- prefixDivergences: armARunner.lastPrefix?.prefixDivergences ?? [],
13990
- armA: {
13991
- command: armAExec.command,
13992
- exitCode: armAExec.exitCode,
13993
- wallMs: armAExec.wallMs,
13994
- prefix: armARunner.lastPrefix,
13995
- failureSignatureMatch
13996
- },
13997
- armB: armBExec && armBRunner ? {
13998
- command: armBExec.command,
13999
- exitCode: armBExec.exitCode,
14000
- wallMs: armBExec.wallMs,
14001
- prefix: armBRunner.lastPrefix,
14002
- failureVanished: armBExec.exitCode === 0 && (signature && armBOutput ? !armBOutput.includes(signature) : true)
14003
- } : null,
14004
- timings: { armAMs, armBMs, totalMs: Date.now() - totalStart },
14005
- runIds: {
14006
- original: runId,
14007
- armA: armAResult.counterfactualRunId,
14008
- armB: armBRunId
14009
- }
14010
- };
14011
- mkdirSync(options.out, { recursive: true });
14012
- writeFileSync(join24(options.out, "replay-verdict.json"), `${JSON.stringify(verdict, null, 2)}
14013
- `);
14014
- writeFileSync(
14015
- join24(options.out, "report.md"),
14016
- renderReport2(verdict, target, armAExec, armBExec)
14017
- );
14018
- return verdict;
14019
- }
14020
- function requireOption(value, name) {
14021
- if (value === void 0) throw new Error(`replay-verify: ${name} is required`);
14022
- return value;
14023
- }
14024
- function requireExec(runner, arm) {
14025
- if (!runner.lastArm) throw new Error(`replay-verify: ${arm} finished without executing step k`);
14026
- return runner.lastArm;
13924
+ }
14027
13925
  }
14028
- function renderReport2(verdict, target, armA, armB) {
14029
- const lines = [];
14030
- const divergencePct = verdict.prefixExecuted > 0 ? (verdict.prefixDivergences.length / verdict.prefixExecuted * 100).toFixed(1) : "0.0";
14031
- lines.push(`# Replay verification \u2014 ${verdict.case}`);
14032
- lines.push("");
14033
- lines.push(`- image: \`${verdict.image}\` (driver: ${verdict.driver})`);
14034
- lines.push(`- error-critical step k = ${verdict.k}, cwd \`${verdict.cwd}\``);
14035
- lines.push(`- recorded returncode at k: ${verdict.recordedReturncode ?? "none recorded"}`);
14036
- lines.push(
14037
- `- failure signature (${verdict.signatureBasis}): ${verdict.signature ? `\`${verdict.signature}\`` : "\u2014"}`
14038
- );
14039
- lines.push("");
14040
- lines.push(`## Prefix replay (steps 1..${verdict.k - 1})`);
14041
- lines.push("");
14042
- lines.push(
14043
- `Executed ${verdict.prefixExecuted} steps; ${verdict.prefixDivergences.length} returncode divergences (${divergencePct}%).`
14044
- );
14045
- if (verdict.prefixDivergences.length > 0) {
14046
- lines.push("");
14047
- lines.push("| step | recorded rc | replayed exit |");
14048
- lines.push("| --- | --- | --- |");
14049
- for (const d of verdict.prefixDivergences) {
14050
- lines.push(`| ${d.step} | ${d.expectedReturncode} | ${d.actualExit} |`);
13926
+ function zaiChatCaller(options) {
13927
+ async function callOnce(system, user, maxTokens) {
13928
+ try {
13929
+ const response = await fetch(`${options.baseUrl.replace(/\/$/, "")}/chat/completions`, {
13930
+ method: "POST",
13931
+ headers: { authorization: `Bearer ${options.apiKey}`, "content-type": "application/json" },
13932
+ body: JSON.stringify({
13933
+ model: options.model,
13934
+ messages: [
13935
+ { role: "system", content: system },
13936
+ { role: "user", content: user }
13937
+ ],
13938
+ temperature: 0,
13939
+ max_tokens: maxTokens
13940
+ }),
13941
+ signal: AbortSignal.timeout(options.timeoutMs ?? 18e4)
13942
+ });
13943
+ const bodyText = await response.text();
13944
+ if (!response.ok) return { succeeded: false, error: `HTTP ${response.status}: ${bodyText.slice(0, 400)}` };
13945
+ const body = JSON.parse(bodyText);
13946
+ if (body.error) return { succeeded: false, error: String(body.error.message ?? "provider error") };
13947
+ const content = body.choices?.[0]?.message?.content;
13948
+ if (typeof content !== "string" || content.length === 0) {
13949
+ return { succeeded: false, error: `empty completion content: ${bodyText.slice(0, 400)}` };
13950
+ }
13951
+ const promptTokens = body.usage?.prompt_tokens;
13952
+ const completionTokens = body.usage?.completion_tokens;
13953
+ const usage = typeof promptTokens === "number" && typeof completionTokens === "number" ? { promptTokens, completionTokens } : null;
13954
+ return { succeeded: true, value: { content, usage } };
13955
+ } catch (err) {
13956
+ return { succeeded: false, error: err instanceof Error ? err.message : String(err) };
14051
13957
  }
14052
13958
  }
14053
- lines.push("");
14054
- lines.push("## Arm A \u2014 recorded step k, replayed");
14055
- lines.push("");
14056
- lines.push("```sh");
14057
- lines.push(armA.command);
14058
- lines.push("```");
14059
- lines.push("");
14060
- lines.push(
14061
- `Exit ${armA.exitCode} in ${armA.wallMs}ms \u2014 failureSignatureMatch: **${verdict.armA.failureSignatureMatch}**`
14062
- );
14063
- lines.push("");
14064
- lines.push("Recorded observation excerpt:");
14065
- lines.push("");
14066
- lines.push("```");
14067
- lines.push(excerpt(parseObservationOutput(target.observation)));
14068
- lines.push("```");
14069
- lines.push("");
14070
- lines.push("Replayed stdout+stderr excerpt:");
14071
- lines.push("");
14072
- lines.push("```");
14073
- lines.push(excerpt(`${armA.stdout}
14074
- ${armA.stderr}`.trim()));
14075
- lines.push("```");
14076
- if (armB && verdict.armB) {
14077
- lines.push("");
14078
- lines.push("## Arm B \u2014 corrected step k");
14079
- lines.push("");
14080
- lines.push("```sh");
14081
- lines.push(armB.command);
14082
- lines.push("```");
14083
- lines.push("");
14084
- lines.push(
14085
- `Exit ${armB.exitCode} in ${armB.wallMs}ms \u2014 failureVanished: **${verdict.armB.failureVanished}** (prefix re-replayed in a fresh sandbox: ${verdict.armB.prefix.prefixExecuted} steps, ${verdict.armB.prefix.prefixDivergences.length} divergences)`
14086
- );
14087
- lines.push("");
14088
- lines.push("Replayed stdout+stderr excerpt:");
14089
- lines.push("");
14090
- lines.push("```");
14091
- lines.push(excerpt(`${armB.stdout}
14092
- ${armB.stderr}`.trim()));
14093
- lines.push("```");
14094
- }
14095
- lines.push("");
14096
- lines.push("## Timings");
14097
- lines.push("");
14098
- lines.push(
14099
- `arm A ${verdict.timings.armAMs}ms, arm B ${verdict.timings.armBMs ?? "\u2014"}ms, total ${verdict.timings.totalMs}ms.`
14100
- );
14101
- lines.push("");
14102
- return lines.join("\n");
13959
+ return {
13960
+ async complete(system, user) {
13961
+ const baseCap = options.maxTokens ?? 16384;
13962
+ const first = await callOnce(system, user, baseCap);
13963
+ if (first.succeeded || !/finish_reason":"length/.test(first.error)) return first;
13964
+ return callOnce(system, user, baseCap * 2);
13965
+ }
13966
+ };
14103
13967
  }
14104
- function parseReplayVerifyArgs(argv) {
13968
+ function parseFlags2(command, argv, switches = []) {
14105
13969
  const values = /* @__PURE__ */ new Map();
13970
+ const corpora = [];
13971
+ const seen = /* @__PURE__ */ new Set();
14106
13972
  for (let i = 0; i < argv.length; i++) {
14107
13973
  const arg = argv[i];
14108
13974
  if (arg === "--help" || arg === "-h") return "help";
14109
- if (!arg.startsWith("--")) throw new Error(`unexpected positional argument: ${arg}`);
13975
+ if (switches.includes(arg)) {
13976
+ seen.add(arg);
13977
+ continue;
13978
+ }
13979
+ if (!arg.startsWith("--")) throw new Error(`${command}: unexpected positional argument: ${arg}`);
14110
13980
  const value = argv[++i];
14111
- if (value === void 0) throw new Error(`missing value for ${arg}`);
14112
- values.set(arg, value);
13981
+ if (value === void 0) throw new Error(`${command}: missing value for ${arg}`);
13982
+ if (arg === "--corpus") corpora.push(parseCorpusFlag(value));
13983
+ else values.set(arg, value);
14113
13984
  }
14114
- const required = (flag) => {
14115
- const v = values.get(flag);
14116
- if (v === void 0) throw new Error(`replay-verify: ${flag} is required`);
14117
- return v;
14118
- };
14119
- const optionalNumber = (flag) => {
14120
- const v = values.get(flag);
14121
- if (v === void 0) return void 0;
14122
- const n = Number(v);
14123
- if (!Number.isFinite(n)) throw new Error(`replay-verify: ${flag} must be a number, got ${v}`);
14124
- return n;
13985
+ return { values, corpora, switches: seen };
13986
+ }
13987
+ function flagReader(command, values) {
13988
+ return {
13989
+ required(flag) {
13990
+ const v = values.get(flag);
13991
+ if (v === void 0) throw new Error(`${command}: ${flag} is required`);
13992
+ return v;
13993
+ },
13994
+ number(flag) {
13995
+ const v = values.get(flag);
13996
+ if (v === void 0) return void 0;
13997
+ const n = Number(v);
13998
+ if (!Number.isFinite(n)) throw new Error(`${command}: ${flag} must be a number, got ${v}`);
13999
+ return n;
14000
+ }
14125
14001
  };
14126
- const stepsPath = required("--steps");
14127
- const image = required("--image");
14128
- const at = Number(required("--at"));
14129
- const cwd = required("--cwd");
14130
- const out = required("--out");
14131
- if (!Number.isInteger(at) || at < 1) {
14132
- throw new Error(`replay-verify: --at must be a positive step_id, got ${values.get("--at")}`);
14133
- }
14134
- let fixCommand = values.get("--fix-command");
14135
- const fixFile = values.get("--fix-command-file");
14136
- if (fixCommand !== void 0 && fixFile !== void 0) {
14137
- throw new Error("replay-verify: pass --fix-command or --fix-command-file, not both");
14002
+ }
14003
+ function sandboxOptions(command, values, requireKey) {
14004
+ const apiKeyEnv = values.get("--api-key-env") ?? "SANDBOX_API_KEY";
14005
+ const apiKey = process.env[apiKeyEnv];
14006
+ if (requireKey && !apiKey) {
14007
+ throw new Error(`${command}: env var ${apiKeyEnv} is empty \u2014 pass --api-key-env or export it`);
14138
14008
  }
14139
- if (fixFile !== void 0) fixCommand = readFileSync3(fixFile, "utf8").trimEnd();
14009
+ const maxLifetime = flagReader(command, values).number("--max-lifetime");
14140
14010
  return {
14141
- stepsPath,
14142
- image,
14143
- at,
14144
- fixCommand,
14145
- cwd,
14146
- out,
14147
- caseId: values.get("--case"),
14148
- signature: values.get("--signature"),
14149
- stepTimeoutMs: optionalNumber("--step-timeout"),
14150
- prefixLimit: optionalNumber("--prefix-limit"),
14151
- driverLabel: values.get("--driver-label"),
14152
- baseUrl: values.get("--base-url") ?? process.env.SANDBOX_API_URL ?? "http://127.0.0.1:4097",
14153
- apiKeyEnv: values.get("--api-key-env") ?? "SANDBOX_API_KEY",
14154
- maxLifetimeSeconds: optionalNumber("--max-lifetime")
14011
+ apiKey: apiKey ?? "",
14012
+ baseUrl: values.get("--base-url") ?? process.env.SANDBOX_API_URL ?? DEFAULT_SANDBOX_BASE_URL,
14013
+ ...maxLifetime === void 0 ? {} : { maxLifetimeSeconds: maxLifetime }
14155
14014
  };
14156
14015
  }
14157
- function replayVerifyUsage() {
14158
- return `traces replay-verify \u2014 replay a trajectory prefix in a sandbox and execute the counterfactual at step k
14016
+ var progress = (message) => process.stderr.write(`${message}
14017
+ `);
14018
+ var REPLAY_VERIFY_USAGE = `traces replay-verify \u2014 replay a trajectory prefix in a sandbox and execute the counterfactual at step k
14159
14019
 
14160
14020
  Usage:
14161
14021
  traces replay-verify --steps STEPS.json --image IMG --at K --cwd DIR --out DIR \\
@@ -14173,1928 +14033,153 @@ Usage:
14173
14033
  line of the recorded observation containing "error")
14174
14034
  --prefix-limit execute at most N prefix steps (weakens fidelity; reported)
14175
14035
  --step-timeout per-exec timeout in ms (default 300000)
14176
- --base-url sandbox API url (default $SANDBOX_API_URL or http://127.0.0.1:4097)
14036
+ --base-url sandbox API url (default $SANDBOX_API_URL or ${DEFAULT_SANDBOX_BASE_URL})
14177
14037
  --api-key-env env var holding the sandbox API key (default SANDBOX_API_KEY)
14178
14038
 
14179
14039
  Requires a running sandbox orchestrator; only SWE-style trajectories that carry a
14180
14040
  docker image are replayable. See docs/replay-verify.md for setup and limits.
14181
14041
  `;
14182
- }
14183
14042
  async function cmdReplayVerify(argv) {
14184
- const parsed = parseReplayVerifyArgs(argv);
14043
+ const parsed = parseFlags2("replay-verify", argv);
14185
14044
  if (parsed === "help") {
14186
- process.stdout.write(replayVerifyUsage());
14045
+ process.stdout.write(REPLAY_VERIFY_USAGE);
14187
14046
  return;
14188
14047
  }
14189
- const apiKey = process.env[parsed.apiKeyEnv];
14190
- if (!apiKey) {
14191
- throw new Error(
14192
- `replay-verify: env var ${parsed.apiKeyEnv} is empty \u2014 pass --api-key-env or export it`
14193
- );
14048
+ const { values } = parsed;
14049
+ const flags = flagReader("replay-verify", values);
14050
+ const at = Number(flags.required("--at"));
14051
+ if (!Number.isInteger(at) || at < 1) {
14052
+ throw new Error(`replay-verify: --at must be a positive step_id, got ${values.get("--at")}`);
14053
+ }
14054
+ const fixFile = values.get("--fix-command-file");
14055
+ if (values.has("--fix-command") && fixFile !== void 0) {
14056
+ throw new Error("replay-verify: pass --fix-command or --fix-command-file, not both");
14194
14057
  }
14058
+ const fixCommand = fixFile === void 0 ? values.get("--fix-command") : readFileSync3(fixFile, "utf8").trimEnd();
14059
+ const image = flags.required("--image");
14060
+ const out = flags.required("--out");
14195
14061
  const verdict = await replayVerify({
14196
- stepsPath: parsed.stepsPath,
14197
- image: parsed.image,
14198
- at: parsed.at,
14199
- fixCommand: parsed.fixCommand,
14200
- cwd: parsed.cwd,
14201
- out: parsed.out,
14202
- caseId: parsed.caseId,
14203
- signature: parsed.signature,
14204
- stepTimeoutMs: parsed.stepTimeoutMs,
14205
- prefixLimit: parsed.prefixLimit,
14206
- driverLabel: parsed.driverLabel,
14207
- baseUrl: parsed.baseUrl,
14208
- apiKey,
14209
- maxLifetimeSeconds: parsed.maxLifetimeSeconds,
14210
- onProgress: (message) => process.stderr.write(`${message}
14211
- `)
14062
+ stepsPath: flags.required("--steps"),
14063
+ image,
14064
+ at,
14065
+ fixCommand,
14066
+ cwd: flags.required("--cwd"),
14067
+ out,
14068
+ caseId: values.get("--case"),
14069
+ signature: values.get("--signature"),
14070
+ stepTimeoutMs: flags.number("--step-timeout"),
14071
+ prefixLimit: flags.number("--prefix-limit"),
14072
+ driverLabel: values.get("--driver-label"),
14073
+ backend: sandboxReplayBackend({ ...sandboxOptions("replay-verify", values, true), image }),
14074
+ onProgress: progress
14212
14075
  });
14213
14076
  process.stdout.write(`${JSON.stringify(verdict, null, 2)}
14214
14077
  `);
14215
14078
  const verdictLine = verdict.armB ? `armA.failureSignatureMatch=${verdict.armA.failureSignatureMatch} armB.failureVanished=${verdict.armB.failureVanished}` : `armA.failureSignatureMatch=${verdict.armA.failureSignatureMatch}`;
14216
- process.stderr.write(
14217
- `replay-verify: ${verdictLine} (${verdict.prefixDivergences.length}/${verdict.prefixExecuted} prefix divergences) \u2192 ${parsed.out}
14218
- `
14079
+ progress(
14080
+ `replay-verify: ${verdictLine} (${verdict.prefixDivergences.length}/${verdict.prefixExecuted} prefix divergences) \u2192 ${out}`
14219
14081
  );
14220
14082
  }
14083
+ var REPLAY_BATCH_USAGE = `traces replay-verify-batch \u2014 measure replayability and fix-flip rates across gold-labeled corpora
14221
14084
 
14222
- // src/replay-corpus.ts
14223
- import { existsSync as existsSync2, readdirSync as readdirSync2, readFileSync as readFileSync4 } from "fs";
14224
- import { join as join25 } from "path";
14225
- function parseCorpusFlag(value) {
14226
- const eq = value.indexOf("=");
14227
- if (eq <= 0) throw new Error(`--corpus must be name=<labels>::<prepared>, got: ${value}`);
14228
- const name = value.slice(0, eq);
14229
- const rest = value.slice(eq + 1);
14230
- const sep4 = rest.indexOf("::");
14231
- if (sep4 <= 0 || sep4 === rest.length - 2) {
14232
- throw new Error(`--corpus must be name=<labels>::<prepared>, got: ${value}`);
14233
- }
14234
- return { name, labelsPath: rest.slice(0, sep4), preparedDir: rest.slice(sep4 + 2) };
14235
- }
14236
- var SUBMIT_ACTION_SIGNATURE = "COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT";
14237
- function isSubmitAction(action) {
14238
- return action.includes(SUBMIT_ACTION_SIGNATURE);
14239
- }
14240
- function readLabelEntries(labelsPath) {
14241
- const parsed = JSON.parse(readFileSync4(labelsPath, "utf8"));
14242
- if (!Array.isArray(parsed)) throw new Error(`${labelsPath} is not a JSON array of label entries`);
14243
- for (const entry of parsed) {
14244
- if (typeof entry.traj_id !== "string") {
14245
- throw new Error(`${labelsPath} has an entry without a string traj_id`);
14246
- }
14247
- }
14248
- return parsed;
14249
- }
14250
- function goldIncorrectSteps(entry) {
14251
- const ids = /* @__PURE__ */ new Set();
14252
- for (const stage of entry.incorrect_stages ?? []) {
14253
- for (const id of stage.incorrect_step_ids ?? []) ids.add(id);
14254
- }
14255
- return [...ids].sort((a, b) => a - b);
14256
- }
14257
- function findRawTrajFiles(sweRawDir) {
14258
- if (!existsSync2(sweRawDir)) return [];
14259
- const entries = readdirSync2(sweRawDir, { recursive: true, encoding: "utf8" });
14260
- return entries.filter((relative7) => relative7.endsWith(".traj.json")).map((relative7) => join25(sweRawDir, relative7)).sort();
14261
- }
14262
- function cwdFromPwdObservation(steps) {
14263
- for (const step of steps) {
14264
- if (step.action.trim() !== "pwd") continue;
14265
- const line = parseObservationOutput(step.observation).split("\n").map((l) => l.trim()).find((l) => l.startsWith("/"));
14266
- if (line) return line;
14085
+ Usage:
14086
+ traces replay-verify-batch \\
14087
+ --corpus NAME=<labels.json>::<prepared-dir> [--corpus ...] \\
14088
+ --out DIR [--fix none|generate|loop] [--fix-attempts 3] [--enumerate-only] \\
14089
+ [--fix-model glm-5.2] [--fix-base-url URL] [--fix-api-key-env ZAI_GLM_API_KEY] \\
14090
+ [--max-fix-cases 30] [--seed 17] [--step-timeout MS] [--prefix-limit N] \\
14091
+ [--case-filter SUBSTRING] [--case-limit N] \\
14092
+ [--base-url URL] [--api-key-env SANDBOX_API_KEY] [--max-lifetime SECONDS]
14093
+
14094
+ Replayable = the raw trajectory carries info.docker_config.base_image AND the
14095
+ labels mark at least one gold incorrect step that is a real mid-trajectory
14096
+ action. Excluded cases and image pull failures are report rows, never
14097
+ skipped silently. --enumerate-only prints the enumeration without touching a
14098
+ sandbox.
14099
+
14100
+ --fix generate runs ONE LLM call per arm-A-reproduced case (cap
14101
+ --max-fix-cases, seeded sample beyond it) and executes the corrected command
14102
+ as arm B in a fresh sandbox. --fix loop feeds each failed attempt's REAL
14103
+ executed output into the next prompt, up to --fix-attempts attempts.
14104
+
14105
+ Outputs batch-report.json, batch-report.md, cases.jsonl (incremental), and one
14106
+ directory per case with the full replay-verify artifacts.
14107
+ See docs/replay-verify.md for orchestrator setup and honest limits.
14108
+ `;
14109
+ async function cmdReplayVerifyBatch(argv) {
14110
+ const parsed = parseFlags2("replay-verify-batch", argv, ["--enumerate-only"]);
14111
+ if (parsed === "help") {
14112
+ process.stdout.write(REPLAY_BATCH_USAGE);
14113
+ return;
14267
14114
  }
14268
- return null;
14269
- }
14270
- function taskStatementOf(preparedDir, trajId, raw) {
14271
- const taskPath = join25(preparedDir, "normalized", trajId, "task.md");
14272
- if (existsSync2(taskPath)) {
14273
- const text2 = readFileSync4(taskPath, "utf8").trim();
14274
- if (text2.length > 0) return text2;
14115
+ const { values, corpora } = parsed;
14116
+ const flags = flagReader("replay-verify-batch", values);
14117
+ if (corpora.length === 0) {
14118
+ throw new Error("replay-verify-batch: at least one --corpus name=<labels>::<prepared> is required");
14275
14119
  }
14276
- const userMessage = raw.messages?.find((m) => m.role === "user");
14277
- if (typeof userMessage?.content === "string" && userMessage.content.trim().length > 0) {
14278
- return userMessage.content.trim();
14120
+ if (parsed.switches.has("--enumerate-only")) {
14121
+ const enumeration = enumerateReplayableCases(corpora);
14122
+ const replayable = enumeration.replayable.map((c) => ({
14123
+ corpus: c.corpus,
14124
+ trajId: c.trajId,
14125
+ image: c.image,
14126
+ cwd: c.cwd,
14127
+ cwdSource: c.cwdSource,
14128
+ k: c.k,
14129
+ submitGoldsSkipped: c.submitGoldsSkipped,
14130
+ recordedReturncodeAtK: c.recordedReturncodeAtK,
14131
+ goldIncorrectSteps: c.goldIncorrectSteps
14132
+ }));
14133
+ const report2 = { labelEntries: enumeration.labelEntryCount, replayable, excluded: enumeration.excluded };
14134
+ process.stdout.write(`${JSON.stringify(report2, null, 2)}
14135
+ `);
14136
+ return;
14279
14137
  }
14280
- return null;
14281
- }
14282
- function resolveCaseResources(corpus, trajId) {
14283
- const sweRawDir = join25(corpus.preparedDir, "extracted", trajId, "swe_raw");
14284
- const rawFiles = findRawTrajFiles(sweRawDir);
14285
- if (rawFiles.length === 0) {
14286
- return { resolved: false, reason: "no-swe-raw-trajectory", detail: sweRawDir };
14138
+ const out = flags.required("--out");
14139
+ const fix = values.get("--fix") ?? "none";
14140
+ if (fix !== "none" && fix !== "generate" && fix !== "loop") {
14141
+ throw new Error(`replay-verify-batch: --fix must be none, generate, or loop, got ${fix}`);
14287
14142
  }
14288
- if (rawFiles.length > 1) {
14289
- return { resolved: false, reason: "ambiguous-swe-raw-trajectory", detail: rawFiles.join(", ") };
14143
+ const fixAttempts = flags.number("--fix-attempts") ?? 3;
14144
+ if (!Number.isInteger(fixAttempts) || fixAttempts < 1) {
14145
+ throw new Error(`replay-verify-batch: --fix-attempts must be a positive integer, got ${fixAttempts}`);
14290
14146
  }
14291
- let raw;
14292
- try {
14293
- raw = JSON.parse(readFileSync4(rawFiles[0], "utf8"));
14294
- } catch (err) {
14295
- return {
14296
- resolved: false,
14297
- reason: "unreadable-raw-trajectory",
14298
- detail: `${rawFiles[0]}: ${err instanceof Error ? err.message : String(err)}`
14299
- };
14147
+ const fixModel = values.get("--fix-model") ?? "glm-5.2";
14148
+ let fixCaller;
14149
+ if (fix !== "none") {
14150
+ const fixApiKeyEnv = values.get("--fix-api-key-env") ?? "ZAI_GLM_API_KEY";
14151
+ const fixApiKey = process.env[fixApiKeyEnv];
14152
+ if (!fixApiKey) {
14153
+ throw new Error(`replay-verify-batch: env var ${fixApiKeyEnv} is empty \u2014 pass --fix-api-key-env or export it`);
14154
+ }
14155
+ fixCaller = zaiChatCaller({
14156
+ baseUrl: values.get("--fix-base-url") ?? "https://api.z.ai/api/coding/paas/v4",
14157
+ apiKey: fixApiKey,
14158
+ model: fixModel
14159
+ });
14300
14160
  }
14301
- const dockerConfig = raw.info?.docker_config;
14302
- const image = dockerConfig?.base_image;
14303
- if (typeof image !== "string" || image.length === 0) {
14304
- return { resolved: false, reason: "no-docker-image", detail: rawFiles[0] };
14305
- }
14306
- const stepsPath = join25(corpus.preparedDir, "normalized", trajId, "steps.json");
14307
- if (!existsSync2(stepsPath)) {
14308
- return { resolved: false, reason: "missing-steps-json", detail: stepsPath };
14309
- }
14310
- const steps = JSON.parse(readFileSync4(stepsPath, "utf8"));
14311
- const runConfigCwd = raw.info?.config?.environment?.cwd;
14312
- const dockerCwd = dockerConfig?.cwd;
14313
- let cwd;
14314
- let cwdSource;
14315
- if (typeof runConfigCwd === "string" && runConfigCwd.length > 0) {
14316
- cwd = runConfigCwd;
14317
- cwdSource = "run-config";
14318
- } else if (typeof dockerCwd === "string" && dockerCwd.length > 0) {
14319
- cwd = dockerCwd;
14320
- cwdSource = "docker-config";
14321
- } else {
14322
- const observed = cwdFromPwdObservation(steps);
14323
- if (!observed) return { resolved: false, reason: "cwd-underivable" };
14324
- cwd = observed;
14325
- cwdSource = "pwd-observation";
14326
- }
14327
- const recordedTimeout = raw.info?.config?.environment?.timeout;
14328
- return {
14329
- resolved: true,
14330
- resources: {
14331
- corpus: corpus.name,
14332
- trajId,
14333
- stepsPath,
14334
- steps,
14335
- taskStatement: taskStatementOf(corpus.preparedDir, trajId, raw),
14336
- image,
14337
- cwd,
14338
- cwdSource,
14339
- recordedStepTimeoutMs: typeof recordedTimeout === "number" && recordedTimeout > 0 ? recordedTimeout * 1e3 : null
14340
- }
14341
- };
14342
- }
14343
- function enumerateReplayableCases(corpora) {
14344
- const replayable = [];
14345
- const excluded = [];
14346
- let labelEntryCount = 0;
14347
- for (const corpus of corpora) {
14348
- for (const entry of readLabelEntries(corpus.labelsPath)) {
14349
- labelEntryCount += 1;
14350
- const trajId = entry.traj_id;
14351
- const resolution = resolveCaseResources(corpus, trajId);
14352
- if (!resolution.resolved) {
14353
- excluded.push({ corpus: corpus.name, trajId, reason: resolution.reason, detail: resolution.detail });
14354
- continue;
14355
- }
14356
- const gold = goldIncorrectSteps(entry);
14357
- if (gold.length === 0) {
14358
- excluded.push({ corpus: corpus.name, trajId, reason: "no-gold-incorrect-step" });
14359
- continue;
14360
- }
14361
- const resources = resolution.resources;
14362
- let submitGoldsSkipped = 0;
14363
- let target = null;
14364
- let missingGoldId = null;
14365
- for (const goldId of gold) {
14366
- const step = resources.steps.find((s) => s.step_id === goldId);
14367
- if (!step) {
14368
- missingGoldId = goldId;
14369
- break;
14370
- }
14371
- if (isSubmitAction(step.action)) {
14372
- submitGoldsSkipped += 1;
14373
- continue;
14374
- }
14375
- target = step;
14376
- break;
14377
- }
14378
- if (missingGoldId !== null) {
14379
- excluded.push({
14380
- corpus: corpus.name,
14381
- trajId,
14382
- reason: "gold-step-outside-steps",
14383
- detail: `k=${missingGoldId}, steps=${resources.steps.length}`
14384
- });
14385
- continue;
14386
- }
14387
- if (!target) {
14388
- excluded.push({
14389
- corpus: corpus.name,
14390
- trajId,
14391
- reason: "gold-only-submit-step",
14392
- detail: `${submitGoldsSkipped} gold step(s), all submit commands`
14393
- });
14394
- continue;
14395
- }
14396
- replayable.push({
14397
- ...resources,
14398
- goldIncorrectSteps: gold,
14399
- k: target.step_id,
14400
- submitGoldsSkipped,
14401
- recordedReturncodeAtK: parseRecordedReturncode(target.observation)
14402
- });
14403
- }
14404
- }
14405
- return { replayable, excluded, labelEntryCount };
14406
- }
14407
-
14408
- // src/replay-fix.ts
14409
- function zaiChatCaller(options) {
14410
- const fetchImpl = options.fetchImpl ?? fetch;
14411
- return {
14412
- async complete(system, user) {
14413
- const baseCap = options.maxTokens ?? 16384;
14414
- let lastError = "empty completion content";
14415
- for (const cap of [baseCap, baseCap * 2]) {
14416
- const outcome = await callOnce(cap);
14417
- if (outcome.succeeded) return outcome;
14418
- lastError = outcome.error;
14419
- if (!/finish_reason":"length/.test(outcome.error)) return outcome;
14420
- }
14421
- return { succeeded: false, error: lastError };
14422
- async function callOnce(maxTokens) {
14423
- const controller = new AbortController();
14424
- const timer = setTimeout(() => controller.abort(), options.timeoutMs ?? 18e4);
14425
- try {
14426
- const response = await fetchImpl(`${options.baseUrl.replace(/\/$/, "")}/chat/completions`, {
14427
- method: "POST",
14428
- headers: {
14429
- authorization: `Bearer ${options.apiKey}`,
14430
- "content-type": "application/json"
14431
- },
14432
- body: JSON.stringify({
14433
- model: options.model,
14434
- messages: [
14435
- { role: "system", content: system },
14436
- { role: "user", content: user }
14437
- ],
14438
- temperature: 0,
14439
- max_tokens: maxTokens
14440
- }),
14441
- signal: controller.signal
14442
- });
14443
- const bodyText = await response.text();
14444
- if (!response.ok) {
14445
- return { succeeded: false, error: `HTTP ${response.status}: ${bodyText.slice(0, 400)}` };
14446
- }
14447
- const body = JSON.parse(bodyText);
14448
- if (body.error) {
14449
- return { succeeded: false, error: String(body.error.message ?? "provider error") };
14450
- }
14451
- const content = body.choices?.[0]?.message?.content;
14452
- if (typeof content !== "string" || content.length === 0) {
14453
- return { succeeded: false, error: `empty completion content: ${bodyText.slice(0, 400)}` };
14454
- }
14455
- const promptTokens = body.usage?.prompt_tokens;
14456
- const completionTokens = body.usage?.completion_tokens;
14457
- const usage = typeof promptTokens === "number" && typeof completionTokens === "number" ? { promptTokens, completionTokens } : null;
14458
- return { succeeded: true, value: { content, usage } };
14459
- } catch (err) {
14460
- return { succeeded: false, error: err instanceof Error ? err.message : String(err) };
14461
- } finally {
14462
- clearTimeout(timer);
14463
- }
14464
- }
14465
- }
14466
- };
14467
- }
14468
- function clipText(text2, limit) {
14469
- if (text2.length <= limit) return text2;
14470
- const half = Math.floor(limit / 2);
14471
- return `${text2.slice(0, half)}
14472
- \u2026 [${text2.length - limit} chars elided] \u2026
14473
- ${text2.slice(-half)}`;
14474
- }
14475
- function renderStep(step, marker) {
14476
- const rc = parseRecordedReturncode(step.observation);
14477
- const output = clipText(parseObservationOutput(step.observation).trim(), 1600);
14478
- return [
14479
- `### step ${step.step_id}${marker}`,
14480
- "```sh",
14481
- clipText(step.action, 2e3),
14482
- "```",
14483
- `returncode: ${rc ?? "none recorded"}`,
14484
- output.length > 0 ? `output:
14485
- \`\`\`
14486
- ${output}
14487
- \`\`\`` : "output: (empty)"
14488
- ].join("\n");
14489
- }
14490
- function promptBody(input) {
14491
- const radius = input.contextRadius ?? 3;
14492
- const target = input.steps.find((s) => s.step_id === input.k);
14493
- if (!target) throw new Error(`replay-fix: no step with step_id ${input.k}`);
14494
- const context = input.steps.filter(
14495
- (s) => s.step_id !== input.k && Math.abs(s.step_id - input.k) <= radius
14496
- );
14497
- return [
14498
- "## Task the agent was solving",
14499
- input.taskStatement ? clipText(input.taskStatement, 4e3) : "(no task statement recorded)",
14500
- "",
14501
- "## Surrounding steps",
14502
- ...context.map((s) => renderStep(s, "")),
14503
- "",
14504
- "## Failing step to correct",
14505
- renderStep(target, " (INCORRECT \u2014 correct this one)")
14506
- ];
14507
- }
14508
- function buildFixPrompt(input) {
14509
- const system = [
14510
- "You repair one failed shell command from a recorded coding-agent trajectory.",
14511
- "The trajectory replays inside the original docker image; every command runs as a fresh /bin/sh subshell from a fixed working directory.",
14512
- "You are given the failing step, its recorded output, surrounding steps, and the task statement.",
14513
- "Reply with exactly ONE corrected shell command (compound commands with && or pipes are fine) inside a single ```sh fenced block.",
14514
- "The corrected command must accomplish the failing step's intent and exit 0. No prose outside the fenced block."
14515
- ].join("\n");
14516
- const user = [
14517
- ...promptBody(input),
14518
- "",
14519
- "Output the single corrected replacement for the failing step now."
14520
- ].join("\n");
14521
- return { system, user };
14522
- }
14523
- function renderFailedAttempt(prior) {
14524
- if (prior.command === null) {
14525
- return [
14526
- `### attempt ${prior.attempt}`,
14527
- `model call failed before producing a command: ${prior.llmError ?? "unknown error"}`
14528
- ].join("\n");
14529
- }
14530
- const stdout = (prior.stdoutTail ?? "").trim();
14531
- const stderr = (prior.stderrTail ?? "").trim();
14532
- return [
14533
- `### attempt ${prior.attempt}`,
14534
- "```sh",
14535
- clipText(prior.command, 2e3),
14536
- "```",
14537
- `exit code: ${prior.exitCode ?? "not executed"}`,
14538
- stdout.length > 0 ? `stdout:
14539
- \`\`\`
14540
- ${stdout}
14541
- \`\`\`` : "stdout: (empty)",
14542
- stderr.length > 0 ? `stderr:
14543
- \`\`\`
14544
- ${stderr}
14545
- \`\`\`` : "stderr: (empty)"
14546
- ].join("\n");
14547
- }
14548
- function buildRetryFixPrompt(input, priorAttempts, maxScriptCommands = 5) {
14549
- if (priorAttempts.length === 0) {
14550
- throw new Error("replay-fix: buildRetryFixPrompt requires at least one prior attempt");
14551
- }
14552
- const system = [
14553
- "You repair one failed shell command from a recorded coding-agent trajectory.",
14554
- "The trajectory replays inside the original docker image; every command runs as a fresh /bin/sh subshell from a fixed working directory.",
14555
- "Earlier corrected commands were executed for real and failed; their actual output is included below.",
14556
- `Reply with a corrected fix inside a single \`\`\`sh fenced block: either one command, or a short script of at most ${maxScriptCommands} commands (one per line).`,
14557
- "The whole block executes as ONE /bin/sh unit from the fixed working directory and must exit 0.",
14558
- "Do not repeat a command that already failed. Keep reasoning brief. No prose outside the fenced block."
14559
- ].join("\n");
14560
- const user = [
14561
- ...promptBody(input),
14562
- "",
14563
- "## Previous fix attempts (executed for real \u2014 all failed)",
14564
- ...priorAttempts.map((prior) => renderFailedAttempt(prior)),
14565
- "",
14566
- `Output a corrected fix now \u2014 one \`\`\`sh block, at most ${maxScriptCommands} commands.`
14567
- ].join("\n");
14568
- return { system, user };
14569
- }
14570
- function countScriptCommands(script) {
14571
- return script.split("\n").filter((line) => line.trim().length > 0 && !line.trim().startsWith("#")).length;
14572
- }
14573
- function extractFixCommand(content) {
14574
- const blocks = [...content.matchAll(/```(?:sh|bash|shell)?\n([\s\S]*?)```/g)];
14575
- const candidate = blocks.length > 0 ? blocks[blocks.length - 1][1] : content;
14576
- const command = candidate.trim();
14577
- if (command.length === 0 || command.includes("```")) return null;
14578
- return command;
14579
- }
14580
- async function generateFixCommand(caller, input) {
14581
- const { system, user } = buildFixPrompt(input);
14582
- const outcome = await caller.complete(system, user);
14583
- if (!outcome.succeeded) return outcome;
14584
- const command = extractFixCommand(outcome.value.content);
14585
- if (command === null) {
14586
- return {
14587
- succeeded: false,
14588
- error: `completion carried no usable command: ${clipText(outcome.value.content, 300)}`
14589
- };
14590
- }
14591
- return { succeeded: true, value: { command, usage: outcome.value.usage } };
14592
- }
14593
-
14594
- // src/replay-fix-loop.ts
14595
- function toFailedAttempt(record2) {
14596
- return {
14597
- attempt: record2.attempt,
14598
- command: record2.command,
14599
- exitCode: record2.exitCode,
14600
- stdoutTail: record2.stdoutTail,
14601
- stderrTail: record2.stderrTail,
14602
- llmError: record2.llmError ?? record2.armBError
14603
- };
14604
- }
14605
- async function runFixLoop(caller, input, executor, options) {
14606
- if (!Number.isInteger(options.maxAttempts) || options.maxAttempts < 1) {
14607
- throw new Error(
14608
- `replay-fix-loop: maxAttempts must be a positive integer, got ${options.maxAttempts}`
14609
- );
14610
- }
14611
- const scriptCap = options.maxScriptCommands ?? 5;
14612
- const tailChars = options.outputTailChars ?? 1600;
14613
- const attempts = [];
14614
- let llmCalls = 0;
14615
- let llmFailures = 0;
14616
- let promptTokens = 0;
14617
- let completionTokens = 0;
14618
- const unexecuted = (attempt, llmError, usage, command, started) => ({
14619
- attempt,
14620
- command,
14621
- llmError,
14622
- usage,
14623
- executed: false,
14624
- exitCode: null,
14625
- prefixExecuted: null,
14626
- prefixDivergences: null,
14627
- failureVanished: null,
14628
- stdoutTail: null,
14629
- stderrTail: null,
14630
- armBError: null,
14631
- wallMs: Date.now() - started
14632
- });
14633
- for (let attempt = 1; attempt <= options.maxAttempts; attempt++) {
14634
- const started = Date.now();
14635
- const prompt = attempt === 1 ? buildFixPrompt(input) : buildRetryFixPrompt(input, attempts.map(toFailedAttempt), scriptCap);
14636
- llmCalls += 1;
14637
- const outcome = await caller.complete(prompt.system, prompt.user);
14638
- if (!outcome.succeeded) {
14639
- llmFailures += 1;
14640
- attempts.push(unexecuted(attempt, outcome.error, null, null, started));
14641
- options.onProgress?.(
14642
- `fix-loop attempt ${attempt}: LLM failed \u2014 ${outcome.error.slice(0, 160)}`
14643
- );
14644
- continue;
14645
- }
14646
- const usage = outcome.value.usage;
14647
- promptTokens += usage?.promptTokens ?? 0;
14648
- completionTokens += usage?.completionTokens ?? 0;
14649
- const command = extractFixCommand(outcome.value.content);
14650
- if (command === null) {
14651
- llmFailures += 1;
14652
- attempts.push(
14653
- unexecuted(
14654
- attempt,
14655
- `completion carried no usable command: ${clipText(outcome.value.content, 300)}`,
14656
- usage,
14657
- null,
14658
- started
14659
- )
14660
- );
14661
- continue;
14662
- }
14663
- if (attempt > 1) {
14664
- const commandLines = countScriptCommands(command);
14665
- if (commandLines > scriptCap) {
14666
- llmFailures += 1;
14667
- attempts.push(
14668
- unexecuted(
14669
- attempt,
14670
- `script exceeds ${scriptCap} command lines (${commandLines})`,
14671
- usage,
14672
- null,
14673
- started
14674
- )
14675
- );
14676
- options.onProgress?.(
14677
- `fix-loop attempt ${attempt}: rejected script with ${commandLines} command lines`
14678
- );
14679
- continue;
14680
- }
14681
- }
14682
- let execution;
14683
- try {
14684
- execution = await executor(command, attempt);
14685
- } catch (err) {
14686
- const message = err instanceof Error ? err.message : String(err);
14687
- attempts.push({
14688
- ...unexecuted(attempt, "", usage, command, started),
14689
- llmError: null,
14690
- armBError: message.slice(0, 500)
14691
- });
14692
- options.onProgress?.(
14693
- `fix-loop attempt ${attempt}: sandbox error \u2014 ${message.slice(0, 160)}`
14694
- );
14695
- return {
14696
- flipped: false,
14697
- flippedAtAttempt: null,
14698
- attempts,
14699
- aborted: true,
14700
- llmCalls,
14701
- llmFailures,
14702
- promptTokens,
14703
- completionTokens
14704
- };
14705
- }
14706
- attempts.push({
14707
- attempt,
14708
- command,
14709
- llmError: null,
14710
- usage,
14711
- executed: true,
14712
- exitCode: execution.exitCode,
14713
- prefixExecuted: execution.prefixExecuted,
14714
- prefixDivergences: execution.prefixDivergences,
14715
- failureVanished: execution.failureVanished,
14716
- stdoutTail: clipText(execution.stdout, tailChars),
14717
- stderrTail: clipText(execution.stderr, tailChars),
14718
- armBError: null,
14719
- wallMs: Date.now() - started
14720
- });
14721
- options.onProgress?.(
14722
- `fix-loop attempt ${attempt}: exit=${execution.exitCode} failureVanished=${execution.failureVanished}`
14723
- );
14724
- if (execution.failureVanished) {
14725
- return {
14726
- flipped: true,
14727
- flippedAtAttempt: attempt,
14728
- attempts,
14729
- aborted: false,
14730
- llmCalls,
14731
- llmFailures,
14732
- promptTokens,
14733
- completionTokens
14734
- };
14735
- }
14736
- }
14737
- return {
14738
- flipped: false,
14739
- flippedAtAttempt: null,
14740
- attempts,
14741
- aborted: false,
14742
- llmCalls,
14743
- llmFailures,
14744
- promptTokens,
14745
- completionTokens
14746
- };
14747
- }
14748
-
14749
- // src/replay-batch.ts
14750
- import { execFile } from "child_process";
14751
- import { createHash as createHash12 } from "crypto";
14752
- import { appendFileSync, mkdirSync as mkdirSync2, mkdtempSync, writeFileSync as writeFileSync2 } from "fs";
14753
- import { tmpdir as tmpdir5 } from "os";
14754
- import { join as join26 } from "path";
14755
- import { promisify } from "util";
14756
- import {
14757
- InMemoryTraceStore as InMemoryTraceStore3,
14758
- runCounterfactual as runCounterfactual2
14759
- } from "@tangle-network/agent-eval";
14760
- var execFileAsync = promisify(execFile);
14761
- function derivedImageTag(image, cwd) {
14762
- const digest = createHash12("sha256").update(`${image}
14763
- ${cwd}`).digest("hex").slice(0, 12);
14764
- return `ctb-replay:${digest}-uid1000`;
14765
- }
14766
- function dockerImagePreparer(options = {}) {
14767
- const pullTimeoutMs = options.pullTimeoutMs ?? 12e5;
14768
- const buildTimeoutMs = options.buildTimeoutMs ?? 9e5;
14769
- const imageExists = async (tag) => {
14770
- try {
14771
- await execFileAsync("docker", ["image", "inspect", tag], { maxBuffer: 8 * 1024 * 1024 });
14772
- return true;
14773
- } catch {
14774
- return false;
14775
- }
14776
- };
14777
- const errorTail = (err) => {
14778
- const raw = err && typeof err === "object" && "stderr" in err && typeof err.stderr === "string" && err.stderr.length > 0 ? err.stderr : err instanceof Error ? err.message : String(err);
14779
- return raw.trim().split("\n").slice(-3).join(" | ").slice(0, 400);
14780
- };
14781
- return {
14782
- async ensure(image, cwd) {
14783
- if (cwd === "/") {
14784
- if (await imageExists(image)) {
14785
- return { succeeded: true, value: { derivedImage: image, pulled: false, built: false } };
14786
- }
14787
- try {
14788
- await execFileAsync("docker", ["pull", image], {
14789
- timeout: pullTimeoutMs,
14790
- maxBuffer: 32 * 1024 * 1024
14791
- });
14792
- } catch (err) {
14793
- return { succeeded: false, error: `pull ${image}: ${errorTail(err)}` };
14794
- }
14795
- return { succeeded: true, value: { derivedImage: image, pulled: true, built: false } };
14796
- }
14797
- const derived = derivedImageTag(image, cwd);
14798
- if (await imageExists(derived)) {
14799
- return { succeeded: true, value: { derivedImage: derived, pulled: false, built: false } };
14800
- }
14801
- let pulled = false;
14802
- if (!await imageExists(image)) {
14803
- try {
14804
- await execFileAsync("docker", ["pull", image], {
14805
- timeout: pullTimeoutMs,
14806
- maxBuffer: 32 * 1024 * 1024
14807
- });
14808
- pulled = true;
14809
- } catch (err) {
14810
- return { succeeded: false, error: `pull ${image}: ${errorTail(err)}` };
14811
- }
14812
- }
14813
- const contextDir = mkdtempSync(join26(tmpdir5(), "ctb-replay-image-"));
14814
- const quotedCwd = `'${cwd.replaceAll("'", `'\\''`)}'`;
14815
- writeFileSync2(
14816
- join26(contextDir, "Dockerfile"),
14817
- `FROM ${image}
14818
- RUN chown -R 1000:1000 ${quotedCwd}
14819
- `
14820
- );
14821
- try {
14822
- await execFileAsync("docker", ["build", "-t", derived, contextDir], {
14823
- timeout: buildTimeoutMs,
14824
- maxBuffer: 32 * 1024 * 1024
14825
- });
14826
- } catch (err) {
14827
- return { succeeded: false, error: `build ${derived} from ${image}: ${errorTail(err)}` };
14828
- }
14829
- return { succeeded: true, value: { derivedImage: derived, pulled, built: true } };
14830
- }
14831
- };
14832
- }
14833
- function mulberry32(seed) {
14834
- let state = seed >>> 0;
14835
- return () => {
14836
- state = state + 1831565813 >>> 0;
14837
- let t = state;
14838
- t = Math.imul(t ^ t >>> 15, t | 1);
14839
- t ^= t + Math.imul(t ^ t >>> 7, t | 61);
14840
- return ((t ^ t >>> 14) >>> 0) / 4294967296;
14841
- };
14842
- }
14843
- function seededSample(items, size, seed) {
14844
- const pool = [...items];
14845
- const random = mulberry32(seed);
14846
- for (let i = pool.length - 1; i > 0; i--) {
14847
- const j = Math.floor(random() * (i + 1));
14848
- [pool[i], pool[j]] = [pool[j], pool[i]];
14849
- }
14850
- return new Set(pool.slice(0, size));
14851
- }
14852
- function caseOutDirName(row) {
14853
- return `${row.corpus}--${row.trajId}`.replaceAll(/[^A-Za-z0-9._-]/g, "_").slice(0, 180);
14854
- }
14855
- async function executeArmB(replayCase, fixCommand, backend, signature, stepTimeoutMs, prefixLimit, onProgress) {
14856
- const store = new InMemoryTraceStore3();
14857
- const { runId } = await ingestCodeTraceBenchSteps(store, replayCase.steps, replayCase.trajId);
14858
- const runner = new SandboxCounterfactualRunner(backend, {
14859
- cwd: replayCase.cwd,
14860
- stepTimeoutMs,
14861
- prefixLimit,
14862
- onProgress
14863
- });
14864
- await runCounterfactual2(
14865
- store,
14866
- runId,
14867
- {
14868
- kind: "custom",
14869
- at: replayCase.k - 1,
14870
- describe: "arm-B corrected step",
14871
- apply: (step) => ({
14872
- ...step,
14873
- span: { ...step.span, args: { command: fixCommand } }
14874
- })
14875
- },
14876
- runner
14877
- );
14878
- const exec = runner.lastArm;
14879
- if (!exec) throw new Error("arm B finished without executing the corrected step");
14880
- const output = `${exec.stdout}
14881
- ${exec.stderr}`;
14882
- return {
14883
- exitCode: exec.exitCode,
14884
- prefixExecuted: runner.lastPrefix?.prefixExecuted ?? 0,
14885
- prefixDivergences: runner.lastPrefix?.prefixDivergences.length ?? 0,
14886
- failureVanished: exec.exitCode === 0 && (signature ? !output.includes(signature) : true),
14887
- stdout: exec.stdout,
14888
- stderr: exec.stderr
14889
- };
14890
- }
14891
- function rate(numerator, denominator) {
14892
- return denominator > 0 ? numerator / denominator : null;
14893
- }
14894
- async function runReplayBatch(options) {
14895
- const onProgress = options.onProgress ?? (() => {
14896
- });
14897
- const enumeration = enumerateReplayableCases(options.corpora);
14898
- let selected = enumeration.replayable;
14899
- if (options.caseFilter) {
14900
- selected = selected.filter((c) => c.trajId.includes(options.caseFilter));
14901
- }
14902
- if (options.caseLimit !== void 0) selected = selected.slice(0, options.caseLimit);
14903
- onProgress(
14904
- `enumerated ${enumeration.labelEntryCount} label entries \u2192 ${enumeration.replayable.length} replayable, ${enumeration.excluded.length} excluded; executing ${selected.length}`
14905
- );
14906
- const preparer = options.preparer ?? dockerImagePreparer();
14907
- const backendFactory = options.backendFactory ?? ((derivedImage) => sandboxReplayBackend({
14908
- image: derivedImage,
14909
- apiKey: requireOption2(options.apiKey, "apiKey"),
14910
- baseUrl: requireOption2(options.baseUrl, "baseUrl"),
14911
- maxLifetimeSeconds: options.maxLifetimeSeconds ?? 3600
14912
- }));
14913
- mkdirSync2(options.out, { recursive: true });
14914
- const progressPath = join26(options.out, "cases.jsonl");
14915
- const rows = [];
14916
- const pullFailures = [];
14917
- const verdictByTraj = /* @__PURE__ */ new Map();
14918
- for (const [index, replayCase] of selected.entries()) {
14919
- const caseStart = Date.now();
14920
- const label = `[${index + 1}/${selected.length}] ${replayCase.corpus}/${replayCase.trajId}`;
14921
- onProgress(`${label}: preparing image ${replayCase.image}`);
14922
- const preparation = await preparer.ensure(replayCase.image, replayCase.cwd);
14923
- const base = {
14924
- corpus: replayCase.corpus,
14925
- trajId: replayCase.trajId,
14926
- image: replayCase.image,
14927
- cwd: replayCase.cwd,
14928
- cwdSource: replayCase.cwdSource,
14929
- k: replayCase.k,
14930
- stepCount: replayCase.steps.length,
14931
- goldIncorrectSteps: replayCase.goldIncorrectSteps,
14932
- submitGoldsSkipped: replayCase.submitGoldsSkipped,
14933
- recordedReturncodeAtK: replayCase.recordedReturncodeAtK
14934
- };
14935
- if (!preparation.succeeded) {
14936
- pullFailures.push({
14937
- corpus: replayCase.corpus,
14938
- trajId: replayCase.trajId,
14939
- image: replayCase.image,
14940
- error: preparation.error
14941
- });
14942
- const row = {
14943
- ...base,
14944
- derivedImage: null,
14945
- signature: null,
14946
- status: "image-unavailable",
14947
- error: preparation.error,
14948
- imagePulled: false,
14949
- imageBuilt: false,
14950
- prefixExecuted: null,
14951
- prefixDivergences: null,
14952
- prefixDivergencePct: null,
14953
- armAExit: null,
14954
- armAReturncodeMatch: false,
14955
- armASignatureMatch: false,
14956
- replayed: false,
14957
- fix: null,
14958
- wallMs: Date.now() - caseStart
14959
- };
14960
- rows.push(row);
14961
- appendFileSync(progressPath, `${JSON.stringify(row)}
14962
- `);
14963
- onProgress(`${label}: image unavailable \u2014 ${preparation.error}`);
14964
- continue;
14965
- }
14966
- const { derivedImage, pulled, built } = preparation.value;
14967
- const stepTimeoutMs = options.stepTimeoutMs ?? replayCase.recordedStepTimeoutMs ?? 12e4;
14968
- const caseOut = join26(options.out, caseOutDirName(replayCase));
14969
- onProgress(`${label}: arm A on ${derivedImage} (k=${replayCase.k}, timeout ${stepTimeoutMs}ms)`);
14970
- try {
14971
- const verdict = await replayVerify({
14972
- stepsPath: replayCase.stepsPath,
14973
- image: derivedImage,
14974
- at: replayCase.k,
14975
- cwd: replayCase.cwd,
14976
- out: caseOut,
14977
- caseId: replayCase.trajId,
14978
- stepTimeoutMs,
14979
- prefixLimit: options.prefixLimit,
14980
- backend: backendFactory(derivedImage),
14981
- onProgress: (message) => onProgress(`${label}: ${message}`)
14982
- });
14983
- verdictByTraj.set(replayCase.trajId, verdict);
14984
- const divergencePct = verdict.prefixExecuted > 0 ? verdict.prefixDivergences.length / verdict.prefixExecuted * 100 : 0;
14985
- const returncodeMatch = verdict.recordedReturncode !== null && verdict.armA.exitCode === verdict.recordedReturncode;
14986
- const row = {
14987
- ...base,
14988
- derivedImage,
14989
- signature: verdict.signature,
14990
- status: "ok",
14991
- error: null,
14992
- imagePulled: pulled,
14993
- imageBuilt: built,
14994
- prefixExecuted: verdict.prefixExecuted,
14995
- prefixDivergences: verdict.prefixDivergences.length,
14996
- prefixDivergencePct: Number(divergencePct.toFixed(1)),
14997
- armAExit: verdict.armA.exitCode,
14998
- armAReturncodeMatch: returncodeMatch,
14999
- armASignatureMatch: verdict.armA.failureSignatureMatch,
15000
- replayed: divergencePct <= 10 && returncodeMatch,
15001
- fix: null,
15002
- wallMs: Date.now() - caseStart
15003
- };
15004
- rows.push(row);
15005
- appendFileSync(progressPath, `${JSON.stringify(row)}
15006
- `);
15007
- onProgress(
15008
- `${label}: armA exit=${verdict.armA.exitCode} rcMatch=${returncodeMatch} divergences=${verdict.prefixDivergences.length}/${verdict.prefixExecuted}`
15009
- );
15010
- } catch (err) {
15011
- const message = err instanceof Error ? err.message : String(err);
15012
- const row = {
15013
- ...base,
15014
- derivedImage,
15015
- signature: null,
15016
- status: "replay-error",
15017
- error: message.slice(0, 500),
15018
- imagePulled: pulled,
15019
- imageBuilt: built,
15020
- prefixExecuted: null,
15021
- prefixDivergences: null,
15022
- prefixDivergencePct: null,
15023
- armAExit: null,
15024
- armAReturncodeMatch: false,
15025
- armASignatureMatch: false,
15026
- replayed: false,
15027
- fix: null,
15028
- wallMs: Date.now() - caseStart
15029
- };
15030
- rows.push(row);
15031
- appendFileSync(progressPath, `${JSON.stringify(row)}
15032
- `);
15033
- onProgress(`${label}: replay error \u2014 ${message.slice(0, 200)}`);
15034
- }
15035
- }
15036
- let llm = null;
15037
- if (options.fix !== "none") {
15038
- const caller = options.fixCaller;
15039
- if (!caller) throw new Error(`replay-verify-batch: fix=${options.fix} requires a fixCaller`);
15040
- const fixAttempts = options.fixAttempts ?? 3;
15041
- const maxFixCases = options.maxFixCases ?? 30;
15042
- const seed = options.seed ?? 17;
15043
- const eligible = rows.filter((r) => r.status === "ok" && r.replayed);
15044
- const sampled = eligible.length > maxFixCases ? seededSample(eligible, maxFixCases, seed) : new Set(eligible);
15045
- if (eligible.length > maxFixCases) {
15046
- onProgress(
15047
- `fix phase: ${eligible.length} eligible > cap ${maxFixCases}; seeded sample (seed=${seed})`
15048
- );
15049
- }
15050
- let calls = 0;
15051
- let failures = 0;
15052
- let promptTokens = 0;
15053
- let completionTokens = 0;
15054
- for (const row of eligible) {
15055
- const index = rows.indexOf(row);
15056
- if (!sampled.has(row)) {
15057
- rows[index] = {
15058
- ...row,
15059
- fix: {
15060
- attempted: false,
15061
- sampledOut: true,
15062
- command: null,
15063
- llmError: null,
15064
- usage: null,
15065
- armBExit: null,
15066
- armBPrefixExecuted: null,
15067
- armBPrefixDivergences: null,
15068
- failureVanished: null,
15069
- armBError: null,
15070
- attempts: null,
15071
- flippedAtAttempt: null
15072
- }
15073
- };
15074
- continue;
15075
- }
15076
- const replayCase = selected.find((c) => c.trajId === row.trajId && c.corpus === row.corpus);
15077
- const label = `fix ${row.corpus}/${row.trajId}`;
15078
- const signature = verdictByTraj.get(row.trajId)?.signature ?? null;
15079
- const stepTimeoutMs = options.stepTimeoutMs ?? replayCase.recordedStepTimeoutMs ?? 12e4;
15080
- const promptInput = {
15081
- taskStatement: replayCase.taskStatement,
15082
- steps: replayCase.steps,
15083
- k: replayCase.k
15084
- };
15085
- const caseOut = join26(options.out, caseOutDirName(row));
15086
- if (options.fix === "loop") {
15087
- onProgress(`${label}: fix loop (budget ${fixAttempts} attempts)`);
15088
- const result = await runFixLoop(
15089
- caller,
15090
- promptInput,
15091
- async (command, attempt) => {
15092
- onProgress(
15093
- `${label}: attempt ${attempt} arm B \u2014 ${command.split("\n")[0].slice(0, 120)}`
15094
- );
15095
- const armB = await executeArmB(
15096
- replayCase,
15097
- command,
15098
- backendFactory(row.derivedImage),
15099
- signature,
15100
- stepTimeoutMs,
15101
- options.prefixLimit,
15102
- (message) => onProgress(`${label}: attempt ${attempt}: ${message}`)
15103
- );
15104
- mkdirSync2(caseOut, { recursive: true });
15105
- writeFileSync2(
15106
- join26(caseOut, `armB-attempt${attempt}-result.json`),
15107
- `${JSON.stringify({ command, ...armB }, null, 2)}
15108
- `
15109
- );
15110
- writeFileSync2(
15111
- join26(caseOut, "armB-result.json"),
15112
- `${JSON.stringify({ command, attempt, ...armB }, null, 2)}
15113
- `
15114
- );
15115
- return armB;
15116
- },
15117
- { maxAttempts: fixAttempts, onProgress: (message) => onProgress(`${label}: ${message}`) }
15118
- );
15119
- calls += result.llmCalls;
15120
- failures += result.llmFailures;
15121
- promptTokens += result.promptTokens;
15122
- completionTokens += result.completionTokens;
15123
- const usage = result.promptTokens + result.completionTokens > 0 ? { promptTokens: result.promptTokens, completionTokens: result.completionTokens } : null;
15124
- const flippedRecord = result.flippedAtAttempt !== null ? result.attempts.find((a) => a.attempt === result.flippedAtAttempt) : null;
15125
- const summary = flippedRecord ?? [...result.attempts].reverse().find((a) => a.executed) ?? null;
15126
- const lastRecord = result.attempts.at(-1) ?? null;
15127
- rows[index] = {
15128
- ...row,
15129
- fix: summary ? {
15130
- attempted: true,
15131
- sampledOut: false,
15132
- command: summary.command,
15133
- llmError: null,
15134
- usage,
15135
- armBExit: summary.exitCode,
15136
- armBPrefixExecuted: summary.prefixExecuted,
15137
- armBPrefixDivergences: summary.prefixDivergences,
15138
- failureVanished: summary.failureVanished,
15139
- armBError: null,
15140
- attempts: result.attempts,
15141
- flippedAtAttempt: result.flippedAtAttempt
15142
- } : result.aborted ? {
15143
- attempted: true,
15144
- sampledOut: false,
15145
- command: lastRecord?.command ?? null,
15146
- llmError: null,
15147
- usage,
15148
- armBExit: null,
15149
- armBPrefixExecuted: null,
15150
- armBPrefixDivergences: null,
15151
- failureVanished: null,
15152
- armBError: lastRecord?.armBError ?? "sandbox error",
15153
- attempts: result.attempts,
15154
- flippedAtAttempt: null
15155
- } : {
15156
- attempted: true,
15157
- sampledOut: false,
15158
- command: null,
15159
- llmError: lastRecord?.llmError ?? "no attempt produced a runnable fix",
15160
- usage,
15161
- armBExit: null,
15162
- armBPrefixExecuted: null,
15163
- armBPrefixDivergences: null,
15164
- failureVanished: null,
15165
- armBError: null,
15166
- attempts: result.attempts,
15167
- flippedAtAttempt: null
15168
- }
15169
- };
15170
- onProgress(
15171
- `${label}: loop done \u2014 flipped=${result.flipped}` + (result.flippedAtAttempt !== null ? ` at attempt ${result.flippedAtAttempt}` : "") + ` (${result.llmCalls} calls, ${result.attempts.filter((a) => a.executed).length} arms)`
15172
- );
15173
- continue;
15174
- }
15175
- onProgress(`${label}: generating corrected command`);
15176
- calls += 1;
15177
- const generated = await generateFixCommand(caller, promptInput);
15178
- if (!generated.succeeded) {
15179
- failures += 1;
15180
- rows[index] = {
15181
- ...row,
15182
- fix: {
15183
- attempted: true,
15184
- sampledOut: false,
15185
- command: null,
15186
- llmError: generated.error,
15187
- usage: null,
15188
- armBExit: null,
15189
- armBPrefixExecuted: null,
15190
- armBPrefixDivergences: null,
15191
- failureVanished: null,
15192
- armBError: null,
15193
- attempts: null,
15194
- flippedAtAttempt: null
15195
- }
15196
- };
15197
- onProgress(`${label}: LLM failed \u2014 ${generated.error.slice(0, 200)}`);
15198
- continue;
15199
- }
15200
- promptTokens += generated.value.usage?.promptTokens ?? 0;
15201
- completionTokens += generated.value.usage?.completionTokens ?? 0;
15202
- onProgress(`${label}: arm B \u2014 ${generated.value.command.split("\n")[0].slice(0, 120)}`);
15203
- try {
15204
- const armB = await executeArmB(
15205
- replayCase,
15206
- generated.value.command,
15207
- backendFactory(row.derivedImage),
15208
- signature,
15209
- stepTimeoutMs,
15210
- options.prefixLimit,
15211
- (message) => onProgress(`${label}: ${message}`)
15212
- );
15213
- rows[index] = {
15214
- ...row,
15215
- fix: {
15216
- attempted: true,
15217
- sampledOut: false,
15218
- command: generated.value.command,
15219
- llmError: null,
15220
- usage: generated.value.usage,
15221
- armBExit: armB.exitCode,
15222
- armBPrefixExecuted: armB.prefixExecuted,
15223
- armBPrefixDivergences: armB.prefixDivergences,
15224
- failureVanished: armB.failureVanished,
15225
- armBError: null,
15226
- attempts: null,
15227
- flippedAtAttempt: armB.failureVanished ? 1 : null
15228
- }
15229
- };
15230
- mkdirSync2(caseOut, { recursive: true });
15231
- writeFileSync2(
15232
- join26(caseOut, "armB-result.json"),
15233
- `${JSON.stringify({ command: generated.value.command, ...armB }, null, 2)}
15234
- `
15235
- );
15236
- onProgress(`${label}: armB exit=${armB.exitCode} failureVanished=${armB.failureVanished}`);
15237
- } catch (err) {
15238
- const message = err instanceof Error ? err.message : String(err);
15239
- rows[index] = {
15240
- ...row,
15241
- fix: {
15242
- attempted: true,
15243
- sampledOut: false,
15244
- command: generated.value.command,
15245
- llmError: null,
15246
- usage: generated.value.usage,
15247
- armBExit: null,
15248
- armBPrefixExecuted: null,
15249
- armBPrefixDivergences: null,
15250
- failureVanished: null,
15251
- armBError: message.slice(0, 500),
15252
- attempts: null,
15253
- flippedAtAttempt: null
15254
- }
15255
- };
15256
- onProgress(`${label}: arm B error \u2014 ${message.slice(0, 200)}`);
15257
- }
15258
- }
15259
- llm = {
15260
- model: options.fixModelLabel ?? "unknown",
15261
- calls,
15262
- failures,
15263
- promptTokens,
15264
- completionTokens
15265
- };
15266
- }
15267
- const executedRows = rows.filter((r) => r.status === "ok");
15268
- const replayed = executedRows.filter((r) => r.replayed);
15269
- const signatureStrict = executedRows.filter(
15270
- (r) => r.replayed && r.armASignatureMatch
15271
- );
15272
- const armBExecuted = rows.filter((r) => r.fix?.attempted && r.fix.failureVanished !== null);
15273
- const flipped = armBExecuted.filter((r) => r.fix.failureVanished === true);
15274
- const armBNonzeroRc = armBExecuted.filter(
15275
- (r) => r.recordedReturncodeAtK !== null && r.recordedReturncodeAtK !== 0
15276
- );
15277
- const flippedNonzeroRc = armBNonzeroRc.filter((r) => r.fix.failureVanished === true);
15278
- const attempt1Executed = rows.filter(
15279
- (r) => r.fix?.attempts?.find((a) => a.attempt === 1)?.executed === true
15280
- );
15281
- const flippedAt1 = rows.filter((r) => r.fix?.flippedAtAttempt === 1);
15282
- const flipsByAttempt = {};
15283
- for (const row of rows) {
15284
- const at = row.fix?.flippedAtAttempt;
15285
- if (typeof at === "number") flipsByAttempt[String(at)] = (flipsByAttempt[String(at)] ?? 0) + 1;
15286
- }
15287
- const excludedByReason = {};
15288
- for (const excluded of enumeration.excluded) {
15289
- excludedByReason[excluded.reason] = (excludedByReason[excluded.reason] ?? 0) + 1;
15290
- }
15291
- const submitGoldsByCorpus = {};
15292
- const submitEntry = (corpus) => submitGoldsByCorpus[corpus] ??= { submitOnlyCases: 0, goldsSkippedWithinReplayable: 0 };
15293
- for (const excluded of enumeration.excluded) {
15294
- if (excluded.reason === "gold-only-submit-step") submitEntry(excluded.corpus).submitOnlyCases += 1;
15295
- }
15296
- for (const replayCase of enumeration.replayable) {
15297
- if (replayCase.submitGoldsSkipped > 0) {
15298
- submitEntry(replayCase.corpus).goldsSkippedWithinReplayable += replayCase.submitGoldsSkipped;
15299
- }
15300
- }
15301
- const report = {
15302
- generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
15303
- corpora: options.corpora.map((c) => ({
15304
- name: c.name,
15305
- labelsPath: c.labelsPath,
15306
- preparedDir: c.preparedDir
15307
- })),
15308
- totals: {
15309
- labelEntries: enumeration.labelEntryCount,
15310
- replayable: enumeration.replayable.length,
15311
- executed: selected.length,
15312
- excludedByReason,
15313
- submitGoldsByCorpus
15314
- },
15315
- headline: {
15316
- replayabilityRate: {
15317
- numerator: replayed.length,
15318
- denominator: selected.length,
15319
- value: rate(replayed.length, selected.length)
15320
- },
15321
- signatureStrictRate: {
15322
- numerator: signatureStrict.length,
15323
- denominator: selected.length,
15324
- value: rate(signatureStrict.length, selected.length)
15325
- },
15326
- fixFlipRate: options.fix !== "none" ? {
15327
- numerator: flipped.length,
15328
- denominator: armBExecuted.length,
15329
- value: rate(flipped.length, armBExecuted.length)
15330
- } : null,
15331
- fixFlipRateNonzeroRc: options.fix !== "none" ? {
15332
- numerator: flippedNonzeroRc.length,
15333
- denominator: armBNonzeroRc.length,
15334
- value: rate(flippedNonzeroRc.length, armBNonzeroRc.length)
15335
- } : null,
15336
- fixFlipAttempt1: options.fix === "loop" ? {
15337
- numerator: flippedAt1.length,
15338
- denominator: attempt1Executed.length,
15339
- value: rate(flippedAt1.length, attempt1Executed.length)
15340
- } : null,
15341
- flipsByAttempt: options.fix === "loop" ? flipsByAttempt : null
15342
- },
15343
- llm,
15344
- excluded: enumeration.excluded,
15345
- pullFailures,
15346
- cases: rows
15347
- };
15348
- writeFileSync2(join26(options.out, "batch-report.json"), `${JSON.stringify(report, null, 2)}
15349
- `);
15350
- writeFileSync2(join26(options.out, "batch-report.md"), renderBatchReport(report));
15351
- return report;
15352
- }
15353
- function requireOption2(value, name) {
15354
- if (value === void 0) throw new Error(`replay-verify-batch: ${name} is required`);
15355
- return value;
15356
- }
15357
- function pct(value) {
15358
- return value === null ? "\u2014" : `${(value * 100).toFixed(1)}%`;
15359
- }
15360
- function renderBatchReport(report) {
15361
- const lines = [];
15362
- const { headline, totals } = report;
15363
- lines.push("# Replay-verify batch report");
15364
- lines.push("");
15365
- lines.push(`Generated ${report.generatedAt}.`);
15366
- lines.push("");
15367
- lines.push("## Headline");
15368
- lines.push("");
15369
- lines.push(
15370
- `- **Replayability rate: ${pct(headline.replayabilityRate.value)}** (${headline.replayabilityRate.numerator}/${headline.replayabilityRate.denominator} replayable cases where the prefix replayed with \u226410% returncode divergence AND arm A reproduced the recorded returncode at the gold step k).`
15371
- );
15372
- lines.push(
15373
- `- Signature-strict rate: ${pct(headline.signatureStrictRate.value)} (${headline.signatureStrictRate.numerator}/${headline.signatureStrictRate.denominator}; additionally requires the recorded error substring in arm A output).`
15374
- );
15375
- if (headline.fixFlipRate) {
15376
- lines.push(
15377
- `- **Fix-flip rate: ${pct(headline.fixFlipRate.value)}** (${headline.fixFlipRate.numerator}/${headline.fixFlipRate.denominator} arm-B-executed cases where the generated fix made the failure vanish).`
15378
- );
15379
- }
15380
- if (headline.fixFlipRateNonzeroRc) {
15381
- lines.push(
15382
- `- Fix-flip rate on recorded-rc\u22600 cases: ${pct(headline.fixFlipRateNonzeroRc.value)} (${headline.fixFlipRateNonzeroRc.numerator}/${headline.fixFlipRateNonzeroRc.denominator}; real recorded failures \u2014 a gold step recorded with rc 0 flips vacuously).`
15383
- );
15384
- }
15385
- if (headline.fixFlipAttempt1) {
15386
- lines.push(
15387
- `- Fix-flip@1: ${pct(headline.fixFlipAttempt1.value)} (${headline.fixFlipAttempt1.numerator}/${headline.fixFlipAttempt1.denominator} cases whose attempt 1 executed \u2014 the one-shot-comparable number).`
15388
- );
15389
- }
15390
- if (headline.flipsByAttempt && Object.keys(headline.flipsByAttempt).length > 0) {
15391
- const parts = Object.entries(headline.flipsByAttempt).sort((a, b) => Number(a[0]) - Number(b[0])).map(([attempt, count2]) => `attempt ${attempt}: ${count2}`);
15392
- lines.push(`- Flips by attempt: ${parts.join(", ")}.`);
15393
- }
15394
- lines.push("");
15395
- lines.push("## Enumeration");
15396
- lines.push("");
15397
- lines.push(
15398
- `${totals.labelEntries} label entries across ${report.corpora.length} corpora \u2192 ${totals.replayable} replayable (SWE-style docker image + \u22651 gold incorrect step), ${totals.executed} executed.`
15399
- );
15400
- lines.push("");
15401
- lines.push("| exclusion reason | count |");
15402
- lines.push("| --- | --- |");
15403
- for (const [reason, count2] of Object.entries(totals.excludedByReason).sort((a, b) => b[1] - a[1])) {
15404
- lines.push(`| ${reason} | ${count2} |`);
15405
- }
15406
- const submitEntries = Object.entries(totals.submitGoldsByCorpus);
15407
- if (submitEntries.length > 0) {
15408
- lines.push("");
15409
- lines.push(
15410
- "Submit-command golds are never counterfactual targets (a gold on the submit step marks a bad submit decision, not a failed command):"
15411
- );
15412
- lines.push("");
15413
- lines.push("| corpus | cases excluded (all golds = submit) | golds skipped within replayable cases |");
15414
- lines.push("| --- | --- | --- |");
15415
- for (const [corpus, stats] of submitEntries.sort((a, b) => a[0].localeCompare(b[0]))) {
15416
- lines.push(`| ${corpus} | ${stats.submitOnlyCases} | ${stats.goldsSkippedWithinReplayable} |`);
15417
- }
15418
- }
15419
- if (report.pullFailures.length > 0) {
15420
- lines.push("");
15421
- lines.push("## Image pull/build failures");
15422
- lines.push("");
15423
- lines.push("| corpus | trajectory | image | error |");
15424
- lines.push("| --- | --- | --- | --- |");
15425
- for (const failure2 of report.pullFailures) {
15426
- lines.push(
15427
- `| ${failure2.corpus} | ${failure2.trajId} | \`${failure2.image}\` | ${failure2.error.replaceAll("|", "\\|")} |`
15428
- );
15429
- }
15430
- }
15431
- lines.push("");
15432
- lines.push("## Per-case results");
15433
- lines.push("");
15434
- lines.push(
15435
- "| corpus | trajectory | k | rc@k | prefix | div | div% | armA exit | rc match | sig match | replayed | fix | armB exit | vanished | wall s |"
15436
- );
15437
- lines.push(
15438
- "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |"
15439
- );
15440
- for (const row of report.cases) {
15441
- const fix = row.fix;
15442
- const fixCell = !fix ? "\u2014" : fix.sampledOut ? "sampled-out" : fix.llmError ? "llm-failed" : fix.armBError ? "armB-error" : fix.attempts ? fix.flippedAtAttempt !== null ? `flip@${fix.flippedAtAttempt}` : `exhausted(${fix.attempts.length})` : "generated";
15443
- lines.push(
15444
- [
15445
- row.corpus,
15446
- row.trajId.length > 48 ? `${row.trajId.slice(0, 45)}\u2026` : row.trajId,
15447
- row.k,
15448
- row.recordedReturncodeAtK ?? "null",
15449
- row.prefixExecuted ?? "\u2014",
15450
- row.prefixDivergences ?? "\u2014",
15451
- row.prefixDivergencePct ?? "\u2014",
15452
- row.status === "ok" ? row.armAExit : row.status,
15453
- row.armAReturncodeMatch ? "yes" : "no",
15454
- row.armASignatureMatch ? "yes" : "no",
15455
- row.replayed ? "**yes**" : "no",
15456
- fixCell,
15457
- fix?.armBExit ?? "\u2014",
15458
- fix?.failureVanished === null || fix === null ? "\u2014" : fix.failureVanished ? "**yes**" : "no",
15459
- (row.wallMs / 1e3).toFixed(1)
15460
- ].join(" | ")
15461
- );
15462
- }
15463
- if (report.llm) {
15464
- lines.push("");
15465
- lines.push("## LLM fix generation");
15466
- lines.push("");
15467
- lines.push(
15468
- `Model ${report.llm.model}: ${report.llm.calls} calls (${report.llm.failures} failed), ${report.llm.promptTokens} prompt + ${report.llm.completionTokens} completion tokens.`
15469
- );
15470
- }
15471
- lines.push("");
15472
- return lines.join("\n");
15473
- }
15474
- function parseReplayBatchArgs(argv) {
15475
- const corpora = [];
15476
- const values = /* @__PURE__ */ new Map();
15477
- let enumerateOnly = false;
15478
- for (let i = 0; i < argv.length; i++) {
15479
- const arg = argv[i];
15480
- if (arg === "--help" || arg === "-h") return "help";
15481
- if (arg === "--enumerate-only") {
15482
- enumerateOnly = true;
15483
- continue;
15484
- }
15485
- if (!arg.startsWith("--")) throw new Error(`unexpected positional argument: ${arg}`);
15486
- const value = argv[++i];
15487
- if (value === void 0) throw new Error(`missing value for ${arg}`);
15488
- if (arg === "--corpus") corpora.push(parseCorpusFlag(value));
15489
- else values.set(arg, value);
15490
- }
15491
- if (corpora.length === 0) {
15492
- throw new Error("replay-verify-batch: at least one --corpus name=<labels>::<prepared> is required");
15493
- }
15494
- const out = values.get("--out");
15495
- if (!out && !enumerateOnly) throw new Error("replay-verify-batch: --out is required");
15496
- const optionalNumber = (flag) => {
15497
- const v = values.get(flag);
15498
- if (v === void 0) return void 0;
15499
- const n = Number(v);
15500
- if (!Number.isFinite(n)) throw new Error(`replay-verify-batch: ${flag} must be a number, got ${v}`);
15501
- return n;
15502
- };
15503
- const fix = values.get("--fix") ?? "none";
15504
- if (fix !== "none" && fix !== "generate" && fix !== "loop") {
15505
- throw new Error(`replay-verify-batch: --fix must be none, generate, or loop, got ${fix}`);
15506
- }
15507
- const fixAttempts = optionalNumber("--fix-attempts") ?? 3;
15508
- if (!Number.isInteger(fixAttempts) || fixAttempts < 1) {
15509
- throw new Error(
15510
- `replay-verify-batch: --fix-attempts must be a positive integer, got ${values.get("--fix-attempts")}`
15511
- );
15512
- }
15513
- return {
14161
+ const report = await runReplayBatch({
15514
14162
  corpora,
15515
- out: out ?? ".",
14163
+ out,
15516
14164
  fix,
15517
14165
  fixAttempts,
15518
- fixModel: values.get("--fix-model") ?? "glm-5.2",
15519
- fixBaseUrl: values.get("--fix-base-url") ?? "https://api.z.ai/api/coding/paas/v4",
15520
- fixApiKeyEnv: values.get("--fix-api-key-env") ?? "ZAI_GLM_API_KEY",
15521
- maxFixCases: optionalNumber("--max-fix-cases") ?? 30,
15522
- seed: optionalNumber("--seed") ?? 17,
15523
- stepTimeoutMs: optionalNumber("--step-timeout"),
15524
- prefixLimit: optionalNumber("--prefix-limit"),
15525
- caseFilter: values.get("--case-filter"),
15526
- caseLimit: optionalNumber("--case-limit"),
15527
- baseUrl: values.get("--base-url") ?? process.env.SANDBOX_API_URL ?? "http://127.0.0.1:4097",
15528
- apiKeyEnv: values.get("--api-key-env") ?? "SANDBOX_API_KEY",
15529
- maxLifetimeSeconds: optionalNumber("--max-lifetime"),
15530
- enumerateOnly
15531
- };
15532
- }
15533
- function replayBatchUsage() {
15534
- return `traces replay-verify-batch \u2014 measure replayability and fix-flip rates across gold-labeled corpora
15535
-
15536
- Usage:
15537
- traces replay-verify-batch \\
15538
- --corpus NAME=<labels.json>::<prepared-dir> [--corpus ...] \\
15539
- --out DIR [--fix none|generate|loop] [--fix-attempts 3] [--enumerate-only] \\
15540
- [--fix-model glm-5.2] [--fix-base-url URL] [--fix-api-key-env ZAI_GLM_API_KEY] \\
15541
- [--max-fix-cases 30] [--seed 17] [--step-timeout MS] [--prefix-limit N] \\
15542
- [--case-filter SUBSTRING] [--case-limit N] \\
15543
- [--base-url URL] [--api-key-env SANDBOX_API_KEY] [--max-lifetime SECONDS]
15544
-
15545
- Replayable = the raw trajectory carries info.docker_config.base_image AND the
15546
- labels mark at least one gold incorrect step that is a real mid-trajectory
15547
- action. k = the first such gold step; golds whose action is the agent's
15548
- submit command (COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT) are skipped and
15549
- counted, and cases whose golds are all submit steps are excluded as
15550
- gold-only-submit-step. Excluded cases and image pull failures are report
15551
- rows, never skipped silently. --enumerate-only prints the enumeration
15552
- without touching a sandbox.
15553
-
15554
- --fix generate runs ONE LLM call per arm-A-reproduced case (cap
15555
- --max-fix-cases, seeded sample beyond it) and executes the corrected command
15556
- as arm B in a fresh sandbox.
15557
-
15558
- --fix loop iterates: attempt 1 is the one-shot prompt; when an arm fails
15559
- (nonzero exit or the failure signature persists) or the model call fails,
15560
- the next prompt carries every prior command with its REAL executed
15561
- stdout/stderr, up to --fix-attempts attempts (default 3). Retries may
15562
- answer with a short script (<=5 commands) executed as one /bin/sh unit.
15563
- Every attempt runs in its own fresh sandbox with the same replayed prefix.
15564
-
15565
- Outputs batch-report.json, batch-report.md, cases.jsonl (incremental), and one
15566
- directory per case with the full replay-verify artifacts.
15567
- See docs/replay-verify.md for orchestrator setup and honest limits.
15568
- `;
15569
- }
15570
- async function cmdReplayVerifyBatch(argv) {
15571
- const parsed = parseReplayBatchArgs(argv);
15572
- if (parsed === "help") {
15573
- process.stdout.write(replayBatchUsage());
15574
- return;
15575
- }
15576
- if (parsed.enumerateOnly) {
15577
- const enumeration = enumerateReplayableCases(parsed.corpora);
15578
- process.stdout.write(
15579
- `${JSON.stringify(
15580
- {
15581
- labelEntries: enumeration.labelEntryCount,
15582
- replayable: enumeration.replayable.map((c) => ({
15583
- corpus: c.corpus,
15584
- trajId: c.trajId,
15585
- image: c.image,
15586
- cwd: c.cwd,
15587
- cwdSource: c.cwdSource,
15588
- k: c.k,
15589
- submitGoldsSkipped: c.submitGoldsSkipped,
15590
- recordedReturncodeAtK: c.recordedReturncodeAtK,
15591
- goldIncorrectSteps: c.goldIncorrectSteps
15592
- })),
15593
- excluded: enumeration.excluded
15594
- },
15595
- null,
15596
- 2
15597
- )}
15598
- `
15599
- );
15600
- return;
15601
- }
15602
- const apiKey = process.env[parsed.apiKeyEnv];
15603
- if (!apiKey) {
15604
- throw new Error(
15605
- `replay-verify-batch: env var ${parsed.apiKeyEnv} is empty \u2014 pass --api-key-env or export it`
15606
- );
15607
- }
15608
- let fixCaller;
15609
- if (parsed.fix !== "none") {
15610
- const fixApiKey = process.env[parsed.fixApiKeyEnv];
15611
- if (!fixApiKey) {
15612
- throw new Error(
15613
- `replay-verify-batch: env var ${parsed.fixApiKeyEnv} is empty \u2014 pass --fix-api-key-env or export it`
15614
- );
15615
- }
15616
- fixCaller = zaiChatCaller({
15617
- baseUrl: parsed.fixBaseUrl,
15618
- apiKey: fixApiKey,
15619
- model: parsed.fixModel
15620
- });
15621
- }
15622
- const report = await runReplayBatch({
15623
- corpora: parsed.corpora,
15624
- out: parsed.out,
15625
- baseUrl: parsed.baseUrl,
15626
- apiKey,
15627
- fix: parsed.fix,
15628
- fixAttempts: parsed.fixAttempts,
15629
14166
  fixCaller,
15630
- fixModelLabel: parsed.fixModel,
15631
- maxFixCases: parsed.maxFixCases,
15632
- seed: parsed.seed,
15633
- stepTimeoutMs: parsed.stepTimeoutMs,
15634
- prefixLimit: parsed.prefixLimit,
15635
- caseFilter: parsed.caseFilter,
15636
- caseLimit: parsed.caseLimit,
15637
- maxLifetimeSeconds: parsed.maxLifetimeSeconds,
15638
- onProgress: (message) => process.stderr.write(`${message}
15639
- `)
14167
+ fixModelLabel: fixModel,
14168
+ maxFixCases: flags.number("--max-fix-cases") ?? 30,
14169
+ seed: flags.number("--seed") ?? 17,
14170
+ stepTimeoutMs: flags.number("--step-timeout"),
14171
+ prefixLimit: flags.number("--prefix-limit"),
14172
+ caseFilter: values.get("--case-filter"),
14173
+ caseLimit: flags.number("--case-limit"),
14174
+ backendFactory: sandboxBackendFactory(sandboxOptions("replay-verify-batch", values, true)),
14175
+ onProgress: progress
15640
14176
  });
15641
14177
  const { replayabilityRate, fixFlipRate } = report.headline;
15642
- process.stderr.write(
15643
- `replay-verify-batch: replayability ${replayabilityRate.numerator}/${replayabilityRate.denominator}` + (fixFlipRate ? `, fix-flip ${fixFlipRate.numerator}/${fixFlipRate.denominator}` : "") + ` \u2192 ${parsed.out}
15644
- `
15645
- );
15646
- }
15647
-
15648
- // src/replay-wire.ts
15649
- var SUBJECT_PATTERN = /^incorrect-steps-(\d+)-(\d+)-(escaped|unescaped)-consequence-(\d+)$/;
15650
- function parseIncorrectStepsSubject(subject) {
15651
- const match = SUBJECT_PATTERN.exec(subject);
15652
- if (!match) return null;
15653
- return {
15654
- firstStep: Number(match[1]),
15655
- lastStep: Number(match[2]),
15656
- escapeStatus: match[3],
15657
- consequenceStep: Number(match[4])
15658
- };
15659
- }
15660
- function resolveFindingInvocation(finding2, corpora) {
15661
- const subject = parseIncorrectStepsSubject(finding2.subject);
15662
- if (!subject) {
15663
- throw new Error(
15664
- `replay-wire: subject '${finding2.subject}' is not incorrect-steps-<first>-<last>-<escaped|unescaped>-consequence-<step>`
15665
- );
15666
- }
15667
- const failures = [];
15668
- for (const corpus of corpora) {
15669
- const resolution = resolveCaseResources(corpus, finding2.trajId);
15670
- if (resolution.resolved) {
15671
- const at = subject.firstStep;
15672
- const step = resolution.resources.steps.find((s) => s.step_id === at);
15673
- if (!step) {
15674
- throw new Error(
15675
- `replay-wire: finding step ${at} is outside ${finding2.trajId} (${resolution.resources.steps.length} steps)`
15676
- );
15677
- }
15678
- return { resources: resolution.resources, subject, at };
15679
- }
15680
- failures.push(`${corpus.name}: ${resolution.reason}`);
15681
- }
15682
- throw new Error(
15683
- `replay-wire: trajectory ${finding2.trajId} is not replayable in any corpus \u2014 ${failures.join("; ")}`
15684
- );
15685
- }
15686
- async function replayVerifyFinding(finding2, options) {
15687
- if (options.fixCaller && options.fixCommand !== void 0) {
15688
- throw new Error("replay-wire: pass fixCaller or fixCommand, not both");
15689
- }
15690
- const invocation = resolveFindingInvocation(finding2, options.corpora);
15691
- let fixCommand = options.fixCommand ?? null;
15692
- if (options.fixCaller) {
15693
- const generated = await generateFixCommand(options.fixCaller, {
15694
- taskStatement: invocation.resources.taskStatement,
15695
- steps: invocation.resources.steps,
15696
- k: invocation.at
15697
- });
15698
- if (!generated.succeeded) {
15699
- throw new Error(`replay-wire: fix generation failed \u2014 ${generated.error}`);
15700
- }
15701
- fixCommand = generated.value.command;
15702
- }
15703
- const verdict = await replayVerify({
15704
- stepsPath: invocation.resources.stepsPath,
15705
- image: invocation.resources.image,
15706
- at: invocation.at,
15707
- fixCommand: fixCommand ?? void 0,
15708
- cwd: invocation.resources.cwd,
15709
- out: options.out,
15710
- caseId: invocation.resources.trajId,
15711
- stepTimeoutMs: options.stepTimeoutMs ?? invocation.resources.recordedStepTimeoutMs ?? void 0,
15712
- prefixLimit: options.prefixLimit,
15713
- backend: options.backend,
15714
- apiKey: options.apiKey,
15715
- baseUrl: options.baseUrl,
15716
- onProgress: options.onProgress
15717
- });
15718
- return { invocation, fixCommand, verdict };
15719
- }
15720
-
15721
- // src/analyze-verify.ts
15722
- import { mkdirSync as mkdirSync3, readFileSync as readFileSync5, writeFileSync as writeFileSync3 } from "fs";
15723
- import { join as join27 } from "path";
15724
- var SINGLE_STEP_SUBJECT = /^incorrect-step-(\d+)$/;
15725
- function findingReplayStep(finding2) {
15726
- const fromMetadata = finding2.metadata?.block_first_step;
15727
- if (typeof fromMetadata === "number" && Number.isInteger(fromMetadata) && fromMetadata >= 1) {
15728
- return fromMetadata;
15729
- }
15730
- const subject = finding2.subject ?? "";
15731
- const wire = parseIncorrectStepsSubject(subject);
15732
- if (wire) return wire.firstStep;
15733
- const single = SINGLE_STEP_SUBJECT.exec(subject);
15734
- if (single) return Number(single[1]);
15735
- return null;
15736
- }
15737
- var TRACE_EVIDENCE_URI = /^trace:\/\/([^/]+)\//;
15738
- function findingTrajectoryId(finding2) {
15739
- for (const ref of finding2.evidence_refs ?? []) {
15740
- if (typeof ref.uri !== "string") continue;
15741
- const match = TRACE_EVIDENCE_URI.exec(ref.uri);
15742
- if (match) return match[1];
15743
- }
15744
- return null;
15745
- }
15746
- function checkStep(steps, at) {
15747
- const step = steps.find((s) => s.step_id === at);
15748
- if (!step) {
15749
- return { ok: false, reason: `step ${at} is outside the trajectory (${steps.length} steps)` };
15750
- }
15751
- if (isSubmitAction(step.action)) {
15752
- return {
15753
- ok: false,
15754
- reason: `step ${at} is the submit action \u2014 a submit decision has no executable failure to replay`
15755
- };
15756
- }
15757
- const recordedReturncode = parseRecordedReturncode(step.observation);
15758
- if (recordedReturncode === null) {
15759
- return {
15760
- ok: false,
15761
- reason: `step ${at} recorded no returncode \u2014 there is no executable failure signature to reproduce`
15762
- };
15763
- }
15764
- return { ok: true, recordedReturncode };
15765
- }
15766
- function resolveFindingReplayability(finding2, source) {
15767
- const at = findingReplayStep(finding2);
15768
- if (at === null) {
15769
- return {
15770
- replayable: false,
15771
- reason: `subject '${finding2.subject ?? "(none)"}' names no trajectory step (expected incorrect-step-<n>, incorrect-steps-<f>-<l>-\u2026, or metadata.block_first_step)`
15772
- };
15773
- }
15774
- if (source.kind === "direct") {
15775
- const trajId2 = findingTrajectoryId(finding2);
15776
- if (trajId2 && source.caseId && trajId2 !== source.caseId) {
15777
- return {
15778
- replayable: false,
15779
- reason: `finding cites trajectory '${trajId2}' but the supplied steps are case '${source.caseId}'`
15780
- };
15781
- }
15782
- let steps;
15783
- try {
15784
- steps = JSON.parse(readFileSync5(source.stepsPath, "utf8"));
15785
- } catch (err) {
15786
- throw new Error(
15787
- `verify-findings: cannot read steps file ${source.stepsPath} \u2014 ${err instanceof Error ? err.message : String(err)}`
15788
- );
15789
- }
15790
- if (!Array.isArray(steps) || steps.length === 0) {
15791
- throw new Error(`verify-findings: ${source.stepsPath} is not a non-empty steps array`);
15792
- }
15793
- const step = checkStep(steps, at);
15794
- if (!step.ok) return { replayable: false, reason: step.reason };
15795
- return {
15796
- replayable: true,
15797
- resolved: {
15798
- caseId: source.caseId ?? trajId2 ?? source.stepsPath,
15799
- stepsPath: source.stepsPath,
15800
- image: source.image,
15801
- cwd: source.cwd,
15802
- at,
15803
- recordedReturncode: step.recordedReturncode,
15804
- recordedStepTimeoutMs: null
15805
- }
15806
- };
15807
- }
15808
- const trajId = findingTrajectoryId(finding2);
15809
- if (!trajId) {
15810
- return {
15811
- replayable: false,
15812
- reason: "finding carries no trace://<trajectory>/ evidence ref naming its trajectory"
15813
- };
15814
- }
15815
- const failures = [];
15816
- for (const corpus of source.corpora) {
15817
- const resolution = resolveCaseResources(corpus, trajId);
15818
- if (!resolution.resolved) {
15819
- failures.push(`${corpus.name}: ${resolution.reason}${resolution.detail ? ` (${resolution.detail})` : ""}`);
15820
- continue;
15821
- }
15822
- const step = checkStep(resolution.resources.steps, at);
15823
- if (!step.ok) return { replayable: false, reason: step.reason };
15824
- return {
15825
- replayable: true,
15826
- resolved: {
15827
- caseId: trajId,
15828
- stepsPath: resolution.resources.stepsPath,
15829
- image: resolution.resources.image,
15830
- cwd: resolution.resources.cwd,
15831
- at,
15832
- recordedReturncode: step.recordedReturncode,
15833
- recordedStepTimeoutMs: resolution.resources.recordedStepTimeoutMs
15834
- }
15835
- };
15836
- }
15837
- return {
15838
- replayable: false,
15839
- reason: `trajectory ${trajId} is not replayable in any corpus \u2014 ${failures.join("; ")}`
15840
- };
15841
- }
15842
- function classifyVerdict(verdict) {
15843
- if (!verdict.armA.failureSignatureMatch) return "divergent";
15844
- if (verdict.armB?.failureVanished) return "fix-flipped";
15845
- return "reproduced";
15846
- }
15847
- var DEFAULT_SANDBOX_BASE_URL = "http://127.0.0.1:4097";
15848
- async function assertSandboxReachable(baseUrl) {
15849
- let response;
15850
- try {
15851
- response = await fetch(new URL("/health", baseUrl), { signal: AbortSignal.timeout(5e3) });
15852
- } catch (err) {
15853
- throw new Error(
15854
- `verify-findings: sandbox API unreachable at ${baseUrl} \u2014 ${err instanceof Error ? err.message : String(err)}. Executing findings requires a running sandbox orchestrator + SDK adapter (see docs/replay-verify.md); start them or pass --base-url.`
15855
- );
15856
- }
15857
- if (!response.ok) {
15858
- throw new Error(
15859
- `verify-findings: sandbox API at ${baseUrl} answered /health with HTTP ${response.status} \u2014 refusing to run proofs against degraded infrastructure`
15860
- );
15861
- }
15862
- }
15863
- function receiptExecution(verdict) {
15864
- return {
15865
- image: verdict.image,
15866
- cwd: verdict.cwd,
15867
- recordedReturncode: verdict.recordedReturncode,
15868
- signature: verdict.signature,
15869
- signatureBasis: verdict.signatureBasis,
15870
- armA: {
15871
- command: verdict.armA.command,
15872
- exitCode: verdict.armA.exitCode,
15873
- wallMs: verdict.armA.wallMs,
15874
- failureSignatureMatch: verdict.armA.failureSignatureMatch
15875
- },
15876
- armB: verdict.armB ? {
15877
- command: verdict.armB.command,
15878
- exitCode: verdict.armB.exitCode,
15879
- wallMs: verdict.armB.wallMs,
15880
- failureVanished: verdict.armB.failureVanished
15881
- } : null,
15882
- prefixExecuted: verdict.prefixExecuted,
15883
- prefixDivergences: verdict.prefixDivergences.length,
15884
- totalMs: verdict.timings.totalMs
15885
- };
15886
- }
15887
- function writeReceipt(receiptDir, finding2, verification, execution) {
15888
- const receipt = {
15889
- schema_version: "1.0.0",
15890
- produced_at: (/* @__PURE__ */ new Date()).toISOString(),
15891
- finding_id: verification.finding_id,
15892
- analyst_id: finding2.analyst_id ?? null,
15893
- subject: verification.subject,
15894
- claim: typeof finding2.claim === "string" ? finding2.claim.slice(0, 600) : null,
15895
- trajectory_id: verification.trajectory_id,
15896
- step: verification.step,
15897
- verified: verification.verified,
15898
- reason: verification.reason,
15899
- execution,
15900
- verdict_path: verification.verdict_path,
15901
- deduplicated_with: verification.deduplicated_with
15902
- };
15903
- writeFileSync3(join27(receiptDir, "receipt.json"), `${JSON.stringify(receipt, null, 2)}
15904
- `);
15905
- }
15906
- function receiptDirName(index, finding2) {
15907
- const id = typeof finding2.finding_id === "string" && finding2.finding_id.length > 0 ? finding2.finding_id.replace(/[^A-Za-z0-9_-]/g, "_") : "finding";
15908
- return `${String(index + 1).padStart(3, "0")}-${id}`;
15909
- }
15910
- async function verifyFindings(findings, options) {
15911
- if (findings.length === 0) throw new Error("verify-findings: no findings to verify");
15912
- mkdirSync3(options.out, { recursive: true });
15913
- const resolutions = findings.map((finding2) => resolveFindingReplayability(finding2, options.source));
15914
- const baseUrl = options.baseUrl ?? DEFAULT_SANDBOX_BASE_URL;
15915
- if (resolutions.some((resolution) => resolution.replayable) && !options.backend) {
15916
- if (!options.apiKey) {
15917
- throw new Error(
15918
- "verify-findings: replayable findings need a sandbox API key \u2014 export SANDBOX_API_KEY or pass --api-key-env"
15919
- );
15920
- }
15921
- await assertSandboxReachable(baseUrl);
15922
- }
15923
- const preparer = options.source.kind === "corpus" && !options.backend ? options.source.preparer ?? dockerImagePreparer() : null;
15924
- const preparedImages = /* @__PURE__ */ new Map();
15925
- const executedByKey = /* @__PURE__ */ new Map();
15926
- const verifications = [];
15927
- let executions = 0;
15928
- for (let index = 0; index < findings.length; index++) {
15929
- const finding2 = findings[index];
15930
- const resolution = resolutions[index];
15931
- const receiptDir = join27(options.out, receiptDirName(index, finding2));
15932
- mkdirSync3(receiptDir, { recursive: true });
15933
- const identity2 = {
15934
- finding_id: finding2.finding_id ?? null,
15935
- subject: finding2.subject ?? null,
15936
- trajectory_id: findingTrajectoryId(finding2)
15937
- };
15938
- if (!resolution.replayable) {
15939
- const verification2 = {
15940
- ...identity2,
15941
- step: findingReplayStep(finding2),
15942
- verified: "not-replayable",
15943
- reason: resolution.reason,
15944
- receipt: receiptDir,
15945
- verdict_path: null,
15946
- deduplicated_with: null
15947
- };
15948
- writeReceipt(receiptDir, finding2, verification2, null);
15949
- verifications.push(verification2);
15950
- options.onProgress?.(`${identity2.finding_id ?? `finding ${index + 1}`}: not-replayable \u2014 ${resolution.reason}`);
15951
- continue;
15952
- }
15953
- const resolved = resolution.resolved;
15954
- const dedupeKey = `${resolved.caseId}::${resolved.at}::${options.fixCommand ?? ""}`;
15955
- const prior = executedByKey.get(dedupeKey);
15956
- if (prior) {
15957
- const verification2 = {
15958
- ...identity2,
15959
- step: resolved.at,
15960
- verified: prior.verified,
15961
- reason: prior.reason,
15962
- receipt: receiptDir,
15963
- verdict_path: prior.verdict_path,
15964
- deduplicated_with: prior.receipt
15965
- };
15966
- const priorVerdict = prior.verdict_path ? JSON.parse(readFileSync5(prior.verdict_path, "utf8")) : null;
15967
- writeReceipt(receiptDir, finding2, verification2, priorVerdict ? receiptExecution(priorVerdict) : null);
15968
- verifications.push(verification2);
15969
- options.onProgress?.(
15970
- `${identity2.finding_id ?? `finding ${index + 1}`}: ${prior.verified} (shares proof with ${prior.finding_id ?? prior.receipt})`
15971
- );
15972
- continue;
15973
- }
15974
- let image = resolved.image;
15975
- if (preparer) {
15976
- const preparationKey = `${resolved.image}::${resolved.cwd}`;
15977
- let preparation = preparedImages.get(preparationKey);
15978
- if (!preparation) {
15979
- const ensured = await preparer.ensure(resolved.image, resolved.cwd);
15980
- preparation = ensured.succeeded ? { succeeded: true, image: ensured.value.derivedImage } : { succeeded: false, error: ensured.error };
15981
- preparedImages.set(preparationKey, preparation);
15982
- }
15983
- if (!preparation.succeeded) {
15984
- const verification2 = {
15985
- ...identity2,
15986
- step: resolved.at,
15987
- verified: "not-replayable",
15988
- reason: `replay image could not be prepared \u2014 ${preparation.error}`,
15989
- receipt: receiptDir,
15990
- verdict_path: null,
15991
- deduplicated_with: null
15992
- };
15993
- writeReceipt(receiptDir, finding2, verification2, null);
15994
- verifications.push(verification2);
15995
- options.onProgress?.(`${identity2.finding_id ?? `finding ${index + 1}`}: not-replayable \u2014 image preparation failed`);
15996
- continue;
15997
- }
15998
- image = preparation.image;
15999
- }
16000
- options.onProgress?.(
16001
- `${identity2.finding_id ?? `finding ${index + 1}`}: executing arm A at step ${resolved.at} of ${resolved.caseId} on ${image}`
16002
- );
16003
- const verdict = await replayVerify({
16004
- stepsPath: resolved.stepsPath,
16005
- image,
16006
- at: resolved.at,
16007
- fixCommand: options.fixCommand,
16008
- cwd: resolved.cwd,
16009
- out: receiptDir,
16010
- caseId: resolved.caseId,
16011
- stepTimeoutMs: options.stepTimeoutMs ?? resolved.recordedStepTimeoutMs ?? void 0,
16012
- prefixLimit: options.prefixLimit,
16013
- backend: options.backend,
16014
- apiKey: options.apiKey,
16015
- baseUrl,
16016
- maxLifetimeSeconds: options.maxLifetimeSeconds,
16017
- onProgress: options.onProgress
16018
- });
16019
- executions += 1;
16020
- const verification = {
16021
- ...identity2,
16022
- step: resolved.at,
16023
- verified: classifyVerdict(verdict),
16024
- reason: null,
16025
- receipt: receiptDir,
16026
- verdict_path: join27(receiptDir, "replay-verdict.json"),
16027
- deduplicated_with: null
16028
- };
16029
- executedByKey.set(dedupeKey, verification);
16030
- writeReceipt(receiptDir, finding2, verification, receiptExecution(verdict));
16031
- verifications.push(verification);
16032
- options.onProgress?.(`${identity2.finding_id ?? `finding ${index + 1}`}: ${verification.verified}`);
16033
- }
16034
- const counts = {
16035
- reproduced: 0,
16036
- "fix-flipped": 0,
16037
- divergent: 0,
16038
- "not-replayable": 0
16039
- };
16040
- for (const verification of verifications) counts[verification.verified] += 1;
16041
- const run = { out: options.out, verifications, counts, executions };
16042
- writeFileSync3(
16043
- join27(options.out, "verifications.json"),
16044
- `${JSON.stringify({ schema_version: "1.0.0", ...run }, null, 2)}
16045
- `
16046
- );
16047
- return run;
16048
- }
16049
- function verdictCell(verification) {
16050
- switch (verification.verified) {
16051
- case "reproduced":
16052
- return "**VERIFIED** \u2014 reproduced";
16053
- case "fix-flipped":
16054
- return "**VERIFIED** \u2014 fix-flipped";
16055
- case "divergent":
16056
- return "DIVERGENT \u2014 recorded failure did not reproduce";
16057
- case "not-replayable":
16058
- return `UNVERIFIABLE \u2014 ${verification.reason ?? "no reason recorded"}`;
16059
- }
16060
- }
16061
- function renderVerifiedFindingsSection(run) {
16062
- const lines = ["## Verified findings (executed replay)", ""];
16063
- const total = run.verifications.length;
16064
- lines.push(
16065
- `${total} finding(s) \u2192 ${run.counts.reproduced} reproduced, ${run.counts["fix-flipped"]} fix-flipped, ${run.counts.divergent} divergent, ${run.counts["not-replayable"]} not replayable (${run.executions} sandbox execution(s); findings accusing the same step share one proof).`
16066
- );
16067
- lines.push("");
16068
- lines.push("| Finding | Subject | Step | Verdict | Receipt |");
16069
- lines.push("|---|---|---:|---|---|");
16070
- for (const verification of run.verifications) {
16071
- const shared = verification.deduplicated_with ? " (shared proof)" : "";
16072
- lines.push(
16073
- `| \`${verification.finding_id ?? "\u2014"}\` | \`${verification.subject ?? "\u2014"}\` | ${verification.step ?? "\u2014"} | ${verdictCell(verification)} | \`${verification.receipt}\`${shared} |`
16074
- );
16075
- }
16076
- lines.push("");
16077
- lines.push(
16078
- "VERIFIED = the accused step was re-executed in a sandbox after replaying the trajectory prefix, and the recorded failure signature reproduced (fix-flipped: a corrected command additionally made it vanish). Each receipt directory carries receipt.json and, when executed, replay-verdict.json + report.md with real stdout/stderr."
14178
+ progress(
14179
+ `replay-verify-batch: replayability ${replayabilityRate.numerator}/${replayabilityRate.denominator}` + (fixFlipRate ? `, fix-flip ${fixFlipRate.numerator}/${fixFlipRate.denominator}` : "") + ` \u2192 ${out}`
16079
14180
  );
16080
- lines.push("");
16081
- return lines.join("\n");
16082
- }
16083
- function readFindingsFile(path) {
16084
- const parsed = JSON.parse(readFileSync5(path, "utf8"));
16085
- const array = Array.isArray(parsed) ? parsed : parsed && typeof parsed === "object" && Array.isArray(parsed.findings) ? parsed.findings : null;
16086
- if (!array) {
16087
- throw new Error(`${path} is neither a findings array nor an object with a findings array`);
16088
- }
16089
- for (const entry of array) {
16090
- if (!entry || typeof entry !== "object") {
16091
- throw new Error(`${path}: every finding must be an object, got ${JSON.stringify(entry)}`);
16092
- }
16093
- }
16094
- return array;
16095
14181
  }
16096
- function verifyFindingsUsage() {
16097
- return `traces verify-findings \u2014 execute analyst findings as sandbox replay proofs
14182
+ var VERIFY_FINDINGS_USAGE = `traces verify-findings \u2014 execute analyst findings as sandbox replay proofs
16098
14183
 
16099
14184
  Usage:
16100
14185
  traces verify-findings --findings FINDINGS.json --out DIR \\
@@ -16111,7 +14196,7 @@ Usage:
16111
14196
  verify against one trajectory; the image must be replay-ready
16112
14197
  (uid-1000 derived), exactly like traces replay-verify
16113
14198
  --case trajectory id of --steps; findings citing other trajectories
16114
- are honestly marked not-replayable
14199
+ are marked not-replayable
16115
14200
  --corpus resolve each finding's trajectory in CodeTraceBench corpora;
16116
14201
  images are derived via the batch uid-1000 preparer (docker)
16117
14202
  --fix-command corrected step for arm B; a reproduced finding whose fix
@@ -16124,110 +14209,83 @@ with a receipt directory carrying the executed evidence or the precise reason.
16124
14209
  Requires a running sandbox orchestrator when any finding is replayable \u2014 infra
16125
14210
  absence is an error, never a silent skip. See docs/replay-verify.md for setup.
16126
14211
  `;
14212
+ function verifyFindingsInSandbox(findings, options) {
14213
+ if (findings.length === 0) throw new Error("verify-findings: no findings to verify");
14214
+ const { sandbox, ...rest } = options;
14215
+ return verifyFindings(findings, {
14216
+ ...rest,
14217
+ backendFactory: sandboxBackendFactory(sandbox),
14218
+ preflight: async () => {
14219
+ if (!sandbox.apiKey) {
14220
+ throw new Error(
14221
+ "verify-findings: replayable findings need a sandbox API key \u2014 export SANDBOX_API_KEY or pass --api-key-env"
14222
+ );
14223
+ }
14224
+ await assertSandboxReachable(sandbox.baseUrl);
14225
+ },
14226
+ onProgress: progress
14227
+ });
16127
14228
  }
16128
- function parseVerifyFindingsArgs(argv) {
16129
- const values = /* @__PURE__ */ new Map();
16130
- const corpora = [];
16131
- for (let i = 0; i < argv.length; i++) {
16132
- const arg = argv[i];
16133
- if (arg === "--help" || arg === "-h") return "help";
16134
- if (!arg.startsWith("--")) throw new Error(`unexpected positional argument: ${arg}`);
16135
- const value = argv[++i];
16136
- if (value === void 0) throw new Error(`missing value for ${arg}`);
16137
- if (arg === "--corpus") {
16138
- corpora.push(parseCorpusFlag(value));
16139
- continue;
16140
- }
16141
- values.set(arg, value);
14229
+ async function cmdVerifyFindings(argv) {
14230
+ const parsed = parseFlags2("verify-findings", argv);
14231
+ if (parsed === "help") {
14232
+ process.stdout.write(VERIFY_FINDINGS_USAGE);
14233
+ return;
16142
14234
  }
16143
- const required = (flag) => {
16144
- const v = values.get(flag);
16145
- if (v === void 0) throw new Error(`verify-findings: ${flag} is required`);
16146
- return v;
16147
- };
16148
- const optionalNumber = (flag) => {
16149
- const v = values.get(flag);
16150
- if (v === void 0) return void 0;
16151
- const n = Number(v);
16152
- if (!Number.isFinite(n)) throw new Error(`verify-findings: ${flag} must be a number, got ${v}`);
16153
- return n;
16154
- };
16155
- const findingsPath = required("--findings");
16156
- const out = required("--out");
14235
+ const { values, corpora } = parsed;
14236
+ const flags = flagReader("verify-findings", values);
14237
+ const findingsPath = flags.required("--findings");
14238
+ const out = flags.required("--out");
16157
14239
  const direct = values.has("--steps") || values.has("--image") || values.has("--cwd");
16158
14240
  if (direct && corpora.length > 0) {
16159
14241
  throw new Error("verify-findings: pass --steps/--image/--cwd or --corpus, not both");
16160
14242
  }
16161
14243
  let source;
16162
14244
  if (direct) {
14245
+ const caseId = values.get("--case");
16163
14246
  source = {
16164
14247
  kind: "direct",
16165
- stepsPath: required("--steps"),
16166
- image: required("--image"),
16167
- cwd: required("--cwd"),
16168
- ...values.has("--case") ? { caseId: values.get("--case") } : {}
14248
+ stepsPath: flags.required("--steps"),
14249
+ image: flags.required("--image"),
14250
+ cwd: flags.required("--cwd"),
14251
+ ...caseId === void 0 ? {} : { caseId }
16169
14252
  };
16170
14253
  } else if (corpora.length > 0) {
16171
14254
  source = { kind: "corpus", corpora };
16172
14255
  } else {
16173
14256
  throw new Error("verify-findings: a replay source is required \u2014 --steps/--image/--cwd or --corpus");
16174
14257
  }
16175
- return {
16176
- findingsPath,
14258
+ const run = await verifyFindingsInSandbox(readFindingsFile(findingsPath), {
16177
14259
  source,
16178
14260
  out,
14261
+ sandbox: sandboxOptions("verify-findings", values, false),
16179
14262
  fixCommand: values.get("--fix-command"),
16180
- stepTimeoutMs: optionalNumber("--step-timeout"),
16181
- prefixLimit: optionalNumber("--prefix-limit"),
16182
- baseUrl: values.get("--base-url") ?? process.env.SANDBOX_API_URL ?? DEFAULT_SANDBOX_BASE_URL,
16183
- apiKeyEnv: values.get("--api-key-env") ?? "SANDBOX_API_KEY",
16184
- maxLifetimeSeconds: optionalNumber("--max-lifetime")
16185
- };
16186
- }
16187
- async function cmdVerifyFindings(argv) {
16188
- const parsed = parseVerifyFindingsArgs(argv);
16189
- if (parsed === "help") {
16190
- process.stdout.write(verifyFindingsUsage());
16191
- return;
16192
- }
16193
- const findings = readFindingsFile(parsed.findingsPath);
16194
- const apiKey = process.env[parsed.apiKeyEnv];
16195
- const run = await verifyFindings(findings, {
16196
- source: parsed.source,
16197
- out: parsed.out,
16198
- fixCommand: parsed.fixCommand,
16199
- stepTimeoutMs: parsed.stepTimeoutMs,
16200
- prefixLimit: parsed.prefixLimit,
16201
- apiKey,
16202
- baseUrl: parsed.baseUrl,
16203
- maxLifetimeSeconds: parsed.maxLifetimeSeconds,
16204
- onProgress: (message) => process.stderr.write(`${message}
16205
- `)
14263
+ stepTimeoutMs: flags.number("--step-timeout"),
14264
+ prefixLimit: flags.number("--prefix-limit")
16206
14265
  });
16207
14266
  process.stdout.write(`${JSON.stringify(run, null, 2)}
16208
14267
  `);
16209
- process.stderr.write(
16210
- `verify-findings: ${run.verifications.length} finding(s) \u2192 ${run.counts.reproduced} reproduced, ${run.counts["fix-flipped"]} fix-flipped, ${run.counts.divergent} divergent, ${run.counts["not-replayable"]} not-replayable (${run.executions} execution(s)) \u2192 ${run.out}
16211
- `
14268
+ progress(
14269
+ `verify-findings: ${run.verifications.length} finding(s) \u2192 ${run.counts.reproduced} reproduced, ${run.counts["fix-flipped"]} fix-flipped, ${run.counts.divergent} divergent, ${run.counts["not-replayable"]} not-replayable (${run.executions} execution(s)) \u2192 ${run.out}`
16212
14270
  );
16213
14271
  }
16214
14272
 
16215
14273
  // src/hodoscope.ts
16216
- import { createHash as createHash13 } from "crypto";
14274
+ import { createHash as createHash12 } from "crypto";
16217
14275
  import { mkdir as mkdir5, mkdtemp as mkdtemp5, readFile as readFile11, readdir as readdir17, writeFile as writeFile7 } from "fs/promises";
16218
- import { endianness, tmpdir as tmpdir6 } from "os";
16219
- import { join as join28, resolve as resolve10 } from "path";
14276
+ import { endianness, tmpdir as tmpdir5 } from "os";
14277
+ import { join as join24, resolve as resolve10 } from "path";
16220
14278
  var HODOSCOPE_VERSION = "0.2.4";
16221
14279
  async function writeHodoscopeInput(otlpPath, options = {}) {
16222
14280
  const explicitDirectory = options.directory ? resolve10(options.directory) : void 0;
16223
- const directory = explicitDirectory ?? await mkdtemp5(join28(tmpdir6(), "traces-hodoscope-input-"));
14281
+ const directory = explicitDirectory ?? await mkdtemp5(join24(tmpdir5(), "traces-hodoscope-input-"));
16224
14282
  if (explicitDirectory) {
16225
14283
  await mkdir5(directory, { recursive: true });
16226
14284
  if ((await readdir17(directory)).length > 0) {
16227
14285
  throw new Error(`Hodoscope explicit directory must be empty: ${directory}`);
16228
14286
  }
16229
14287
  }
16230
- const samples = join28(directory, "samples");
14288
+ const samples = join24(directory, "samples");
16231
14289
  await mkdir5(samples);
16232
14290
  const rows = [];
16233
14291
  for await (const value of readJsonl(otlpPath)) rows.push(parseOpenInferenceRow(value));
@@ -16252,8 +14310,8 @@ async function writeHodoscopeInput(otlpPath, options = {}) {
16252
14310
  throw new Error(`Hodoscope trace '${traceId}' contains no assistant actions`);
16253
14311
  }
16254
14312
  actionCount += actions;
16255
- const file = `${createHash13("sha256").update(traceId).digest("hex").slice(0, 24)}.json`;
16256
- await writeFile7(join28(samples, file), `${JSON.stringify(trajectory, null, 2)}
14313
+ const file = `${createHash12("sha256").update(traceId).digest("hex").slice(0, 24)}.json`;
14314
+ await writeFile7(join24(samples, file), `${JSON.stringify(trajectory, null, 2)}
16257
14315
  `, "utf8");
16258
14316
  }
16259
14317
  return {
@@ -16275,12 +14333,12 @@ function hodoscopeAnalyzer(options = {}) {
16275
14333
  return {
16276
14334
  name: "hodoscope",
16277
14335
  async analyze(otlpPath, runOptions2 = {}) {
16278
- const outputRoot = options.outputDir ? resolve10(options.outputDir) : tmpdir6();
14336
+ const outputRoot = options.outputDir ? resolve10(options.outputDir) : tmpdir5();
16279
14337
  await mkdir5(outputRoot, { recursive: true });
16280
- const directory = await mkdtemp5(join28(outputRoot, "traces-hodoscope-"));
16281
- const input = join28(directory, "input");
16282
- const analysisFile = join28(directory, "analysis.hodoscope.json");
16283
- const samplesFile = join28(directory, "samples.json");
14338
+ const directory = await mkdtemp5(join24(outputRoot, "traces-hodoscope-"));
14339
+ const input = join24(directory, "input");
14340
+ const analysisFile = join24(directory, "analysis.hodoscope.json");
14341
+ const samplesFile = join24(directory, "samples.json");
16284
14342
  try {
16285
14343
  const prepared = await writeHodoscopeInput(otlpPath, {
16286
14344
  directory: input,
@@ -16900,7 +14958,7 @@ function assertPositiveInteger(name, value) {
16900
14958
  }
16901
14959
 
16902
14960
  // src/analyst-engine-prime.ts
16903
- import { createHash as createHash14 } from "crypto";
14961
+ import { createHash as createHash13 } from "crypto";
16904
14962
  import { request as httpRequest } from "http";
16905
14963
  import { request as httpsRequest } from "https";
16906
14964
  var DEFAULT_BRIDGE_URL = "http://localhost:4181";
@@ -17124,7 +15182,7 @@ function toAnalystFinding(row, index, spanTraces, model, producedAt) {
17124
15182
  const traces = spanTraces.get(spanId);
17125
15183
  return spanEvidenceUri([...traces][0], spanId);
17126
15184
  });
17127
- const findingId = `prime-${createHash14("sha256").update(`${index}
15185
+ const findingId = `prime-${createHash13("sha256").update(`${index}
17128
15186
  ${row.claim}
17129
15187
  ${uris.join("\n")}`).digest("hex").slice(0, 12)}`;
17130
15188
  return {
@@ -17408,10 +15466,10 @@ async function applyRedactor(spans, redactor) {
17408
15466
  }
17409
15467
 
17410
15468
  // src/file-export.ts
17411
- import { createHash as createHash15 } from "crypto";
15469
+ import { createHash as createHash14 } from "crypto";
17412
15470
  import { mkdtemp as mkdtemp6, readFile as readFile12, writeFile as writeFile8 } from "fs/promises";
17413
- import { tmpdir as tmpdir7 } from "os";
17414
- import { join as join29 } from "path";
15471
+ import { tmpdir as tmpdir6 } from "os";
15472
+ import { join as join25 } from "path";
17415
15473
  import {
17416
15474
  firstNumberAttr as firstNumberAttr2,
17417
15475
  LLM_CACHED_TOKEN_ATTR_KEYS,
@@ -17446,7 +15504,7 @@ function stableJson(value) {
17446
15504
  });
17447
15505
  }
17448
15506
  function hashId(value, chars = 16) {
17449
- return createHash15("sha256").update(stableJson(value)).digest("hex").slice(0, chars);
15507
+ return createHash14("sha256").update(stableJson(value)).digest("hex").slice(0, chars);
17450
15508
  }
17451
15509
  function isoTime2(value) {
17452
15510
  if (typeof value === "string") {
@@ -17757,6 +15815,10 @@ function llmEventUsage(row) {
17757
15815
  ...costUsd !== void 0 ? { costUsd } : {}
17758
15816
  };
17759
15817
  }
15818
+ function toolPartId(row) {
15819
+ const part = objectValue2(objectValue2(row.data)?.part) ?? objectValue2(row.part);
15820
+ return part && part.type === "tool" ? stringValue3(part.id) : void 0;
15821
+ }
17760
15822
  function sandboxEventsToSpans(rows, wrapper) {
17761
15823
  const context = wrapper ?? {};
17762
15824
  const sessionId = findStringKey(context, ["session_id", "sessionId"], 3) ?? findStringKey(rows, ["session_id", "sessionId"], 4);
@@ -17788,12 +15850,32 @@ function sandboxEventsToSpans(rows, wrapper) {
17788
15850
  }
17789
15851
  })
17790
15852
  ];
15853
+ const lastEventOfPart = /* @__PURE__ */ new Map();
15854
+ const firstTimeOfPart = /* @__PURE__ */ new Map();
15855
+ const errorOfPart = /* @__PURE__ */ new Map();
15856
+ rows.forEach((row, index) => {
15857
+ const part = toolPartId(row);
15858
+ if (part === void 0) return;
15859
+ lastEventOfPart.set(part, index);
15860
+ const time = eventTimestamp(row);
15861
+ const first = firstTimeOfPart.get(part);
15862
+ if (time && (first === void 0 || time < first)) firstTimeOfPart.set(part, time);
15863
+ const status = eventStatus(row, eventType(row) ?? "event");
15864
+ if (status.status === "ERROR") errorOfPart.set(part, status);
15865
+ });
15866
+ const mergedToolEvents = rows.filter((row, index) => {
15867
+ const part = toolPartId(row);
15868
+ return part !== void 0 && lastEventOfPart.get(part) !== index;
15869
+ }).length;
15870
+ if (mergedToolEvents > 0) spans[0].attributes["traces.event.tool_part_updates_merged"] = mergedToolEvents;
17791
15871
  let previousTime = rootStart;
17792
15872
  rows.forEach((row, index) => {
15873
+ const part = toolPartId(row);
15874
+ if (part !== void 0 && lastEventOfPart.get(part) !== index) return;
17793
15875
  const type = eventType(row) ?? "event";
17794
- const time = eventTimestamp(row) ?? previousTime;
15876
+ const time = (part !== void 0 ? firstTimeOfPart.get(part) : void 0) ?? eventTimestamp(row) ?? previousTime;
17795
15877
  previousTime = time;
17796
- const { status, message } = eventStatus(row, type);
15878
+ const { status, message } = (part !== void 0 ? errorOfPart.get(part) : void 0) ?? eventStatus(row, type);
17797
15879
  const { kind, tool } = eventKind(row, type);
17798
15880
  const name = kind === "TOOL" && tool ? `tool.${tool}` : `event.${type}`;
17799
15881
  const toolInput2 = kind === "TOOL" ? eventValue(row, ["toolInput", "tool_input", "input", "args", "arguments", "full_command"]) : void 0;
@@ -17808,6 +15890,7 @@ function sandboxEventsToSpans(rows, wrapper) {
17808
15890
  }) : {}
17809
15891
  };
17810
15892
  copyDefined(extra, {
15893
+ "traces.event.part_id": part,
17811
15894
  "traces.event.id": stringValue3(row.id),
17812
15895
  "traces.event.raw_type": stringValue3(row.type),
17813
15896
  "traces.event.raw_event": stringValue3(row.event)
@@ -18057,15 +16140,15 @@ async function exportTraceEvidenceFile(inputPath, opts = {}) {
18057
16140
  }
18058
16141
  async function writeTraceEvidenceExportFile(inputPath, outPath, opts = {}) {
18059
16142
  const result = await exportTraceEvidenceFile(inputPath, opts);
18060
- const path = outPath ?? join29(await mkdtemp6(join29(tmpdir7(), "traces-export-")), "spans.openinference.jsonl");
16143
+ const path = outPath ?? join25(await mkdtemp6(join25(tmpdir6(), "traces-export-")), "spans.openinference.jsonl");
18061
16144
  await writeFile8(path, serializeSpans(result.spans), "utf8");
18062
16145
  return { ...result, path };
18063
16146
  }
18064
16147
 
18065
16148
  // src/inspect.ts
18066
16149
  import { mkdtemp as mkdtemp7, readFile as readFile13, writeFile as writeFile9 } from "fs/promises";
18067
- import { tmpdir as tmpdir8 } from "os";
18068
- import { basename as basename15, join as join30 } from "path";
16150
+ import { tmpdir as tmpdir7 } from "os";
16151
+ import { basename as basename15, join as join26 } from "path";
18069
16152
  var severityRank2 = {
18070
16153
  high: 0,
18071
16154
  medium: 1,
@@ -18081,7 +16164,7 @@ function sessionLabel(row) {
18081
16164
  const id = row.session.sessionId.length > 12 ? row.session.sessionId.slice(0, 12) : row.session.sessionId;
18082
16165
  return `${row.session.harness}:${id} (${repoLabel(row)})`;
18083
16166
  }
18084
- function pct2(value) {
16167
+ function pct(value) {
18085
16168
  return `${(value * 100).toFixed(value >= 0.1 ? 0 : 1)}%`;
18086
16169
  }
18087
16170
  function topToolErrors(row) {
@@ -18160,7 +16243,7 @@ function addSessionFindings(index, findings) {
18160
16243
  area: "session",
18161
16244
  title: `High tool-error count/rate in ${errorSessions.length}/${index.totals.sessions} session(s)`,
18162
16245
  impact: errors,
18163
- evidence: errorSessions.slice(0, 5).map((row) => `${sessionLabel(row)}: ${row.metrics.erroredToolCallCount}/${row.metrics.toolCallCount} tool errors (${pct2(row.metrics.toolErrorRate)}); ${topToolErrors(row)}`),
16246
+ evidence: errorSessions.slice(0, 5).map((row) => `${sessionLabel(row)}: ${row.metrics.erroredToolCallCount}/${row.metrics.toolCallCount} tool errors (${pct(row.metrics.toolErrorRate)}); ${topToolErrors(row)}`),
18164
16247
  next: "Fix the most common failing command/tool first, then rerun `traces index` to confirm the error count drops.",
18165
16248
  refs: errorSessions.slice(0, 5).map((row) => row.session.path)
18166
16249
  });
@@ -18354,13 +16437,13 @@ async function readSessionIndexFile(path) {
18354
16437
  return parsed;
18355
16438
  }
18356
16439
  async function writeInspectionReportFile(report, outPath) {
18357
- const path = outPath ?? join30(await mkdtemp7(join30(tmpdir8(), "traces-inspect-")), "inspection-report.md");
16440
+ const path = outPath ?? join26(await mkdtemp7(join26(tmpdir7(), "traces-inspect-")), "inspection-report.md");
18358
16441
  await writeFile9(path, renderInspectionReport(report), "utf8");
18359
16442
  return path;
18360
16443
  }
18361
16444
 
18362
16445
  // src/live.ts
18363
- import { createHash as createHash16 } from "crypto";
16446
+ import { createHash as createHash15 } from "crypto";
18364
16447
  var VERIFY_RE = /\b(test|tests|vitest|jest|pytest|go test|cargo test|tsc|typecheck|lint|biome|eslint|build|check:invariants|preflight)\b/i;
18365
16448
  var CHANGE_RE = /\b(apply_patch|edit|write|create|delete|rename|replace|update|git apply|sed -i|perl -pi|tee\s|cat\s+>)/i;
18366
16449
  var CLAIM_DONE_RE = /\b(done|fixed|complete|completed|implemented|merged|verified|all green|tests pass|ready|works now)\b/i;
@@ -18380,7 +16463,7 @@ var sleep = (ms2, signal) => new Promise((resolve13) => {
18380
16463
  signal?.addEventListener("abort", onAbort, { once: true });
18381
16464
  });
18382
16465
  function hash(parts) {
18383
- return createHash16("sha256").update(parts.join("\0")).digest("hex").slice(0, 16);
16466
+ return createHash15("sha256").update(parts.join("\0")).digest("hex").slice(0, 16);
18384
16467
  }
18385
16468
  function asString(value) {
18386
16469
  if (typeof value === "string" && value.trim()) return value;
@@ -18788,7 +16871,7 @@ async function streamSessions(opts) {
18788
16871
 
18789
16872
  // src/supervisor-run-context.ts
18790
16873
  import { readFile as readFile14, stat as stat16 } from "fs/promises";
18791
- import { basename as basename16, join as join31 } from "path";
16874
+ import { basename as basename16, join as join27 } from "path";
18792
16875
  import { materializeTreeView } from "@tangle-network/agent-runtime/kernel";
18793
16876
  import {
18794
16877
  readRuntimeSupervisorRun
@@ -18950,14 +17033,14 @@ function parseInstant(v) {
18950
17033
  }
18951
17034
  async function readFileRunContext(runDir, opts = {}) {
18952
17035
  const sources = await readRuntimeSupervisorRun(runDir);
18953
- const coordination = readCoordinationLog(await readMaybe(join31(runDir, COORDINATION_LOG_FILE)));
17036
+ const coordination = readCoordinationLog(await readMaybe(join27(runDir, COORDINATION_LOG_FILE)));
18954
17037
  return {
18955
17038
  ...sources,
18956
17039
  instanceId: opts.instanceId !== void 0 ? opts.instanceId : sources.instanceId,
18957
17040
  arm: opts.arm !== void 0 ? opts.arm : basename16(runDir),
18958
17041
  workers: sources.workers !== null && coordination.present ? workerSources(sources.workers, coordination) : sources.workers,
18959
17042
  ...sources.journal !== null ? {
18960
- rootTranscriptRef: join31(runDir, SPAWN_JOURNAL_FILE),
17043
+ rootTranscriptRef: join27(runDir, SPAWN_JOURNAL_FILE),
18961
17044
  traceCommand: `traces watch ${runDir}`
18962
17045
  } : {}
18963
17046
  };
@@ -18966,7 +17049,7 @@ function fileRunContextSupervisorRunReader(runDir, opts = {}) {
18966
17049
  return { runRef: runDir, read: () => readFileRunContext(runDir, opts) };
18967
17050
  }
18968
17051
  async function isFileRunContextDir(runDir) {
18969
- return stat16(join31(runDir, SPAWN_JOURNAL_FILE)).then((s) => s.isFile()).catch(() => false);
17052
+ return stat16(join27(runDir, SPAWN_JOURNAL_FILE)).then((s) => s.isFile()).catch(() => false);
18970
17053
  }
18971
17054
  function parseJsonObject(text2) {
18972
17055
  if (text2 === null) return null;
@@ -19032,9 +17115,9 @@ function mergeRolls(a, b) {
19032
17115
  };
19033
17116
  }
19034
17117
  async function readRunContextSnapshot(runDir) {
19035
- const journal = readRunContextJournal(await readMaybe(join31(runDir, SPAWN_JOURNAL_FILE)));
19036
- const coordination = readCoordinationLog(await readMaybe(join31(runDir, COORDINATION_LOG_FILE)));
19037
- const resultText = await readMaybe(join31(runDir, RESULT_FILE));
17118
+ const journal = readRunContextJournal(await readMaybe(join27(runDir, SPAWN_JOURNAL_FILE)));
17119
+ const coordination = readCoordinationLog(await readMaybe(join27(runDir, COORDINATION_LOG_FILE)));
17120
+ const resultText = await readMaybe(join27(runDir, RESULT_FILE));
19038
17121
  const result = parseResult(resultText);
19039
17122
  const view = safeTreeView(journal.events);
19040
17123
  const nodes = view === null ? [] : orderNodes(view, journal);
@@ -19217,7 +17300,7 @@ function wrap(text2, indent) {
19217
17300
 
19218
17301
  // src/run-span-tree.ts
19219
17302
  import { readdir as readdir18 } from "fs/promises";
19220
- import { join as join32 } from "path";
17303
+ import { join as join28 } from "path";
19221
17304
  import {
19222
17305
  firstNumberAttr as firstNumberAttr3,
19223
17306
  LLM_COST_ATTR_KEYS as LLM_COST_ATTR_KEYS2,
@@ -19492,7 +17575,7 @@ async function findSpanSourceFiles(dir) {
19492
17575
  const found = [];
19493
17576
  const entries = await readdir18(dir, { withFileTypes: true }).catch(() => []);
19494
17577
  for (const entry of entries) {
19495
- const child = join32(dir, entry.name);
17578
+ const child = join28(dir, entry.name);
19496
17579
  if (entry.isFile()) {
19497
17580
  if (SPAN_FILE_PATTERN.test(entry.name)) found.push(child);
19498
17581
  continue;
@@ -19500,7 +17583,7 @@ async function findSpanSourceFiles(dir) {
19500
17583
  if (!entry.isDirectory()) continue;
19501
17584
  const nested = await readdir18(child, { withFileTypes: true }).catch(() => []);
19502
17585
  for (const inner of nested) {
19503
- if (inner.isFile() && SPAN_FILE_PATTERN.test(inner.name)) found.push(join32(child, inner.name));
17586
+ if (inner.isFile() && SPAN_FILE_PATTERN.test(inner.name)) found.push(join28(child, inner.name));
19504
17587
  }
19505
17588
  }
19506
17589
  return found.sort();
@@ -19508,7 +17591,7 @@ async function findSpanSourceFiles(dir) {
19508
17591
 
19509
17592
  // src/supervisor-run-watch.ts
19510
17593
  import { readdir as readdir19 } from "fs/promises";
19511
- import { join as join33 } from "path";
17594
+ import { join as join29 } from "path";
19512
17595
  function budget(node) {
19513
17596
  const parts = [`${int(node.budget.maxIterations)}it`, `${int(node.budget.maxTokens)}tok`];
19514
17597
  if (node.budget.maxUsd !== void 0) parts.push(`$${node.budget.maxUsd.toFixed(2)}`);
@@ -19652,7 +17735,7 @@ async function resolveRunContextDir(dir) {
19652
17735
  const candidates = [];
19653
17736
  for (const entry of entries) {
19654
17737
  if (!entry.isDirectory()) continue;
19655
- const child = join33(dir, entry.name);
17738
+ const child = join29(dir, entry.name);
19656
17739
  if (await isFileRunContextDir(child)) candidates.push(child);
19657
17740
  }
19658
17741
  if (candidates.length === 1) return candidates[0];
@@ -19988,7 +18071,7 @@ function parseJsonAnswer(text2) {
19988
18071
 
19989
18072
  // src/ask.ts
19990
18073
  import { mkdir as mkdir6, readFile as readFile15, writeFile as writeFile10 } from "fs/promises";
19991
- import { join as join34, resolve as resolve11 } from "path";
18074
+ import { join as join30, resolve as resolve11 } from "path";
19992
18075
  import { performance } from "perf_hooks";
19993
18076
  import { CostCeilingReachedError, CostLedger } from "@tangle-network/agent-eval";
19994
18077
  import {
@@ -20543,8 +18626,8 @@ async function writeTraceQuestionsArtifacts(result, outDir) {
20543
18626
  await mkdir6(directory, { recursive: true });
20544
18627
  const paths = {
20545
18628
  directory,
20546
- result: join34(directory, "answers.json"),
20547
- report: join34(directory, "report.md"),
18629
+ result: join30(directory, "answers.json"),
18630
+ report: join30(directory, "report.md"),
20548
18631
  traces: result.otlpPath
20549
18632
  };
20550
18633
  const { report, ...machine } = result;
@@ -20606,9 +18689,9 @@ async function watchSessions(opts) {
20606
18689
  }
20607
18690
 
20608
18691
  // src/bundle-view.ts
20609
- import { createHash as createHash17 } from "crypto";
18692
+ import { createHash as createHash16 } from "crypto";
20610
18693
  import { mkdir as mkdir7, readdir as readdir20, readFile as readFile16, writeFile as writeFile11 } from "fs/promises";
20611
- import { dirname as dirname11, join as join35, resolve as resolve12 } from "path";
18694
+ import { dirname as dirname11, join as join31, resolve as resolve12 } from "path";
20612
18695
  var EVIDENCE_ONLY_ALLOW_LIST = [
20613
18696
  {
20614
18697
  path: "derived/session-index.json",
@@ -20732,7 +18815,7 @@ async function findSessionTextLeaks(opts) {
20732
18815
  const index = /* @__PURE__ */ new Map();
20733
18816
  const carried = [];
20734
18817
  for (const path of opts.carriedPaths) {
20735
- const signatures = sessionTextSignatures(await readFile16(join35(opts.viewDir, path), "utf8"));
18818
+ const signatures = sessionTextSignatures(await readFile16(join31(opts.viewDir, path), "utf8"));
20736
18819
  carried.push({ path, signatures: signatures.size });
20737
18820
  for (const signature of signatures) if (!index.has(signature)) index.set(signature, path);
20738
18821
  }
@@ -20740,7 +18823,7 @@ async function findSessionTextLeaks(opts) {
20740
18823
  const comparedSources = [];
20741
18824
  let sourceSignatures = 0;
20742
18825
  for (const sourcePath of opts.excludedPaths) {
20743
- const signatures = sessionTextSignatures(await readFile16(join35(opts.sourceDir, sourcePath), "utf8"));
18826
+ const signatures = sessionTextSignatures(await readFile16(join31(opts.sourceDir, sourcePath), "utf8"));
20744
18827
  comparedSources.push(sourcePath);
20745
18828
  sourceSignatures += signatures.size;
20746
18829
  for (const signature of signatures) {
@@ -20799,17 +18882,17 @@ async function projectSessionBundle(opts) {
20799
18882
  const candidates = [];
20800
18883
  for (const path of await listSessionBundleFiles(sourceDir)) {
20801
18884
  if (path === "manifest.json") continue;
20802
- const bytes = await readFile16(join35(sourceDir, path));
20803
- const sha2562 = createHash17("sha256").update(bytes).digest("hex");
18885
+ const bytes = await readFile16(join31(sourceDir, path));
18886
+ const sha2562 = createHash16("sha256").update(bytes).digest("hex");
20804
18887
  const recorded = sourceHashes.get(path);
20805
18888
  if (!recorded) {
20806
18889
  throw new Error(
20807
- `${join35(sourceDir, path)} is not listed in the bundle manifest \u2014 the bundle changed after assembly and cannot be projected; re-run \`traces bundle\``
18890
+ `${join31(sourceDir, path)} is not listed in the bundle manifest \u2014 the bundle changed after assembly and cannot be projected; re-run \`traces bundle\``
20808
18891
  );
20809
18892
  }
20810
18893
  if (recorded.sha256 !== sha2562) {
20811
18894
  throw new Error(
20812
- `${join35(sourceDir, path)} does not match its manifest hash (${recorded.sha256} recorded, ${sha2562} on disk) \u2014 the bundle changed after assembly and cannot be projected; re-run \`traces bundle\``
18895
+ `${join31(sourceDir, path)} does not match its manifest hash (${recorded.sha256} recorded, ${sha2562} on disk) \u2014 the bundle changed after assembly and cannot be projected; re-run \`traces bundle\``
20813
18896
  );
20814
18897
  }
20815
18898
  const verdict = classifyEvidenceOnlyPath(path);
@@ -20845,7 +18928,7 @@ async function projectSessionBundle(opts) {
20845
18928
  });
20846
18929
  continue;
20847
18930
  }
20848
- const target = join35(outDir, candidate.path);
18931
+ const target = join31(outDir, candidate.path);
20849
18932
  await mkdir7(dirname11(target), { recursive: true });
20850
18933
  await writeFile11(target, candidate.bytes);
20851
18934
  carriedPaths.push(candidate.path);
@@ -20862,7 +18945,7 @@ async function projectSessionBundle(opts) {
20862
18945
  })),
20863
18946
  matches: 0
20864
18947
  };
20865
- const sourceManifestBytes = await readFile16(join35(sourceDir, "manifest.json"));
18948
+ const sourceManifestBytes = await readFile16(join31(sourceDir, "manifest.json"));
20866
18949
  const manifest = {
20867
18950
  schemaVersion: 2,
20868
18951
  kind: "traces.session_bundle",
@@ -20876,7 +18959,7 @@ async function projectSessionBundle(opts) {
20876
18959
  knownLimits: [...source.knownLimits, ...EVIDENCE_ONLY_LIMITS],
20877
18960
  projection: {
20878
18961
  sourceDirectory: sourceDir,
20879
- sourceManifestSha256: createHash17("sha256").update(sourceManifestBytes).digest("hex"),
18962
+ sourceManifestSha256: createHash16("sha256").update(sourceManifestBytes).digest("hex"),
20880
18963
  leakCheck
20881
18964
  }
20882
18965
  };
@@ -20887,11 +18970,11 @@ async function projectSessionBundle(opts) {
20887
18970
  // src/upload-state.ts
20888
18971
  import { mkdir as mkdir8, readFile as readFile17, writeFile as writeFile12 } from "fs/promises";
20889
18972
  import { homedir as homedir12 } from "os";
20890
- import { dirname as dirname12, join as join36 } from "path";
18973
+ import { dirname as dirname12, join as join32 } from "path";
20891
18974
  import { contentHash } from "@tangle-network/agent-eval";
20892
18975
  function stateFile() {
20893
- const base = process.env.XDG_STATE_HOME ?? join36(homedir12(), ".local", "state");
20894
- return join36(base, "tangle-traces", "uploaded.json");
18976
+ const base = process.env.XDG_STATE_HOME ?? join32(homedir12(), ".local", "state");
18977
+ return join32(base, "tangle-traces", "uploaded.json");
20895
18978
  }
20896
18979
  function uploadKey(harness, sessionId) {
20897
18980
  return `${harness}:${sessionId}`;
@@ -20923,7 +19006,7 @@ function alreadyUploaded(state, harness, sessionId, hash2) {
20923
19006
  import { readFile as readFile18 } from "fs/promises";
20924
19007
  import { createRequire as createRequire2 } from "module";
20925
19008
  import { hostname as hostname2 } from "os";
20926
- import { basename as basename17, join as join37 } from "path";
19009
+ import { basename as basename17, join as join33 } from "path";
20927
19010
  import { hostedClientFromEnv } from "@tangle-network/agent-eval/hosted";
20928
19011
  import { REDACTION_VERSION } from "@tangle-network/agent-eval/traces";
20929
19012
  var require2 = createRequire2(import.meta.url);
@@ -20947,7 +19030,7 @@ async function planUpload(opts = {}) {
20947
19030
  async function gitBranch(cwd) {
20948
19031
  if (!cwd) return void 0;
20949
19032
  try {
20950
- const head = await readFile18(join37(cwd, ".git", "HEAD"), "utf8");
19033
+ const head = await readFile18(join33(cwd, ".git", "HEAD"), "utf8");
20951
19034
  const m = head.match(/^ref:\s*refs\/heads\/(.+)$/m);
20952
19035
  return m?.[1]?.trim();
20953
19036
  } catch {
@@ -21275,60 +19358,20 @@ export {
21275
19358
  hashSessionBundleFiles,
21276
19359
  writeSessionBundleManifest,
21277
19360
  readSessionBundleManifest,
19361
+ verifySessionBundle,
21278
19362
  createBundleSourceReader,
21279
19363
  assertOutsideSourceBundle,
21280
19364
  isChatTrajectoryMessage,
21281
19365
  isChatTrajectory,
21282
19366
  chatTrajectoryToSpans,
21283
19367
  importCodeTraceBench,
21284
- parseRecordedReturncode,
21285
- parseObservationOutput,
21286
- deriveFailureSignature,
21287
- ingestCodeTraceBenchSteps,
19368
+ DEFAULT_SANDBOX_BASE_URL,
21288
19369
  sandboxReplayBackend,
21289
- wrapActionForExec,
21290
- SandboxCounterfactualRunner,
21291
- replayVerify,
21292
- parseReplayVerifyArgs,
21293
- replayVerifyUsage,
21294
- cmdReplayVerify,
21295
- parseCorpusFlag,
21296
- SUBMIT_ACTION_SIGNATURE,
21297
- isSubmitAction,
21298
- readLabelEntries,
21299
- goldIncorrectSteps,
21300
- resolveCaseResources,
21301
- enumerateReplayableCases,
19370
+ assertSandboxReachable,
21302
19371
  zaiChatCaller,
21303
- clipText,
21304
- buildFixPrompt,
21305
- buildRetryFixPrompt,
21306
- countScriptCommands,
21307
- extractFixCommand,
21308
- generateFixCommand,
21309
- runFixLoop,
21310
- derivedImageTag,
21311
- dockerImagePreparer,
21312
- seededSample,
21313
- runReplayBatch,
21314
- renderBatchReport,
21315
- parseReplayBatchArgs,
21316
- replayBatchUsage,
19372
+ cmdReplayVerify,
21317
19373
  cmdReplayVerifyBatch,
21318
- parseIncorrectStepsSubject,
21319
- resolveFindingInvocation,
21320
- replayVerifyFinding,
21321
- findingReplayStep,
21322
- findingTrajectoryId,
21323
- resolveFindingReplayability,
21324
- classifyVerdict,
21325
- DEFAULT_SANDBOX_BASE_URL,
21326
- assertSandboxReachable,
21327
- verifyFindings,
21328
- renderVerifiedFindingsSection,
21329
- readFindingsFile,
21330
- verifyFindingsUsage,
21331
- parseVerifyFindingsArgs,
19374
+ verifyFindingsInSandbox,
21332
19375
  cmdVerifyFindings,
21333
19376
  writeHodoscopeInput,
21334
19377
  hodoscopeAnalyzer,