@tangle-network/traces 0.11.3 → 0.11.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -766,8 +766,8 @@ function serializeSpans(spans) {
766
766
  async function writeOtlpFile(spans, outPath) {
767
767
  const { mkdtemp: mkdtemp8, writeFile: writeFile10 } = await import("fs/promises");
768
768
  const { tmpdir: tmpdir9 } = await import("os");
769
- const { join: join33 } = await import("path");
770
- const path = outPath ?? join33(await mkdtemp8(join33(tmpdir9(), "traces-")), "spans.otlp.jsonl");
769
+ const { join: join34 } = await import("path");
770
+ const path = outPath ?? join34(await mkdtemp8(join34(tmpdir9(), "traces-")), "spans.otlp.jsonl");
771
771
  await writeFile10(path, serializeSpans(spans), "utf8");
772
772
  return path;
773
773
  }
@@ -5503,7 +5503,7 @@ var OpencodeAdapter = class {
5503
5503
  // src/adapters/pi.ts
5504
5504
  import { readdir as readdir12, stat as stat11 } from "fs/promises";
5505
5505
  import { homedir as homedir10 } from "os";
5506
- import { basename as basename11, join as join15 } from "path";
5506
+ import { basename as basename11, join as join15, resolve as resolve5 } from "path";
5507
5507
  var SERVICE9 = "pi";
5508
5508
  function terminalStatus(message) {
5509
5509
  const stopReason = message?.stopReason ?? "missing";
@@ -5548,45 +5548,77 @@ function completeToolSpan(toolSpan, result) {
5548
5548
  }
5549
5549
  var PiAdapter = class {
5550
5550
  harness = "pi";
5551
+ sessionsRoot;
5552
+ constructor(options = {}) {
5553
+ this.sessionsRoot = resolve5(options.sessionsRoot ?? join15(homedir10(), ".pi", "agent", "sessions"));
5554
+ }
5551
5555
  root() {
5552
- return join15(homedir10(), ".pi", "agent", "sessions");
5556
+ return this.sessionsRoot;
5553
5557
  }
5554
5558
  async locate(opts = {}) {
5555
5559
  const root = this.root();
5556
- let dirs;
5560
+ let entries;
5557
5561
  try {
5558
- dirs = await readdir12(root);
5562
+ entries = await readdir12(root, { withFileTypes: true });
5559
5563
  } catch (error) {
5560
5564
  if (isMissingPathError(error)) return [];
5561
5565
  throw error;
5562
5566
  }
5563
- const refs = [];
5564
- for (const dir of dirs) {
5565
- const dp = join15(root, dir);
5567
+ const paths = [];
5568
+ for (const entry of entries) {
5569
+ if (entry.isFile() && entry.name.endsWith(".jsonl")) {
5570
+ paths.push(join15(root, entry.name));
5571
+ continue;
5572
+ }
5573
+ if (!entry.isDirectory()) continue;
5574
+ const dir = join15(root, entry.name);
5566
5575
  let files;
5567
5576
  try {
5568
- files = await readdir12(dp);
5577
+ files = await readdir12(dir);
5569
5578
  } catch (error) {
5570
5579
  if (isMissingPathError(error)) continue;
5571
5580
  throw error;
5572
5581
  }
5573
- const cwd = `/${dir.replace(/^-+/, "").replace(/-+$/, "").replace(/-/g, "/")}`;
5574
- if (opts.cwd && !cwd.startsWith(opts.cwd)) continue;
5575
- for (const f of files) {
5576
- if (!f.endsWith(".jsonl")) continue;
5577
- const path = join15(dp, f);
5578
- let st;
5579
- try {
5580
- st = await stat11(path);
5581
- } catch (error) {
5582
- if (isMissingPathError(error)) continue;
5583
- throw error;
5584
- }
5585
- if (opts.sinceMs && st.mtimeMs < opts.sinceMs) continue;
5586
- const id = basename11(f, ".jsonl").replace(/^[\dTZ.-]+_/, "");
5587
- refs.push({ harness: this.harness, sessionId: id, path, cwd, mtimeMs: st.mtimeMs });
5582
+ for (const file of files) {
5583
+ if (file.endsWith(".jsonl")) paths.push(join15(dir, file));
5588
5584
  }
5589
5585
  }
5586
+ const refs = [];
5587
+ for (const path of paths) {
5588
+ const f = basename11(path);
5589
+ let st;
5590
+ try {
5591
+ st = await stat11(path);
5592
+ } catch (error) {
5593
+ if (isMissingPathError(error)) continue;
5594
+ throw error;
5595
+ }
5596
+ if (opts.sinceMs && st.mtimeMs < opts.sinceMs) continue;
5597
+ let id = basename11(f, ".jsonl").replace(/^[\dTZ.-]+_/, "");
5598
+ let cwd = null;
5599
+ const ref = {
5600
+ harness: this.harness,
5601
+ sessionId: id,
5602
+ path,
5603
+ cwd,
5604
+ mtimeMs: st.mtimeMs
5605
+ };
5606
+ const [session] = await takeJsonl(path, 1, sessionJsonlOptions(ref));
5607
+ if (session?.type === "session") {
5608
+ if (session.id) id = session.id;
5609
+ if (session.cwd) cwd = session.cwd;
5610
+ }
5611
+ ref.sessionId = id;
5612
+ ref.cwd = cwd;
5613
+ if (ref.integrity) {
5614
+ ref.integrity.corruptions = ref.integrity.corruptions.map((receipt) => ({
5615
+ ...receipt,
5616
+ sessionId: id
5617
+ }));
5618
+ }
5619
+ if (opts.cwd && (!cwd || !cwd.startsWith(opts.cwd))) continue;
5620
+ refs.push(ref);
5621
+ }
5590
5622
  return refs.sort((a, b) => b.mtimeMs - a.mtimeMs);
5591
5623
  }
5592
5624
  async parse(ref, options = {}) {
@@ -6332,7 +6364,7 @@ async function stampSpanWorkdirRepoAttrs(spans) {
6332
6364
  async function awaitWithAbort(operation, signal) {
6333
6365
  if (!signal) return operation;
6334
6366
  signal.throwIfAborted();
6335
- return new Promise((resolve7, reject) => {
6367
+ return new Promise((resolve8, reject) => {
6336
6368
  const stop = () => {
6337
6369
  signal.removeEventListener("abort", stop);
6338
6370
  reject(signal.reason);
@@ -6341,7 +6373,7 @@ async function awaitWithAbort(operation, signal) {
6341
6373
  operation.then(
6342
6374
  (value) => {
6343
6375
  signal.removeEventListener("abort", stop);
6344
- resolve7(value);
6376
+ resolve8(value);
6345
6377
  },
6346
6378
  (error) => {
6347
6379
  signal.removeEventListener("abort", stop);
@@ -7387,7 +7419,7 @@ function zaiChatCaller(options) {
7387
7419
  }
7388
7420
  };
7389
7421
  }
7390
- function clip(text2, limit) {
7422
+ function clipText(text2, limit) {
7391
7423
  if (text2.length <= limit) return text2;
7392
7424
  const half = Math.floor(limit / 2);
7393
7425
  return `${text2.slice(0, half)}
@@ -7396,11 +7428,11 @@ ${text2.slice(-half)}`;
7396
7428
  }
7397
7429
  function renderStep(step, marker) {
7398
7430
  const rc = parseRecordedReturncode(step.observation);
7399
- const output = clip(parseObservationOutput(step.observation).trim(), 1600);
7431
+ const output = clipText(parseObservationOutput(step.observation).trim(), 1600);
7400
7432
  return [
7401
7433
  `### step ${step.step_id}${marker}`,
7402
7434
  "```sh",
7403
- clip(step.action, 2e3),
7435
+ clipText(step.action, 2e3),
7404
7436
  "```",
7405
7437
  `returncode: ${rc ?? "none recorded"}`,
7406
7438
  output.length > 0 ? `output:
@@ -7409,13 +7441,25 @@ ${output}
7409
7441
  \`\`\`` : "output: (empty)"
7410
7442
  ].join("\n");
7411
7443
  }
7412
- function buildFixPrompt(input) {
7444
+ function promptBody(input) {
7413
7445
  const radius = input.contextRadius ?? 3;
7414
7446
  const target = input.steps.find((s) => s.step_id === input.k);
7415
7447
  if (!target) throw new Error(`replay-fix: no step with step_id ${input.k}`);
7416
7448
  const context = input.steps.filter(
7417
7449
  (s) => s.step_id !== input.k && Math.abs(s.step_id - input.k) <= radius
7418
7450
  );
7451
+ return [
7452
+ "## Task the agent was solving",
7453
+ input.taskStatement ? clipText(input.taskStatement, 4e3) : "(no task statement recorded)",
7454
+ "",
7455
+ "## Surrounding steps",
7456
+ ...context.map((s) => renderStep(s, "")),
7457
+ "",
7458
+ "## Failing step to correct",
7459
+ renderStep(target, " (INCORRECT \u2014 correct this one)")
7460
+ ];
7461
+ }
7462
+ function buildFixPrompt(input) {
7419
7463
  const system = [
7420
7464
  "You repair one failed shell command from a recorded coding-agent trajectory.",
7421
7465
  "The trajectory replays inside the original docker image; every command runs as a fresh /bin/sh subshell from a fixed working directory.",
@@ -7424,19 +7468,62 @@ function buildFixPrompt(input) {
7424
7468
  "The corrected command must accomplish the failing step's intent and exit 0. No prose outside the fenced block."
7425
7469
  ].join("\n");
7426
7470
  const user = [
7427
- "## Task the agent was solving",
7428
- input.taskStatement ? clip(input.taskStatement, 4e3) : "(no task statement recorded)",
7471
+ ...promptBody(input),
7429
7472
  "",
7430
- "## Surrounding steps",
7431
- ...context.map((s) => renderStep(s, "")),
7473
+ "Output the single corrected replacement for the failing step now."
7474
+ ].join("\n");
7475
+ return { system, user };
7476
+ }
7477
+ function renderFailedAttempt(prior) {
7478
+ if (prior.command === null) {
7479
+ return [
7480
+ `### attempt ${prior.attempt}`,
7481
+ `model call failed before producing a command: ${prior.llmError ?? "unknown error"}`
7482
+ ].join("\n");
7483
+ }
7484
+ const stdout = (prior.stdoutTail ?? "").trim();
7485
+ const stderr = (prior.stderrTail ?? "").trim();
7486
+ return [
7487
+ `### attempt ${prior.attempt}`,
7488
+ "```sh",
7489
+ clipText(prior.command, 2e3),
7490
+ "```",
7491
+ `exit code: ${prior.exitCode ?? "not executed"}`,
7492
+ stdout.length > 0 ? `stdout:
7493
+ \`\`\`
7494
+ ${stdout}
7495
+ \`\`\`` : "stdout: (empty)",
7496
+ stderr.length > 0 ? `stderr:
7497
+ \`\`\`
7498
+ ${stderr}
7499
+ \`\`\`` : "stderr: (empty)"
7500
+ ].join("\n");
7501
+ }
7502
+ function buildRetryFixPrompt(input, priorAttempts, maxScriptCommands = 5) {
7503
+ if (priorAttempts.length === 0) {
7504
+ throw new Error("replay-fix: buildRetryFixPrompt requires at least one prior attempt");
7505
+ }
7506
+ const system = [
7507
+ "You repair one failed shell command from a recorded coding-agent trajectory.",
7508
+ "The trajectory replays inside the original docker image; every command runs as a fresh /bin/sh subshell from a fixed working directory.",
7509
+ "Earlier corrected commands were executed for real and failed; their actual output is included below.",
7510
+ `Reply with a corrected fix inside a single \`\`\`sh fenced block: either one command, or a short script of at most ${maxScriptCommands} commands (one per line).`,
7511
+ "The whole block executes as ONE /bin/sh unit from the fixed working directory and must exit 0.",
7512
+ "Do not repeat a command that already failed. Keep reasoning brief. No prose outside the fenced block."
7513
+ ].join("\n");
7514
+ const user = [
7515
+ ...promptBody(input),
7432
7516
  "",
7433
- "## Failing step to correct",
7434
- renderStep(target, " (INCORRECT \u2014 correct this one)"),
7517
+ "## Previous fix attempts (executed for real \u2014 all failed)",
7518
+ ...priorAttempts.map((prior) => renderFailedAttempt(prior)),
7435
7519
  "",
7436
- "Output the single corrected replacement for the failing step now."
7520
+ `Output a corrected fix now \u2014 one \`\`\`sh block, at most ${maxScriptCommands} commands.`
7437
7521
  ].join("\n");
7438
7522
  return { system, user };
7439
7523
  }
7524
+ function countScriptCommands(script) {
7525
+ return script.split("\n").filter((line) => line.trim().length > 0 && !line.trim().startsWith("#")).length;
7526
+ }
7440
7527
  function extractFixCommand(content) {
7441
7528
  const blocks = [...content.matchAll(/```(?:sh|bash|shell)?\n([\s\S]*?)```/g)];
7442
7529
  const candidate = blocks.length > 0 ? blocks[blocks.length - 1][1] : content;
@@ -7452,12 +7539,167 @@ async function generateFixCommand(caller, input) {
7452
7539
  if (command === null) {
7453
7540
  return {
7454
7541
  succeeded: false,
7455
- error: `completion carried no usable command: ${clip(outcome.value.content, 300)}`
7542
+ error: `completion carried no usable command: ${clipText(outcome.value.content, 300)}`
7456
7543
  };
7457
7544
  }
7458
7545
  return { succeeded: true, value: { command, usage: outcome.value.usage } };
7459
7546
  }
7460
7547
 
7548
+ // src/replay-fix-loop.ts
7549
+ function toFailedAttempt(record2) {
7550
+ return {
7551
+ attempt: record2.attempt,
7552
+ command: record2.command,
7553
+ exitCode: record2.exitCode,
7554
+ stdoutTail: record2.stdoutTail,
7555
+ stderrTail: record2.stderrTail,
7556
+ llmError: record2.llmError ?? record2.armBError
7557
+ };
7558
+ }
7559
+ async function runFixLoop(caller, input, executor, options) {
7560
+ if (!Number.isInteger(options.maxAttempts) || options.maxAttempts < 1) {
7561
+ throw new Error(
7562
+ `replay-fix-loop: maxAttempts must be a positive integer, got ${options.maxAttempts}`
7563
+ );
7564
+ }
7565
+ const scriptCap = options.maxScriptCommands ?? 5;
7566
+ const tailChars = options.outputTailChars ?? 1600;
7567
+ const attempts = [];
7568
+ let llmCalls = 0;
7569
+ let llmFailures = 0;
7570
+ let promptTokens = 0;
7571
+ let completionTokens = 0;
7572
+ const unexecuted = (attempt, llmError, usage, command, started) => ({
7573
+ attempt,
7574
+ command,
7575
+ llmError,
7576
+ usage,
7577
+ executed: false,
7578
+ exitCode: null,
7579
+ prefixExecuted: null,
7580
+ prefixDivergences: null,
7581
+ failureVanished: null,
7582
+ stdoutTail: null,
7583
+ stderrTail: null,
7584
+ armBError: null,
7585
+ wallMs: Date.now() - started
7586
+ });
7587
+ for (let attempt = 1; attempt <= options.maxAttempts; attempt++) {
7588
+ const started = Date.now();
7589
+ const prompt = attempt === 1 ? buildFixPrompt(input) : buildRetryFixPrompt(input, attempts.map(toFailedAttempt), scriptCap);
7590
+ llmCalls += 1;
7591
+ const outcome = await caller.complete(prompt.system, prompt.user);
7592
+ if (!outcome.succeeded) {
7593
+ llmFailures += 1;
7594
+ attempts.push(unexecuted(attempt, outcome.error, null, null, started));
7595
+ options.onProgress?.(
7596
+ `fix-loop attempt ${attempt}: LLM failed \u2014 ${outcome.error.slice(0, 160)}`
7597
+ );
7598
+ continue;
7599
+ }
7600
+ const usage = outcome.value.usage;
7601
+ promptTokens += usage?.promptTokens ?? 0;
7602
+ completionTokens += usage?.completionTokens ?? 0;
7603
+ const command = extractFixCommand(outcome.value.content);
7604
+ if (command === null) {
7605
+ llmFailures += 1;
7606
+ attempts.push(
7607
+ unexecuted(
7608
+ attempt,
7609
+ `completion carried no usable command: ${clipText(outcome.value.content, 300)}`,
7610
+ usage,
7611
+ null,
7612
+ started
7613
+ )
7614
+ );
7615
+ continue;
7616
+ }
7617
+ if (attempt > 1) {
7618
+ const commandLines = countScriptCommands(command);
7619
+ if (commandLines > scriptCap) {
7620
+ llmFailures += 1;
7621
+ attempts.push(
7622
+ unexecuted(
7623
+ attempt,
7624
+ `script exceeds ${scriptCap} command lines (${commandLines})`,
7625
+ usage,
7626
+ null,
7627
+ started
7628
+ )
7629
+ );
7630
+ options.onProgress?.(
7631
+ `fix-loop attempt ${attempt}: rejected script with ${commandLines} command lines`
7632
+ );
7633
+ continue;
7634
+ }
7635
+ }
7636
+ let execution;
7637
+ try {
7638
+ execution = await executor(command, attempt);
7639
+ } catch (err) {
7640
+ const message = err instanceof Error ? err.message : String(err);
7641
+ attempts.push({
7642
+ ...unexecuted(attempt, "", usage, command, started),
7643
+ llmError: null,
7644
+ armBError: message.slice(0, 500)
7645
+ });
7646
+ options.onProgress?.(
7647
+ `fix-loop attempt ${attempt}: sandbox error \u2014 ${message.slice(0, 160)}`
7648
+ );
7649
+ return {
7650
+ flipped: false,
7651
+ flippedAtAttempt: null,
7652
+ attempts,
7653
+ aborted: true,
7654
+ llmCalls,
7655
+ llmFailures,
7656
+ promptTokens,
7657
+ completionTokens
7658
+ };
7659
+ }
7660
+ attempts.push({
7661
+ attempt,
7662
+ command,
7663
+ llmError: null,
7664
+ usage,
7665
+ executed: true,
7666
+ exitCode: execution.exitCode,
7667
+ prefixExecuted: execution.prefixExecuted,
7668
+ prefixDivergences: execution.prefixDivergences,
7669
+ failureVanished: execution.failureVanished,
7670
+ stdoutTail: clipText(execution.stdout, tailChars),
7671
+ stderrTail: clipText(execution.stderr, tailChars),
7672
+ armBError: null,
7673
+ wallMs: Date.now() - started
7674
+ });
7675
+ options.onProgress?.(
7676
+ `fix-loop attempt ${attempt}: exit=${execution.exitCode} failureVanished=${execution.failureVanished}`
7677
+ );
7678
+ if (execution.failureVanished) {
7679
+ return {
7680
+ flipped: true,
7681
+ flippedAtAttempt: attempt,
7682
+ attempts,
7683
+ aborted: false,
7684
+ llmCalls,
7685
+ llmFailures,
7686
+ promptTokens,
7687
+ completionTokens
7688
+ };
7689
+ }
7690
+ }
7691
+ return {
7692
+ flipped: false,
7693
+ flippedAtAttempt: null,
7694
+ attempts,
7695
+ aborted: false,
7696
+ llmCalls,
7697
+ llmFailures,
7698
+ promptTokens,
7699
+ completionTokens
7700
+ };
7701
+ }
7702
+
7461
7703
  // src/replay-batch.ts
7462
7704
  import { execFile } from "child_process";
7463
7705
  import { createHash as createHash8 } from "crypto";
@@ -7746,9 +7988,10 @@ async function runReplayBatch(options) {
7746
7988
  }
7747
7989
  }
7748
7990
  let llm = null;
7749
- if (options.fix === "generate") {
7991
+ if (options.fix !== "none") {
7750
7992
  const caller = options.fixCaller;
7751
- if (!caller) throw new Error("replay-verify-batch: fix=generate requires a fixCaller");
7993
+ if (!caller) throw new Error(`replay-verify-batch: fix=${options.fix} requires a fixCaller`);
7994
+ const fixAttempts = options.fixAttempts ?? 3;
7752
7995
  const maxFixCases = options.maxFixCases ?? 30;
7753
7996
  const seed = options.seed ?? 17;
7754
7997
  const eligible = rows.filter((r) => r.status === "ok" && r.replayed);
@@ -7777,20 +8020,115 @@ async function runReplayBatch(options) {
7777
8020
  armBPrefixExecuted: null,
7778
8021
  armBPrefixDivergences: null,
7779
8022
  failureVanished: null,
7780
- armBError: null
8023
+ armBError: null,
8024
+ attempts: null,
8025
+ flippedAtAttempt: null
7781
8026
  }
7782
8027
  };
7783
8028
  continue;
7784
8029
  }
7785
8030
  const replayCase = selected.find((c) => c.trajId === row.trajId && c.corpus === row.corpus);
7786
8031
  const label = `fix ${row.corpus}/${row.trajId}`;
7787
- onProgress(`${label}: generating corrected command`);
7788
- calls += 1;
7789
- const generated = await generateFixCommand(caller, {
8032
+ const signature = verdictByTraj.get(row.trajId)?.signature ?? null;
8033
+ const stepTimeoutMs = options.stepTimeoutMs ?? replayCase.recordedStepTimeoutMs ?? 12e4;
8034
+ const promptInput = {
7790
8035
  taskStatement: replayCase.taskStatement,
7791
8036
  steps: replayCase.steps,
7792
8037
  k: replayCase.k
7793
- });
8038
+ };
8039
+ const caseOut = join21(options.out, caseOutDirName(row));
8040
+ if (options.fix === "loop") {
8041
+ onProgress(`${label}: fix loop (budget ${fixAttempts} attempts)`);
8042
+ const result = await runFixLoop(
8043
+ caller,
8044
+ promptInput,
8045
+ async (command, attempt) => {
8046
+ onProgress(
8047
+ `${label}: attempt ${attempt} arm B \u2014 ${command.split("\n")[0].slice(0, 120)}`
8048
+ );
8049
+ const armB = await executeArmB(
8050
+ replayCase,
8051
+ command,
8052
+ backendFactory(row.derivedImage),
8053
+ signature,
8054
+ stepTimeoutMs,
8055
+ options.prefixLimit,
8056
+ (message) => onProgress(`${label}: attempt ${attempt}: ${message}`)
8057
+ );
8058
+ mkdirSync2(caseOut, { recursive: true });
8059
+ writeFileSync2(
8060
+ join21(caseOut, `armB-attempt${attempt}-result.json`),
8061
+ `${JSON.stringify({ command, ...armB }, null, 2)}
8062
+ `
8063
+ );
8064
+ writeFileSync2(
8065
+ join21(caseOut, "armB-result.json"),
8066
+ `${JSON.stringify({ command, attempt, ...armB }, null, 2)}
8067
+ `
8068
+ );
8069
+ return armB;
8070
+ },
8071
+ { maxAttempts: fixAttempts, onProgress: (message) => onProgress(`${label}: ${message}`) }
8072
+ );
8073
+ calls += result.llmCalls;
8074
+ failures += result.llmFailures;
8075
+ promptTokens += result.promptTokens;
8076
+ completionTokens += result.completionTokens;
8077
+ const usage = result.promptTokens + result.completionTokens > 0 ? { promptTokens: result.promptTokens, completionTokens: result.completionTokens } : null;
8078
+ const flippedRecord = result.flippedAtAttempt !== null ? result.attempts.find((a) => a.attempt === result.flippedAtAttempt) : null;
8079
+ const summary = flippedRecord ?? [...result.attempts].reverse().find((a) => a.executed) ?? null;
8080
+ const lastRecord = result.attempts.at(-1) ?? null;
8081
+ rows[index] = {
8082
+ ...row,
8083
+ fix: summary ? {
8084
+ attempted: true,
8085
+ sampledOut: false,
8086
+ command: summary.command,
8087
+ llmError: null,
8088
+ usage,
8089
+ armBExit: summary.exitCode,
8090
+ armBPrefixExecuted: summary.prefixExecuted,
8091
+ armBPrefixDivergences: summary.prefixDivergences,
8092
+ failureVanished: summary.failureVanished,
8093
+ armBError: null,
8094
+ attempts: result.attempts,
8095
+ flippedAtAttempt: result.flippedAtAttempt
8096
+ } : result.aborted ? {
8097
+ attempted: true,
8098
+ sampledOut: false,
8099
+ command: lastRecord?.command ?? null,
8100
+ llmError: null,
8101
+ usage,
8102
+ armBExit: null,
8103
+ armBPrefixExecuted: null,
8104
+ armBPrefixDivergences: null,
8105
+ failureVanished: null,
8106
+ armBError: lastRecord?.armBError ?? "sandbox error",
8107
+ attempts: result.attempts,
8108
+ flippedAtAttempt: null
8109
+ } : {
8110
+ attempted: true,
8111
+ sampledOut: false,
8112
+ command: null,
8113
+ llmError: lastRecord?.llmError ?? "no attempt produced a runnable fix",
8114
+ usage,
8115
+ armBExit: null,
8116
+ armBPrefixExecuted: null,
8117
+ armBPrefixDivergences: null,
8118
+ failureVanished: null,
8119
+ armBError: null,
8120
+ attempts: result.attempts,
8121
+ flippedAtAttempt: null
8122
+ }
8123
+ };
8124
+ onProgress(
8125
+ `${label}: loop done \u2014 flipped=${result.flipped}` + (result.flippedAtAttempt !== null ? ` at attempt ${result.flippedAtAttempt}` : "") + ` (${result.llmCalls} calls, ${result.attempts.filter((a) => a.executed).length} arms)`
8126
+ );
8127
+ continue;
8128
+ }
8129
+ onProgress(`${label}: generating corrected command`);
8130
+ calls += 1;
8131
+ const generated = await generateFixCommand(caller, promptInput);
7794
8132
  if (!generated.succeeded) {
7795
8133
  failures += 1;
7796
8134
  rows[index] = {
@@ -7805,7 +8143,9 @@ async function runReplayBatch(options) {
7805
8143
  armBPrefixExecuted: null,
7806
8144
  armBPrefixDivergences: null,
7807
8145
  failureVanished: null,
7808
- armBError: null
8146
+ armBError: null,
8147
+ attempts: null,
8148
+ flippedAtAttempt: null
7809
8149
  }
7810
8150
  };
7811
8151
  onProgress(`${label}: LLM failed \u2014 ${generated.error.slice(0, 200)}`);
@@ -7813,8 +8153,6 @@ async function runReplayBatch(options) {
7813
8153
  }
7814
8154
  promptTokens += generated.value.usage?.promptTokens ?? 0;
7815
8155
  completionTokens += generated.value.usage?.completionTokens ?? 0;
7816
- const signature = verdictByTraj.get(row.trajId)?.signature ?? null;
7817
- const stepTimeoutMs = options.stepTimeoutMs ?? replayCase.recordedStepTimeoutMs ?? 12e4;
7818
8156
  onProgress(`${label}: arm B \u2014 ${generated.value.command.split("\n")[0].slice(0, 120)}`);
7819
8157
  try {
7820
8158
  const armB = await executeArmB(
@@ -7838,10 +8176,11 @@ async function runReplayBatch(options) {
7838
8176
  armBPrefixExecuted: armB.prefixExecuted,
7839
8177
  armBPrefixDivergences: armB.prefixDivergences,
7840
8178
  failureVanished: armB.failureVanished,
7841
- armBError: null
8179
+ armBError: null,
8180
+ attempts: null,
8181
+ flippedAtAttempt: armB.failureVanished ? 1 : null
7842
8182
  }
7843
8183
  };
7844
- const caseOut = join21(options.out, caseOutDirName(row));
7845
8184
  mkdirSync2(caseOut, { recursive: true });
7846
8185
  writeFileSync2(
7847
8186
  join21(caseOut, "armB-result.json"),
@@ -7863,7 +8202,9 @@ async function runReplayBatch(options) {
7863
8202
  armBPrefixExecuted: null,
7864
8203
  armBPrefixDivergences: null,
7865
8204
  failureVanished: null,
7866
- armBError: message.slice(0, 500)
8205
+ armBError: message.slice(0, 500),
8206
+ attempts: null,
8207
+ flippedAtAttempt: null
7867
8208
  }
7868
8209
  };
7869
8210
  onProgress(`${label}: arm B error \u2014 ${message.slice(0, 200)}`);
@@ -7888,6 +8229,15 @@ async function runReplayBatch(options) {
7888
8229
  (r) => r.recordedReturncodeAtK !== null && r.recordedReturncodeAtK !== 0
7889
8230
  );
7890
8231
  const flippedNonzeroRc = armBNonzeroRc.filter((r) => r.fix.failureVanished === true);
8232
+ const attempt1Executed = rows.filter(
8233
+ (r) => r.fix?.attempts?.find((a) => a.attempt === 1)?.executed === true
8234
+ );
8235
+ const flippedAt1 = rows.filter((r) => r.fix?.flippedAtAttempt === 1);
8236
+ const flipsByAttempt = {};
8237
+ for (const row of rows) {
8238
+ const at = row.fix?.flippedAtAttempt;
8239
+ if (typeof at === "number") flipsByAttempt[String(at)] = (flipsByAttempt[String(at)] ?? 0) + 1;
8240
+ }
7891
8241
  const excludedByReason = {};
7892
8242
  for (const excluded of enumeration.excluded) {
7893
8243
  excludedByReason[excluded.reason] = (excludedByReason[excluded.reason] ?? 0) + 1;
@@ -7927,16 +8277,22 @@ async function runReplayBatch(options) {
7927
8277
  denominator: selected.length,
7928
8278
  value: rate(signatureStrict.length, selected.length)
7929
8279
  },
7930
- fixFlipRate: options.fix === "generate" ? {
8280
+ fixFlipRate: options.fix !== "none" ? {
7931
8281
  numerator: flipped.length,
7932
8282
  denominator: armBExecuted.length,
7933
8283
  value: rate(flipped.length, armBExecuted.length)
7934
8284
  } : null,
7935
- fixFlipRateNonzeroRc: options.fix === "generate" ? {
8285
+ fixFlipRateNonzeroRc: options.fix !== "none" ? {
7936
8286
  numerator: flippedNonzeroRc.length,
7937
8287
  denominator: armBNonzeroRc.length,
7938
8288
  value: rate(flippedNonzeroRc.length, armBNonzeroRc.length)
7939
- } : null
8289
+ } : null,
8290
+ fixFlipAttempt1: options.fix === "loop" ? {
8291
+ numerator: flippedAt1.length,
8292
+ denominator: attempt1Executed.length,
8293
+ value: rate(flippedAt1.length, attempt1Executed.length)
8294
+ } : null,
8295
+ flipsByAttempt: options.fix === "loop" ? flipsByAttempt : null
7940
8296
  },
7941
8297
  llm,
7942
8298
  excluded: enumeration.excluded,
@@ -7980,6 +8336,15 @@ function renderBatchReport(report) {
7980
8336
  `- Fix-flip rate on recorded-rc\u22600 cases: ${pct(headline.fixFlipRateNonzeroRc.value)} (${headline.fixFlipRateNonzeroRc.numerator}/${headline.fixFlipRateNonzeroRc.denominator}; real recorded failures \u2014 a gold step recorded with rc 0 flips vacuously).`
7981
8337
  );
7982
8338
  }
8339
+ if (headline.fixFlipAttempt1) {
8340
+ lines.push(
8341
+ `- Fix-flip@1: ${pct(headline.fixFlipAttempt1.value)} (${headline.fixFlipAttempt1.numerator}/${headline.fixFlipAttempt1.denominator} cases whose attempt 1 executed \u2014 the one-shot-comparable number).`
8342
+ );
8343
+ }
8344
+ if (headline.flipsByAttempt && Object.keys(headline.flipsByAttempt).length > 0) {
8345
+ const parts = Object.entries(headline.flipsByAttempt).sort((a, b) => Number(a[0]) - Number(b[0])).map(([attempt, count2]) => `attempt ${attempt}: ${count2}`);
8346
+ lines.push(`- Flips by attempt: ${parts.join(", ")}.`);
8347
+ }
7983
8348
  lines.push("");
7984
8349
  lines.push("## Enumeration");
7985
8350
  lines.push("");
@@ -8028,7 +8393,7 @@ function renderBatchReport(report) {
8028
8393
  );
8029
8394
  for (const row of report.cases) {
8030
8395
  const fix = row.fix;
8031
- const fixCell = !fix ? "\u2014" : fix.sampledOut ? "sampled-out" : fix.llmError ? "llm-failed" : fix.armBError ? "armB-error" : "generated";
8396
+ const fixCell = !fix ? "\u2014" : fix.sampledOut ? "sampled-out" : fix.llmError ? "llm-failed" : fix.armBError ? "armB-error" : fix.attempts ? fix.flippedAtAttempt !== null ? `flip@${fix.flippedAtAttempt}` : `exhausted(${fix.attempts.length})` : "generated";
8032
8397
  lines.push(
8033
8398
  [
8034
8399
  row.corpus,
@@ -8090,13 +8455,20 @@ function parseReplayBatchArgs(argv) {
8090
8455
  return n;
8091
8456
  };
8092
8457
  const fix = values.get("--fix") ?? "none";
8093
- if (fix !== "none" && fix !== "generate") {
8094
- throw new Error(`replay-verify-batch: --fix must be none or generate, got ${fix}`);
8458
+ if (fix !== "none" && fix !== "generate" && fix !== "loop") {
8459
+ throw new Error(`replay-verify-batch: --fix must be none, generate, or loop, got ${fix}`);
8460
+ }
8461
+ const fixAttempts = optionalNumber("--fix-attempts") ?? 3;
8462
+ if (!Number.isInteger(fixAttempts) || fixAttempts < 1) {
8463
+ throw new Error(
8464
+ `replay-verify-batch: --fix-attempts must be a positive integer, got ${values.get("--fix-attempts")}`
8465
+ );
8095
8466
  }
8096
8467
  return {
8097
8468
  corpora,
8098
8469
  out: out ?? ".",
8099
8470
  fix,
8471
+ fixAttempts,
8100
8472
  fixModel: values.get("--fix-model") ?? "glm-5.2",
8101
8473
  fixBaseUrl: values.get("--fix-base-url") ?? "https://api.z.ai/api/coding/paas/v4",
8102
8474
  fixApiKeyEnv: values.get("--fix-api-key-env") ?? "ZAI_GLM_API_KEY",
@@ -8118,7 +8490,7 @@ function replayBatchUsage() {
8118
8490
  Usage:
8119
8491
  traces replay-verify-batch \\
8120
8492
  --corpus NAME=<labels.json>::<prepared-dir> [--corpus ...] \\
8121
- --out DIR [--fix none|generate] [--enumerate-only] \\
8493
+ --out DIR [--fix none|generate|loop] [--fix-attempts 3] [--enumerate-only] \\
8122
8494
  [--fix-model glm-5.2] [--fix-base-url URL] [--fix-api-key-env ZAI_GLM_API_KEY] \\
8123
8495
  [--max-fix-cases 30] [--seed 17] [--step-timeout MS] [--prefix-limit N] \\
8124
8496
  [--case-filter SUBSTRING] [--case-limit N] \\
@@ -8137,6 +8509,13 @@ Usage:
8137
8509
  --max-fix-cases, seeded sample beyond it) and executes the corrected command
8138
8510
  as arm B in a fresh sandbox.
8139
8511
 
8512
+ --fix loop iterates: attempt 1 is the one-shot prompt; when an arm fails
8513
+ (nonzero exit or the failure signature persists) or the model call fails,
8514
+ the next prompt carries every prior command with its REAL executed
8515
+ stdout/stderr, up to --fix-attempts attempts (default 3). Retries may
8516
+ answer with a short script (<=5 commands) executed as one /bin/sh unit.
8517
+ Every attempt runs in its own fresh sandbox with the same replayed prefix.
8518
+
8140
8519
  Outputs batch-report.json, batch-report.md, cases.jsonl (incremental), and one
8141
8520
  directory per case with the full replay-verify artifacts.
8142
8521
  See docs/replay-verify.md for orchestrator setup and honest limits.
@@ -8181,7 +8560,7 @@ async function cmdReplayVerifyBatch(argv) {
8181
8560
  );
8182
8561
  }
8183
8562
  let fixCaller;
8184
- if (parsed.fix === "generate") {
8563
+ if (parsed.fix !== "none") {
8185
8564
  const fixApiKey = process.env[parsed.fixApiKeyEnv];
8186
8565
  if (!fixApiKey) {
8187
8566
  throw new Error(
@@ -8200,6 +8579,7 @@ async function cmdReplayVerifyBatch(argv) {
8200
8579
  baseUrl: parsed.baseUrl,
8201
8580
  apiKey,
8202
8581
  fix: parsed.fix,
8582
+ fixAttempts: parsed.fixAttempts,
8203
8583
  fixCaller,
8204
8584
  fixModelLabel: parsed.fixModel,
8205
8585
  maxFixCases: parsed.maxFixCases,
@@ -8219,6 +8599,573 @@ async function cmdReplayVerifyBatch(argv) {
8219
8599
  );
8220
8600
  }
8221
8601
 
8602
+ // src/replay-wire.ts
8603
+ var SUBJECT_PATTERN = /^incorrect-steps-(\d+)-(\d+)-(escaped|unescaped)-consequence-(\d+)$/;
8604
+ function parseIncorrectStepsSubject(subject) {
8605
+ const match = SUBJECT_PATTERN.exec(subject);
8606
+ if (!match) return null;
8607
+ return {
8608
+ firstStep: Number(match[1]),
8609
+ lastStep: Number(match[2]),
8610
+ escapeStatus: match[3],
8611
+ consequenceStep: Number(match[4])
8612
+ };
8613
+ }
8614
+ function resolveFindingInvocation(finding2, corpora) {
8615
+ const subject = parseIncorrectStepsSubject(finding2.subject);
8616
+ if (!subject) {
8617
+ throw new Error(
8618
+ `replay-wire: subject '${finding2.subject}' is not incorrect-steps-<first>-<last>-<escaped|unescaped>-consequence-<step>`
8619
+ );
8620
+ }
8621
+ const failures = [];
8622
+ for (const corpus of corpora) {
8623
+ const resolution = resolveCaseResources(corpus, finding2.trajId);
8624
+ if (resolution.resolved) {
8625
+ const at = subject.firstStep;
8626
+ const step = resolution.resources.steps.find((s) => s.step_id === at);
8627
+ if (!step) {
8628
+ throw new Error(
8629
+ `replay-wire: finding step ${at} is outside ${finding2.trajId} (${resolution.resources.steps.length} steps)`
8630
+ );
8631
+ }
8632
+ return { resources: resolution.resources, subject, at };
8633
+ }
8634
+ failures.push(`${corpus.name}: ${resolution.reason}`);
8635
+ }
8636
+ throw new Error(
8637
+ `replay-wire: trajectory ${finding2.trajId} is not replayable in any corpus \u2014 ${failures.join("; ")}`
8638
+ );
8639
+ }
8640
+ async function replayVerifyFinding(finding2, options) {
8641
+ if (options.fixCaller && options.fixCommand !== void 0) {
8642
+ throw new Error("replay-wire: pass fixCaller or fixCommand, not both");
8643
+ }
8644
+ const invocation = resolveFindingInvocation(finding2, options.corpora);
8645
+ let fixCommand = options.fixCommand ?? null;
8646
+ if (options.fixCaller) {
8647
+ const generated = await generateFixCommand(options.fixCaller, {
8648
+ taskStatement: invocation.resources.taskStatement,
8649
+ steps: invocation.resources.steps,
8650
+ k: invocation.at
8651
+ });
8652
+ if (!generated.succeeded) {
8653
+ throw new Error(`replay-wire: fix generation failed \u2014 ${generated.error}`);
8654
+ }
8655
+ fixCommand = generated.value.command;
8656
+ }
8657
+ const verdict = await replayVerify({
8658
+ stepsPath: invocation.resources.stepsPath,
8659
+ image: invocation.resources.image,
8660
+ at: invocation.at,
8661
+ fixCommand: fixCommand ?? void 0,
8662
+ cwd: invocation.resources.cwd,
8663
+ out: options.out,
8664
+ caseId: invocation.resources.trajId,
8665
+ stepTimeoutMs: options.stepTimeoutMs ?? invocation.resources.recordedStepTimeoutMs ?? void 0,
8666
+ prefixLimit: options.prefixLimit,
8667
+ backend: options.backend,
8668
+ apiKey: options.apiKey,
8669
+ baseUrl: options.baseUrl,
8670
+ onProgress: options.onProgress
8671
+ });
8672
+ return { invocation, fixCommand, verdict };
8673
+ }
8674
+
8675
+ // src/analyze-verify.ts
8676
+ import { mkdirSync as mkdirSync3, readFileSync as readFileSync4, writeFileSync as writeFileSync3 } from "fs";
8677
+ import { join as join22 } from "path";
8678
+ var SINGLE_STEP_SUBJECT = /^incorrect-step-(\d+)$/;
8679
+ function findingReplayStep(finding2) {
8680
+ const fromMetadata = finding2.metadata?.block_first_step;
8681
+ if (typeof fromMetadata === "number" && Number.isInteger(fromMetadata) && fromMetadata >= 1) {
8682
+ return fromMetadata;
8683
+ }
8684
+ const subject = finding2.subject ?? "";
8685
+ const wire = parseIncorrectStepsSubject(subject);
8686
+ if (wire) return wire.firstStep;
8687
+ const single = SINGLE_STEP_SUBJECT.exec(subject);
8688
+ if (single) return Number(single[1]);
8689
+ return null;
8690
+ }
8691
+ var TRACE_EVIDENCE_URI = /^trace:\/\/([^/]+)\//;
8692
+ function findingTrajectoryId(finding2) {
8693
+ for (const ref of finding2.evidence_refs ?? []) {
8694
+ if (typeof ref.uri !== "string") continue;
8695
+ const match = TRACE_EVIDENCE_URI.exec(ref.uri);
8696
+ if (match) return match[1];
8697
+ }
8698
+ return null;
8699
+ }
8700
+ function checkStep(steps, at) {
8701
+ const step = steps.find((s) => s.step_id === at);
8702
+ if (!step) {
8703
+ return { ok: false, reason: `step ${at} is outside the trajectory (${steps.length} steps)` };
8704
+ }
8705
+ if (isSubmitAction(step.action)) {
8706
+ return {
8707
+ ok: false,
8708
+ reason: `step ${at} is the submit action \u2014 a submit decision has no executable failure to replay`
8709
+ };
8710
+ }
8711
+ const recordedReturncode = parseRecordedReturncode(step.observation);
8712
+ if (recordedReturncode === null) {
8713
+ return {
8714
+ ok: false,
8715
+ reason: `step ${at} recorded no returncode \u2014 there is no executable failure signature to reproduce`
8716
+ };
8717
+ }
8718
+ return { ok: true, recordedReturncode };
8719
+ }
8720
+ function resolveFindingReplayability(finding2, source) {
8721
+ const at = findingReplayStep(finding2);
8722
+ if (at === null) {
8723
+ return {
8724
+ replayable: false,
8725
+ reason: `subject '${finding2.subject ?? "(none)"}' names no trajectory step (expected incorrect-step-<n>, incorrect-steps-<f>-<l>-\u2026, or metadata.block_first_step)`
8726
+ };
8727
+ }
8728
+ if (source.kind === "direct") {
8729
+ const trajId2 = findingTrajectoryId(finding2);
8730
+ if (trajId2 && source.caseId && trajId2 !== source.caseId) {
8731
+ return {
8732
+ replayable: false,
8733
+ reason: `finding cites trajectory '${trajId2}' but the supplied steps are case '${source.caseId}'`
8734
+ };
8735
+ }
8736
+ let steps;
8737
+ try {
8738
+ steps = JSON.parse(readFileSync4(source.stepsPath, "utf8"));
8739
+ } catch (err) {
8740
+ throw new Error(
8741
+ `verify-findings: cannot read steps file ${source.stepsPath} \u2014 ${err instanceof Error ? err.message : String(err)}`
8742
+ );
8743
+ }
8744
+ if (!Array.isArray(steps) || steps.length === 0) {
8745
+ throw new Error(`verify-findings: ${source.stepsPath} is not a non-empty steps array`);
8746
+ }
8747
+ const step = checkStep(steps, at);
8748
+ if (!step.ok) return { replayable: false, reason: step.reason };
8749
+ return {
8750
+ replayable: true,
8751
+ resolved: {
8752
+ caseId: source.caseId ?? trajId2 ?? source.stepsPath,
8753
+ stepsPath: source.stepsPath,
8754
+ image: source.image,
8755
+ cwd: source.cwd,
8756
+ at,
8757
+ recordedReturncode: step.recordedReturncode,
8758
+ recordedStepTimeoutMs: null
8759
+ }
8760
+ };
8761
+ }
8762
+ const trajId = findingTrajectoryId(finding2);
8763
+ if (!trajId) {
8764
+ return {
8765
+ replayable: false,
8766
+ reason: "finding carries no trace://<trajectory>/ evidence ref naming its trajectory"
8767
+ };
8768
+ }
8769
+ const failures = [];
8770
+ for (const corpus of source.corpora) {
8771
+ const resolution = resolveCaseResources(corpus, trajId);
8772
+ if (!resolution.resolved) {
8773
+ failures.push(`${corpus.name}: ${resolution.reason}${resolution.detail ? ` (${resolution.detail})` : ""}`);
8774
+ continue;
8775
+ }
8776
+ const step = checkStep(resolution.resources.steps, at);
8777
+ if (!step.ok) return { replayable: false, reason: step.reason };
8778
+ return {
8779
+ replayable: true,
8780
+ resolved: {
8781
+ caseId: trajId,
8782
+ stepsPath: resolution.resources.stepsPath,
8783
+ image: resolution.resources.image,
8784
+ cwd: resolution.resources.cwd,
8785
+ at,
8786
+ recordedReturncode: step.recordedReturncode,
8787
+ recordedStepTimeoutMs: resolution.resources.recordedStepTimeoutMs
8788
+ }
8789
+ };
8790
+ }
8791
+ return {
8792
+ replayable: false,
8793
+ reason: `trajectory ${trajId} is not replayable in any corpus \u2014 ${failures.join("; ")}`
8794
+ };
8795
+ }
8796
+ function classifyVerdict(verdict) {
8797
+ if (!verdict.armA.failureSignatureMatch) return "divergent";
8798
+ if (verdict.armB?.failureVanished) return "fix-flipped";
8799
+ return "reproduced";
8800
+ }
8801
+ var DEFAULT_SANDBOX_BASE_URL = "http://127.0.0.1:4097";
8802
+ async function assertSandboxReachable(baseUrl) {
8803
+ let response;
8804
+ try {
8805
+ response = await fetch(new URL("/health", baseUrl), { signal: AbortSignal.timeout(5e3) });
8806
+ } catch (err) {
8807
+ throw new Error(
8808
+ `verify-findings: sandbox API unreachable at ${baseUrl} \u2014 ${err instanceof Error ? err.message : String(err)}. Executing findings requires a running sandbox orchestrator + SDK adapter (see docs/replay-verify.md); start them or pass --base-url.`
8809
+ );
8810
+ }
8811
+ if (!response.ok) {
8812
+ throw new Error(
8813
+ `verify-findings: sandbox API at ${baseUrl} answered /health with HTTP ${response.status} \u2014 refusing to run proofs against degraded infrastructure`
8814
+ );
8815
+ }
8816
+ }
8817
+ function receiptExecution(verdict) {
8818
+ return {
8819
+ image: verdict.image,
8820
+ cwd: verdict.cwd,
8821
+ recordedReturncode: verdict.recordedReturncode,
8822
+ signature: verdict.signature,
8823
+ signatureBasis: verdict.signatureBasis,
8824
+ armA: {
8825
+ command: verdict.armA.command,
8826
+ exitCode: verdict.armA.exitCode,
8827
+ wallMs: verdict.armA.wallMs,
8828
+ failureSignatureMatch: verdict.armA.failureSignatureMatch
8829
+ },
8830
+ armB: verdict.armB ? {
8831
+ command: verdict.armB.command,
8832
+ exitCode: verdict.armB.exitCode,
8833
+ wallMs: verdict.armB.wallMs,
8834
+ failureVanished: verdict.armB.failureVanished
8835
+ } : null,
8836
+ prefixExecuted: verdict.prefixExecuted,
8837
+ prefixDivergences: verdict.prefixDivergences.length,
8838
+ totalMs: verdict.timings.totalMs
8839
+ };
8840
+ }
8841
+ function writeReceipt(receiptDir, finding2, verification, execution) {
8842
+ const receipt = {
8843
+ schema_version: "1.0.0",
8844
+ produced_at: (/* @__PURE__ */ new Date()).toISOString(),
8845
+ finding_id: verification.finding_id,
8846
+ analyst_id: finding2.analyst_id ?? null,
8847
+ subject: verification.subject,
8848
+ claim: typeof finding2.claim === "string" ? finding2.claim.slice(0, 600) : null,
8849
+ trajectory_id: verification.trajectory_id,
8850
+ step: verification.step,
8851
+ verified: verification.verified,
8852
+ reason: verification.reason,
8853
+ execution,
8854
+ verdict_path: verification.verdict_path,
8855
+ deduplicated_with: verification.deduplicated_with
8856
+ };
8857
+ writeFileSync3(join22(receiptDir, "receipt.json"), `${JSON.stringify(receipt, null, 2)}
8858
+ `);
8859
+ }
8860
+ function receiptDirName(index, finding2) {
8861
+ const id = typeof finding2.finding_id === "string" && finding2.finding_id.length > 0 ? finding2.finding_id.replace(/[^A-Za-z0-9_-]/g, "_") : "finding";
8862
+ return `${String(index + 1).padStart(3, "0")}-${id}`;
8863
+ }
8864
+ async function verifyFindings(findings, options) {
8865
+ if (findings.length === 0) throw new Error("verify-findings: no findings to verify");
8866
+ mkdirSync3(options.out, { recursive: true });
8867
+ const resolutions = findings.map((finding2) => resolveFindingReplayability(finding2, options.source));
8868
+ const baseUrl = options.baseUrl ?? DEFAULT_SANDBOX_BASE_URL;
8869
+ if (resolutions.some((resolution) => resolution.replayable) && !options.backend) {
8870
+ if (!options.apiKey) {
8871
+ throw new Error(
8872
+ "verify-findings: replayable findings need a sandbox API key \u2014 export SANDBOX_API_KEY or pass --api-key-env"
8873
+ );
8874
+ }
8875
+ await assertSandboxReachable(baseUrl);
8876
+ }
8877
+ const preparer = options.source.kind === "corpus" && !options.backend ? options.source.preparer ?? dockerImagePreparer() : null;
8878
+ const preparedImages = /* @__PURE__ */ new Map();
8879
+ const executedByKey = /* @__PURE__ */ new Map();
8880
+ const verifications = [];
8881
+ let executions = 0;
8882
+ for (let index = 0; index < findings.length; index++) {
8883
+ const finding2 = findings[index];
8884
+ const resolution = resolutions[index];
8885
+ const receiptDir = join22(options.out, receiptDirName(index, finding2));
8886
+ mkdirSync3(receiptDir, { recursive: true });
8887
+ const identity = {
8888
+ finding_id: finding2.finding_id ?? null,
8889
+ subject: finding2.subject ?? null,
8890
+ trajectory_id: findingTrajectoryId(finding2)
8891
+ };
8892
+ if (!resolution.replayable) {
8893
+ const verification2 = {
8894
+ ...identity,
8895
+ step: findingReplayStep(finding2),
8896
+ verified: "not-replayable",
8897
+ reason: resolution.reason,
8898
+ receipt: receiptDir,
8899
+ verdict_path: null,
8900
+ deduplicated_with: null
8901
+ };
8902
+ writeReceipt(receiptDir, finding2, verification2, null);
8903
+ verifications.push(verification2);
8904
+ options.onProgress?.(`${identity.finding_id ?? `finding ${index + 1}`}: not-replayable \u2014 ${resolution.reason}`);
8905
+ continue;
8906
+ }
8907
+ const resolved = resolution.resolved;
8908
+ const dedupeKey = `${resolved.caseId}::${resolved.at}::${options.fixCommand ?? ""}`;
8909
+ const prior = executedByKey.get(dedupeKey);
8910
+ if (prior) {
8911
+ const verification2 = {
8912
+ ...identity,
8913
+ step: resolved.at,
8914
+ verified: prior.verified,
8915
+ reason: prior.reason,
8916
+ receipt: receiptDir,
8917
+ verdict_path: prior.verdict_path,
8918
+ deduplicated_with: prior.receipt
8919
+ };
8920
+ const priorVerdict = prior.verdict_path ? JSON.parse(readFileSync4(prior.verdict_path, "utf8")) : null;
8921
+ writeReceipt(receiptDir, finding2, verification2, priorVerdict ? receiptExecution(priorVerdict) : null);
8922
+ verifications.push(verification2);
8923
+ options.onProgress?.(
8924
+ `${identity.finding_id ?? `finding ${index + 1}`}: ${prior.verified} (shares proof with ${prior.finding_id ?? prior.receipt})`
8925
+ );
8926
+ continue;
8927
+ }
8928
+ let image = resolved.image;
8929
+ if (preparer) {
8930
+ const preparationKey = `${resolved.image}::${resolved.cwd}`;
8931
+ let preparation = preparedImages.get(preparationKey);
8932
+ if (!preparation) {
8933
+ const ensured = await preparer.ensure(resolved.image, resolved.cwd);
8934
+ preparation = ensured.succeeded ? { succeeded: true, image: ensured.value.derivedImage } : { succeeded: false, error: ensured.error };
8935
+ preparedImages.set(preparationKey, preparation);
8936
+ }
8937
+ if (!preparation.succeeded) {
8938
+ const verification2 = {
8939
+ ...identity,
8940
+ step: resolved.at,
8941
+ verified: "not-replayable",
8942
+ reason: `replay image could not be prepared \u2014 ${preparation.error}`,
8943
+ receipt: receiptDir,
8944
+ verdict_path: null,
8945
+ deduplicated_with: null
8946
+ };
8947
+ writeReceipt(receiptDir, finding2, verification2, null);
8948
+ verifications.push(verification2);
8949
+ options.onProgress?.(`${identity.finding_id ?? `finding ${index + 1}`}: not-replayable \u2014 image preparation failed`);
8950
+ continue;
8951
+ }
8952
+ image = preparation.image;
8953
+ }
8954
+ options.onProgress?.(
8955
+ `${identity.finding_id ?? `finding ${index + 1}`}: executing arm A at step ${resolved.at} of ${resolved.caseId} on ${image}`
8956
+ );
8957
+ const verdict = await replayVerify({
8958
+ stepsPath: resolved.stepsPath,
8959
+ image,
8960
+ at: resolved.at,
8961
+ fixCommand: options.fixCommand,
8962
+ cwd: resolved.cwd,
8963
+ out: receiptDir,
8964
+ caseId: resolved.caseId,
8965
+ stepTimeoutMs: options.stepTimeoutMs ?? resolved.recordedStepTimeoutMs ?? void 0,
8966
+ prefixLimit: options.prefixLimit,
8967
+ backend: options.backend,
8968
+ apiKey: options.apiKey,
8969
+ baseUrl,
8970
+ maxLifetimeSeconds: options.maxLifetimeSeconds,
8971
+ onProgress: options.onProgress
8972
+ });
8973
+ executions += 1;
8974
+ const verification = {
8975
+ ...identity,
8976
+ step: resolved.at,
8977
+ verified: classifyVerdict(verdict),
8978
+ reason: null,
8979
+ receipt: receiptDir,
8980
+ verdict_path: join22(receiptDir, "replay-verdict.json"),
8981
+ deduplicated_with: null
8982
+ };
8983
+ executedByKey.set(dedupeKey, verification);
8984
+ writeReceipt(receiptDir, finding2, verification, receiptExecution(verdict));
8985
+ verifications.push(verification);
8986
+ options.onProgress?.(`${identity.finding_id ?? `finding ${index + 1}`}: ${verification.verified}`);
8987
+ }
8988
+ const counts = {
8989
+ reproduced: 0,
8990
+ "fix-flipped": 0,
8991
+ divergent: 0,
8992
+ "not-replayable": 0
8993
+ };
8994
+ for (const verification of verifications) counts[verification.verified] += 1;
8995
+ const run = { out: options.out, verifications, counts, executions };
8996
+ writeFileSync3(
8997
+ join22(options.out, "verifications.json"),
8998
+ `${JSON.stringify({ schema_version: "1.0.0", ...run }, null, 2)}
8999
+ `
9000
+ );
9001
+ return run;
9002
+ }
9003
+ function verdictCell(verification) {
9004
+ switch (verification.verified) {
9005
+ case "reproduced":
9006
+ return "**VERIFIED** \u2014 reproduced";
9007
+ case "fix-flipped":
9008
+ return "**VERIFIED** \u2014 fix-flipped";
9009
+ case "divergent":
9010
+ return "DIVERGENT \u2014 recorded failure did not reproduce";
9011
+ case "not-replayable":
9012
+ return `UNVERIFIABLE \u2014 ${verification.reason ?? "no reason recorded"}`;
9013
+ }
9014
+ }
9015
+ function renderVerifiedFindingsSection(run) {
9016
+ const lines = ["## Verified findings (executed replay)", ""];
9017
+ const total = run.verifications.length;
9018
+ lines.push(
9019
+ `${total} finding(s) \u2192 ${run.counts.reproduced} reproduced, ${run.counts["fix-flipped"]} fix-flipped, ${run.counts.divergent} divergent, ${run.counts["not-replayable"]} not replayable (${run.executions} sandbox execution(s); findings accusing the same step share one proof).`
9020
+ );
9021
+ lines.push("");
9022
+ lines.push("| Finding | Subject | Step | Verdict | Receipt |");
9023
+ lines.push("|---|---|---:|---|---|");
9024
+ for (const verification of run.verifications) {
9025
+ const shared = verification.deduplicated_with ? " (shared proof)" : "";
9026
+ lines.push(
9027
+ `| \`${verification.finding_id ?? "\u2014"}\` | \`${verification.subject ?? "\u2014"}\` | ${verification.step ?? "\u2014"} | ${verdictCell(verification)} | \`${verification.receipt}\`${shared} |`
9028
+ );
9029
+ }
9030
+ lines.push("");
9031
+ lines.push(
9032
+ "VERIFIED = the accused step was re-executed in a sandbox after replaying the trajectory prefix, and the recorded failure signature reproduced (fix-flipped: a corrected command additionally made it vanish). Each receipt directory carries receipt.json and, when executed, replay-verdict.json + report.md with real stdout/stderr."
9033
+ );
9034
+ lines.push("");
9035
+ return lines.join("\n");
9036
+ }
9037
+ function readFindingsFile(path) {
9038
+ const parsed = JSON.parse(readFileSync4(path, "utf8"));
9039
+ const array = Array.isArray(parsed) ? parsed : parsed && typeof parsed === "object" && Array.isArray(parsed.findings) ? parsed.findings : null;
9040
+ if (!array) {
9041
+ throw new Error(`${path} is neither a findings array nor an object with a findings array`);
9042
+ }
9043
+ for (const entry of array) {
9044
+ if (!entry || typeof entry !== "object") {
9045
+ throw new Error(`${path}: every finding must be an object, got ${JSON.stringify(entry)}`);
9046
+ }
9047
+ }
9048
+ return array;
9049
+ }
9050
+ function verifyFindingsUsage() {
9051
+ return `traces verify-findings \u2014 execute analyst findings as sandbox replay proofs
9052
+
9053
+ Usage:
9054
+ traces verify-findings --findings FINDINGS.json --out DIR \\
9055
+ ( --steps STEPS.json --image IMG --cwd DIR [--case ID]
9056
+ | --corpus name=<labels.json>::<preparedDir> [--corpus ...] ) \\
9057
+ [--fix-command CMD] [--step-timeout MS] [--prefix-limit N] \\
9058
+ [--base-url URL] [--api-key-env VAR] [--max-lifetime SECONDS]
9059
+
9060
+ --findings JSON array of analyst findings (or an object with a findings
9061
+ array) \u2014 the shape agent-eval analysts emit: subject
9062
+ incorrect-step-<n>, metadata.block_first_step, trace:// evidence
9063
+ --out receipt root; one directory per finding + verifications.json
9064
+ --steps/--image/--cwd
9065
+ verify against one trajectory; the image must be replay-ready
9066
+ (uid-1000 derived), exactly like traces replay-verify
9067
+ --case trajectory id of --steps; findings citing other trajectories
9068
+ are honestly marked not-replayable
9069
+ --corpus resolve each finding's trajectory in CodeTraceBench corpora;
9070
+ images are derived via the batch uid-1000 preparer (docker)
9071
+ --fix-command corrected step for arm B; a reproduced finding whose fix
9072
+ flips becomes fix-flipped
9073
+ --base-url sandbox API url (default $SANDBOX_API_URL or ${DEFAULT_SANDBOX_BASE_URL})
9074
+ --api-key-env env var holding the sandbox API key (default SANDBOX_API_KEY)
9075
+
9076
+ Every finding gets a verdict: reproduced | fix-flipped | divergent | not-replayable,
9077
+ with a receipt directory carrying the executed evidence or the precise reason.
9078
+ Requires a running sandbox orchestrator when any finding is replayable \u2014 infra
9079
+ absence is an error, never a silent skip. See docs/replay-verify.md for setup.
9080
+ `;
9081
+ }
9082
+ function parseVerifyFindingsArgs(argv) {
9083
+ const values = /* @__PURE__ */ new Map();
9084
+ const corpora = [];
9085
+ for (let i = 0; i < argv.length; i++) {
9086
+ const arg = argv[i];
9087
+ if (arg === "--help" || arg === "-h") return "help";
9088
+ if (!arg.startsWith("--")) throw new Error(`unexpected positional argument: ${arg}`);
9089
+ const value = argv[++i];
9090
+ if (value === void 0) throw new Error(`missing value for ${arg}`);
9091
+ if (arg === "--corpus") {
9092
+ corpora.push(parseCorpusFlag(value));
9093
+ continue;
9094
+ }
9095
+ values.set(arg, value);
9096
+ }
9097
+ const required = (flag) => {
9098
+ const v = values.get(flag);
9099
+ if (v === void 0) throw new Error(`verify-findings: ${flag} is required`);
9100
+ return v;
9101
+ };
9102
+ const optionalNumber = (flag) => {
9103
+ const v = values.get(flag);
9104
+ if (v === void 0) return void 0;
9105
+ const n = Number(v);
9106
+ if (!Number.isFinite(n)) throw new Error(`verify-findings: ${flag} must be a number, got ${v}`);
9107
+ return n;
9108
+ };
9109
+ const findingsPath = required("--findings");
9110
+ const out = required("--out");
9111
+ const direct = values.has("--steps") || values.has("--image") || values.has("--cwd");
9112
+ if (direct && corpora.length > 0) {
9113
+ throw new Error("verify-findings: pass --steps/--image/--cwd or --corpus, not both");
9114
+ }
9115
+ let source;
9116
+ if (direct) {
9117
+ source = {
9118
+ kind: "direct",
9119
+ stepsPath: required("--steps"),
9120
+ image: required("--image"),
9121
+ cwd: required("--cwd"),
9122
+ ...values.has("--case") ? { caseId: values.get("--case") } : {}
9123
+ };
9124
+ } else if (corpora.length > 0) {
9125
+ source = { kind: "corpus", corpora };
9126
+ } else {
9127
+ throw new Error("verify-findings: a replay source is required \u2014 --steps/--image/--cwd or --corpus");
9128
+ }
9129
+ return {
9130
+ findingsPath,
9131
+ source,
9132
+ out,
9133
+ fixCommand: values.get("--fix-command"),
9134
+ stepTimeoutMs: optionalNumber("--step-timeout"),
9135
+ prefixLimit: optionalNumber("--prefix-limit"),
9136
+ baseUrl: values.get("--base-url") ?? process.env.SANDBOX_API_URL ?? DEFAULT_SANDBOX_BASE_URL,
9137
+ apiKeyEnv: values.get("--api-key-env") ?? "SANDBOX_API_KEY",
9138
+ maxLifetimeSeconds: optionalNumber("--max-lifetime")
9139
+ };
9140
+ }
9141
+ async function cmdVerifyFindings(argv) {
9142
+ const parsed = parseVerifyFindingsArgs(argv);
9143
+ if (parsed === "help") {
9144
+ process.stdout.write(verifyFindingsUsage());
9145
+ return;
9146
+ }
9147
+ const findings = readFindingsFile(parsed.findingsPath);
9148
+ const apiKey = process.env[parsed.apiKeyEnv];
9149
+ const run = await verifyFindings(findings, {
9150
+ source: parsed.source,
9151
+ out: parsed.out,
9152
+ fixCommand: parsed.fixCommand,
9153
+ stepTimeoutMs: parsed.stepTimeoutMs,
9154
+ prefixLimit: parsed.prefixLimit,
9155
+ apiKey,
9156
+ baseUrl: parsed.baseUrl,
9157
+ maxLifetimeSeconds: parsed.maxLifetimeSeconds,
9158
+ onProgress: (message) => process.stderr.write(`${message}
9159
+ `)
9160
+ });
9161
+ process.stdout.write(`${JSON.stringify(run, null, 2)}
9162
+ `);
9163
+ process.stderr.write(
9164
+ `verify-findings: ${run.verifications.length} finding(s) \u2192 ${run.counts.reproduced} reproduced, ${run.counts["fix-flipped"]} fix-flipped, ${run.counts.divergent} divergent, ${run.counts["not-replayable"]} not-replayable (${run.executions} execution(s)) \u2192 ${run.out}
9165
+ `
9166
+ );
9167
+ }
9168
+
8222
9169
  // src/external.ts
8223
9170
  import { spawn } from "child_process";
8224
9171
  import { readdir as readdir14, readFile as readFile7 } from "fs/promises";
@@ -8497,13 +9444,13 @@ function terminateWindowsProcessTree(child) {
8497
9444
  forceKill(child);
8498
9445
  return Promise.resolve();
8499
9446
  }
8500
- return new Promise((resolve7) => {
9447
+ return new Promise((resolve8) => {
8501
9448
  let finished = false;
8502
9449
  const finish = () => {
8503
9450
  if (finished) return;
8504
9451
  finished = true;
8505
9452
  forceKill(child);
8506
- resolve7();
9453
+ resolve8();
8507
9454
  };
8508
9455
  let killer;
8509
9456
  try {
@@ -8530,7 +9477,7 @@ function signalProcess(pid, signal) {
8530
9477
  }
8531
9478
  }
8532
9479
  function readProcessTableWithPs() {
8533
- return new Promise((resolve7, reject) => {
9480
+ return new Promise((resolve8, reject) => {
8534
9481
  let child;
8535
9482
  try {
8536
9483
  child = spawn("ps", ["-A", "-o", "pid=", "-o", "ppid="], {
@@ -8584,7 +9531,7 @@ function readProcessTableWithPs() {
8584
9531
  reject(new Error("ps: returned no process records"));
8585
9532
  return;
8586
9533
  }
8587
- resolve7(rows);
9534
+ resolve8(rows);
8588
9535
  });
8589
9536
  });
8590
9537
  });
@@ -8719,7 +9666,7 @@ function runCommand(command, args = [], opts = {}) {
8719
9666
  throw new RangeError("maxBuffer must be a positive safe integer");
8720
9667
  }
8721
9668
  if (opts.signal?.aborted) return Promise.reject(abortError(opts.signal));
8722
- return new Promise((resolve7, reject) => {
9669
+ return new Promise((resolve8, reject) => {
8723
9670
  let child;
8724
9671
  try {
8725
9672
  child = spawn(command, [...args], { detached: process.platform !== "win32" });
@@ -8751,7 +9698,7 @@ function runCommand(command, args = [], opts = {}) {
8751
9698
  } else if (spawnError) {
8752
9699
  reject(spawnError);
8753
9700
  } else {
8754
- resolve7({ stdout: out, stderr: err, code: closeCode });
9701
+ resolve8({ stdout: out, stderr: err, code: closeCode });
8755
9702
  }
8756
9703
  };
8757
9704
  function requestTermination(error) {
@@ -8983,18 +9930,18 @@ function commandRedactor(spec) {
8983
9930
  import { createHash as createHash9 } from "crypto";
8984
9931
  import { mkdir as mkdir3, mkdtemp as mkdtemp3, readFile as readFile8, readdir as readdir15, writeFile as writeFile4 } from "fs/promises";
8985
9932
  import { endianness, tmpdir as tmpdir4 } from "os";
8986
- import { join as join22, resolve as resolve5 } from "path";
9933
+ import { join as join23, resolve as resolve6 } from "path";
8987
9934
  var HODOSCOPE_VERSION = "0.2.4";
8988
9935
  async function writeHodoscopeInput(otlpPath, options = {}) {
8989
- const explicitDirectory = options.directory ? resolve5(options.directory) : void 0;
8990
- const directory = explicitDirectory ?? await mkdtemp3(join22(tmpdir4(), "traces-hodoscope-input-"));
9936
+ const explicitDirectory = options.directory ? resolve6(options.directory) : void 0;
9937
+ const directory = explicitDirectory ?? await mkdtemp3(join23(tmpdir4(), "traces-hodoscope-input-"));
8991
9938
  if (explicitDirectory) {
8992
9939
  await mkdir3(directory, { recursive: true });
8993
9940
  if ((await readdir15(directory)).length > 0) {
8994
9941
  throw new Error(`Hodoscope explicit directory must be empty: ${directory}`);
8995
9942
  }
8996
9943
  }
8997
- const samples = join22(directory, "samples");
9944
+ const samples = join23(directory, "samples");
8998
9945
  await mkdir3(samples);
8999
9946
  const rows = [];
9000
9947
  for await (const value of readJsonl(otlpPath)) rows.push(parseOpenInferenceRow(value));
@@ -9020,7 +9967,7 @@ async function writeHodoscopeInput(otlpPath, options = {}) {
9020
9967
  }
9021
9968
  actionCount += actions;
9022
9969
  const file = `${createHash9("sha256").update(traceId).digest("hex").slice(0, 24)}.json`;
9023
- await writeFile4(join22(samples, file), `${JSON.stringify(trajectory, null, 2)}
9970
+ await writeFile4(join23(samples, file), `${JSON.stringify(trajectory, null, 2)}
9024
9971
  `, "utf8");
9025
9972
  }
9026
9973
  return {
@@ -9042,12 +9989,12 @@ function hodoscopeAnalyzer(options = {}) {
9042
9989
  return {
9043
9990
  name: "hodoscope",
9044
9991
  async analyze(otlpPath, runOptions2 = {}) {
9045
- const outputRoot = options.outputDir ? resolve5(options.outputDir) : tmpdir4();
9992
+ const outputRoot = options.outputDir ? resolve6(options.outputDir) : tmpdir4();
9046
9993
  await mkdir3(outputRoot, { recursive: true });
9047
- const directory = await mkdtemp3(join22(outputRoot, "traces-hodoscope-"));
9048
- const input = join22(directory, "input");
9049
- const analysisFile = join22(directory, "analysis.hodoscope.json");
9050
- const samplesFile = join22(directory, "samples.json");
9994
+ const directory = await mkdtemp3(join23(outputRoot, "traces-hodoscope-"));
9995
+ const input = join23(directory, "input");
9996
+ const analysisFile = join23(directory, "analysis.hodoscope.json");
9997
+ const samplesFile = join23(directory, "samples.json");
9051
9998
  try {
9052
9999
  const prepared = await writeHodoscopeInput(otlpPath, {
9053
10000
  directory: input,
@@ -9246,7 +10193,7 @@ async function readHodoscopeAnalysis(path, expected) {
9246
10193
  throw new Error("Hodoscope analysis.created_at must be a valid date-time");
9247
10194
  }
9248
10195
  const source = requiredString2(value.source, "analysis.source");
9249
- if (resolve5(source) !== resolve5(expected.source)) {
10196
+ if (resolve6(source) !== resolve6(expected.source)) {
9250
10197
  throw new Error(
9251
10198
  `Hodoscope analysis.source '${source}' does not match requested input '${expected.source}'`
9252
10199
  );
@@ -9759,7 +10706,7 @@ async function applyRedactor(spans, redactor) {
9759
10706
  import { createHash as createHash10 } from "crypto";
9760
10707
  import { mkdtemp as mkdtemp4, readFile as readFile9, writeFile as writeFile5 } from "fs/promises";
9761
10708
  import { tmpdir as tmpdir5 } from "os";
9762
- import { join as join23 } from "path";
10709
+ import { join as join24 } from "path";
9763
10710
  import {
9764
10711
  firstNumberAttr,
9765
10712
  LLM_CACHED_TOKEN_ATTR_KEYS,
@@ -10355,7 +11302,7 @@ async function exportTraceEvidenceFile(inputPath, opts = {}) {
10355
11302
  }
10356
11303
  async function writeTraceEvidenceExportFile(inputPath, outPath, opts = {}) {
10357
11304
  const result = await exportTraceEvidenceFile(inputPath, opts);
10358
- const path = outPath ?? join23(await mkdtemp4(join23(tmpdir5(), "traces-export-")), "spans.openinference.jsonl");
11305
+ const path = outPath ?? join24(await mkdtemp4(join24(tmpdir5(), "traces-export-")), "spans.openinference.jsonl");
10359
11306
  await writeFile5(path, serializeSpans(result.spans), "utf8");
10360
11307
  return { ...result, path };
10361
11308
  }
@@ -10363,7 +11310,7 @@ async function writeTraceEvidenceExportFile(inputPath, outPath, opts = {}) {
10363
11310
  // src/inspect.ts
10364
11311
  import { mkdtemp as mkdtemp5, readFile as readFile10, writeFile as writeFile6 } from "fs/promises";
10365
11312
  import { tmpdir as tmpdir6 } from "os";
10366
- import { basename as basename13, join as join24 } from "path";
11313
+ import { basename as basename13, join as join25 } from "path";
10367
11314
  var severityRank = {
10368
11315
  high: 0,
10369
11316
  medium: 1,
@@ -10652,14 +11599,14 @@ async function readSessionIndexFile(path) {
10652
11599
  return parsed;
10653
11600
  }
10654
11601
  async function writeInspectionReportFile(report, outPath) {
10655
- const path = outPath ?? join24(await mkdtemp5(join24(tmpdir6(), "traces-inspect-")), "inspection-report.md");
11602
+ const path = outPath ?? join25(await mkdtemp5(join25(tmpdir6(), "traces-inspect-")), "inspection-report.md");
10656
11603
  await writeFile6(path, renderInspectionReport(report), "utf8");
10657
11604
  return path;
10658
11605
  }
10659
11606
 
10660
11607
  // src/adoption.ts
10661
11608
  import { readFile as readFile11 } from "fs/promises";
10662
- import { join as join25 } from "path";
11609
+ import { join as join26 } from "path";
10663
11610
  function toolName2(s) {
10664
11611
  const n = s.attributes["tool.name"];
10665
11612
  if (typeof n === "string") return n;
@@ -10779,7 +11726,7 @@ async function readLoopRuns(cwds, sessionIds) {
10779
11726
  seenCwds.add(cwd);
10780
11727
  const firstSourceByRun = /* @__PURE__ */ new Map();
10781
11728
  for (const artifactDir of [".agent", ".evolve"]) {
10782
- const path = join25(cwd, artifactDir, "skill-runs.jsonl");
11729
+ const path = join26(cwd, artifactDir, "skill-runs.jsonl");
10783
11730
  let raw;
10784
11731
  try {
10785
11732
  raw = await readFile11(path, "utf8");
@@ -12191,7 +13138,7 @@ function renderAdoption(ar) {
12191
13138
  // src/improvement.ts
12192
13139
  import { access, mkdir as mkdir4, mkdtemp as mkdtemp6, writeFile as writeFile7 } from "fs/promises";
12193
13140
  import { tmpdir as tmpdir7 } from "os";
12194
- import { dirname as dirname7, join as join26, resolve as resolve6 } from "path";
13141
+ import { dirname as dirname7, join as join27, resolve as resolve7 } from "path";
12195
13142
  import { pathToFileURL } from "url";
12196
13143
  import {
12197
13144
  AnalystRegistry,
@@ -12549,7 +13496,7 @@ function registryFromConfig(config) {
12549
13496
  async function existingConfigPath(path) {
12550
13497
  const candidates = path ? [path] : ["traces.config.mjs", "traces.config.js", "traces.config.cjs"];
12551
13498
  for (const candidate of candidates) {
12552
- const resolved = resolve6(candidate);
13499
+ const resolved = resolve7(candidate);
12553
13500
  try {
12554
13501
  await access(resolved);
12555
13502
  return resolved;
@@ -12698,13 +13645,13 @@ async function runTraceStoreInvestigation(opts) {
12698
13645
  };
12699
13646
  }
12700
13647
  async function writeTraceImprovementArtifacts(result, outDir) {
12701
- const directory = outDir ? resolve6(outDir) : await mkdtemp6(join26(tmpdir7(), "traces-improvement-"));
13648
+ const directory = outDir ? resolve7(outDir) : await mkdtemp6(join27(tmpdir7(), "traces-improvement-"));
12702
13649
  await mkdir4(directory, { recursive: true });
12703
13650
  const paths = {
12704
13651
  directory,
12705
- result: join26(directory, "result.json"),
12706
- evidence: join26(directory, "evidence.jsonl"),
12707
- report: join26(directory, "report.md"),
13652
+ result: join27(directory, "result.json"),
13653
+ evidence: join27(directory, "evidence.jsonl"),
13654
+ report: join27(directory, "report.md"),
12708
13655
  traces: result.otlpPath
12709
13656
  };
12710
13657
  const { report, ...machineResult } = result;
@@ -12716,11 +13663,11 @@ async function writeTraceImprovementArtifacts(result, outDir) {
12716
13663
  return paths;
12717
13664
  }
12718
13665
  async function runTraceImprovement(opts) {
12719
- const directory = opts.outDir ? resolve6(opts.outDir) : await mkdtemp6(join26(tmpdir7(), "traces-improvement-"));
13666
+ const directory = opts.outDir ? resolve7(opts.outDir) : await mkdtemp6(join27(tmpdir7(), "traces-improvement-"));
12720
13667
  await mkdir4(directory, { recursive: true });
12721
13668
  const investigation = await runTraceInvestigation({
12722
13669
  ...opts,
12723
- otlpOutPath: opts.otlpOutPath ?? join26(directory, "traces.otlp.jsonl")
13670
+ otlpOutPath: opts.otlpOutPath ?? join27(directory, "traces.otlp.jsonl")
12724
13671
  });
12725
13672
  const result = {
12726
13673
  ...investigation,
@@ -12732,7 +13679,7 @@ async function runTraceImprovement(opts) {
12732
13679
  };
12733
13680
  }
12734
13681
  async function saveReport(path, report) {
12735
- await mkdir4(dirname7(resolve6(path)), { recursive: true });
13682
+ await mkdir4(dirname7(resolve7(path)), { recursive: true });
12736
13683
  await writeFile7(path, report, "utf8");
12737
13684
  }
12738
13685
 
@@ -12741,18 +13688,18 @@ import { createHash as createHash11 } from "crypto";
12741
13688
  var VERIFY_RE = /\b(test|tests|vitest|jest|pytest|go test|cargo test|tsc|typecheck|lint|biome|eslint|build|check:invariants|preflight)\b/i;
12742
13689
  var CHANGE_RE = /\b(apply_patch|edit|write|create|delete|rename|replace|update|git apply|sed -i|perl -pi|tee\s|cat\s+>)/i;
12743
13690
  var CLAIM_DONE_RE = /\b(done|fixed|complete|completed|implemented|merged|verified|all green|tests pass|ready|works now)\b/i;
12744
- var sleep = (ms2, signal) => new Promise((resolve7) => {
13691
+ var sleep = (ms2, signal) => new Promise((resolve8) => {
12745
13692
  if (signal?.aborted) {
12746
- resolve7();
13693
+ resolve8();
12747
13694
  return;
12748
13695
  }
12749
13696
  const onAbort = () => {
12750
13697
  clearTimeout(t);
12751
- resolve7();
13698
+ resolve8();
12752
13699
  };
12753
13700
  const t = setTimeout(() => {
12754
13701
  signal?.removeEventListener("abort", onAbort);
12755
- resolve7();
13702
+ resolve8();
12756
13703
  }, ms2);
12757
13704
  signal?.addEventListener("abort", onAbort, { once: true });
12758
13705
  });
@@ -13163,7 +14110,7 @@ async function streamSessions(opts) {
13163
14110
 
13164
14111
  // src/supervisor-run-context.ts
13165
14112
  import { readFile as readFile12, stat as stat14 } from "fs/promises";
13166
- import { basename as basename14, join as join27 } from "path";
14113
+ import { basename as basename14, join as join28 } from "path";
13167
14114
  import { materializeTreeView } from "@tangle-network/agent-runtime/kernel";
13168
14115
  import {
13169
14116
  NO_SOURCE_LIMITS
@@ -13324,10 +14271,10 @@ function parseInstant(v) {
13324
14271
  return Number.isFinite(t) ? t : null;
13325
14272
  }
13326
14273
  async function readFileRunContext(runDir, opts = {}) {
13327
- const journalText = await readMaybe(join27(runDir, SPAWN_JOURNAL_FILE));
14274
+ const journalText = await readMaybe(join28(runDir, SPAWN_JOURNAL_FILE));
13328
14275
  const journal = readRunContextJournal(journalText);
13329
- const coordination = readCoordinationLog(await readMaybe(join27(runDir, COORDINATION_LOG_FILE)));
13330
- const result = await readMaybe(join27(runDir, RESULT_FILE));
14276
+ const coordination = readCoordinationLog(await readMaybe(join28(runDir, COORDINATION_LOG_FILE)));
14277
+ const result = await readMaybe(join28(runDir, RESULT_FILE));
13331
14278
  return {
13332
14279
  runRef: runDir,
13333
14280
  instanceId: opts.instanceId !== void 0 ? opts.instanceId : journal.root,
@@ -13350,7 +14297,7 @@ async function readFileRunContext(runDir, opts = {}) {
13350
14297
  harnessWorkerTokens: null,
13351
14298
  harnessMissingReason: "the file run context journals each child\u2019s own settled spend, so no external harness join is used",
13352
14299
  limits: sourceLimits(journal, coordination),
13353
- rootTranscriptRef: journal.present ? join27(runDir, SPAWN_JOURNAL_FILE) : null,
14300
+ rootTranscriptRef: journal.present ? join28(runDir, SPAWN_JOURNAL_FILE) : null,
13354
14301
  traceCommand: `traces watch ${runDir}`
13355
14302
  };
13356
14303
  }
@@ -13358,7 +14305,7 @@ function fileRunContextSupervisorRunReader(runDir, opts = {}) {
13358
14305
  return { runRef: runDir, read: () => readFileRunContext(runDir, opts) };
13359
14306
  }
13360
14307
  async function isFileRunContextDir(runDir) {
13361
- return stat14(join27(runDir, SPAWN_JOURNAL_FILE)).then((s) => s.isFile()).catch(() => false);
14308
+ return stat14(join28(runDir, SPAWN_JOURNAL_FILE)).then((s) => s.isFile()).catch(() => false);
13362
14309
  }
13363
14310
  function flattenJournal(journal) {
13364
14311
  return journal.events.map((event) => {
@@ -13504,9 +14451,9 @@ function mergeRolls(a, b) {
13504
14451
  };
13505
14452
  }
13506
14453
  async function readRunContextSnapshot(runDir) {
13507
- const journal = readRunContextJournal(await readMaybe(join27(runDir, SPAWN_JOURNAL_FILE)));
13508
- const coordination = readCoordinationLog(await readMaybe(join27(runDir, COORDINATION_LOG_FILE)));
13509
- const resultText = await readMaybe(join27(runDir, RESULT_FILE));
14454
+ const journal = readRunContextJournal(await readMaybe(join28(runDir, SPAWN_JOURNAL_FILE)));
14455
+ const coordination = readCoordinationLog(await readMaybe(join28(runDir, COORDINATION_LOG_FILE)));
14456
+ const resultText = await readMaybe(join28(runDir, RESULT_FILE));
13510
14457
  const result = parseResult(resultText);
13511
14458
  const view = safeTreeView(journal.events);
13512
14459
  const nodes = view === null ? [] : orderNodes(view, journal);
@@ -13689,7 +14636,7 @@ function wrap(text2, indent) {
13689
14636
 
13690
14637
  // src/run-span-tree.ts
13691
14638
  import { readdir as readdir16 } from "fs/promises";
13692
- import { join as join28 } from "path";
14639
+ import { join as join29 } from "path";
13693
14640
  import {
13694
14641
  firstNumberAttr as firstNumberAttr3,
13695
14642
  LLM_COST_ATTR_KEYS as LLM_COST_ATTR_KEYS2,
@@ -13962,7 +14909,7 @@ async function findSpanSourceFiles(dir) {
13962
14909
  const found = [];
13963
14910
  const entries = await readdir16(dir, { withFileTypes: true }).catch(() => []);
13964
14911
  for (const entry of entries) {
13965
- const child = join28(dir, entry.name);
14912
+ const child = join29(dir, entry.name);
13966
14913
  if (entry.isFile()) {
13967
14914
  if (SPAN_FILE_PATTERN.test(entry.name)) found.push(child);
13968
14915
  continue;
@@ -13970,7 +14917,7 @@ async function findSpanSourceFiles(dir) {
13970
14917
  if (!entry.isDirectory()) continue;
13971
14918
  const nested = await readdir16(child, { withFileTypes: true }).catch(() => []);
13972
14919
  for (const inner of nested) {
13973
- if (inner.isFile() && SPAN_FILE_PATTERN.test(inner.name)) found.push(join28(child, inner.name));
14920
+ if (inner.isFile() && SPAN_FILE_PATTERN.test(inner.name)) found.push(join29(child, inner.name));
13974
14921
  }
13975
14922
  }
13976
14923
  return found.sort();
@@ -13978,7 +14925,7 @@ async function findSpanSourceFiles(dir) {
13978
14925
 
13979
14926
  // src/supervisor-run-watch.ts
13980
14927
  import { readdir as readdir17 } from "fs/promises";
13981
- import { join as join29 } from "path";
14928
+ import { join as join30 } from "path";
13982
14929
  function budget(node) {
13983
14930
  const parts = [`${int(node.budget.maxIterations)}it`, `${int(node.budget.maxTokens)}tok`];
13984
14931
  if (node.budget.maxUsd !== void 0) parts.push(`$${node.budget.maxUsd.toFixed(2)}`);
@@ -14122,7 +15069,7 @@ async function resolveRunContextDir(dir) {
14122
15069
  const candidates = [];
14123
15070
  for (const entry of entries) {
14124
15071
  if (!entry.isDirectory()) continue;
14125
- const child = join29(dir, entry.name);
15072
+ const child = join30(dir, entry.name);
14126
15073
  if (await isFileRunContextDir(child)) candidates.push(child);
14127
15074
  }
14128
15075
  if (candidates.length === 1) return candidates[0];
@@ -14227,13 +15174,13 @@ function aborted(signal) {
14227
15174
  return signal !== void 0 && signal.aborted;
14228
15175
  }
14229
15176
  function sleep2(msDelay, signal) {
14230
- return new Promise((resolve7) => {
14231
- const timer = setTimeout(resolve7, msDelay);
15177
+ return new Promise((resolve8) => {
15178
+ const timer = setTimeout(resolve8, msDelay);
14232
15179
  signal?.addEventListener(
14233
15180
  "abort",
14234
15181
  () => {
14235
15182
  clearTimeout(timer);
14236
- resolve7();
15183
+ resolve8();
14237
15184
  },
14238
15185
  { once: true }
14239
15186
  );
@@ -14702,7 +15649,7 @@ async function collectSessionSelection(groups, options = {}) {
14702
15649
  // src/session-index.ts
14703
15650
  import { mkdtemp as mkdtemp7, readdir as readdir18, readFile as readFile13, stat as stat16, writeFile as writeFile8 } from "fs/promises";
14704
15651
  import { tmpdir as tmpdir8 } from "os";
14705
- import { dirname as dirname8, join as join30 } from "path";
15652
+ import { dirname as dirname8, join as join31 } from "path";
14706
15653
  function indexRow(record2) {
14707
15654
  return {
14708
15655
  session: record2.session,
@@ -14775,7 +15722,7 @@ async function findContextRoot(cwd) {
14775
15722
  if (!s) return null;
14776
15723
  if (!s.isDirectory()) current = dirname8(current);
14777
15724
  for (; ; ) {
14778
- if (await pathStat2(join30(current, ".git")) || await dirExists(join30(current, ".evolve")) || await fileExists(join30(current, "AGENTS.md")) || await fileExists(join30(current, "CLAUDE.md"))) {
15725
+ if (await pathStat2(join31(current, ".git")) || await dirExists(join31(current, ".evolve")) || await fileExists(join31(current, "AGENTS.md")) || await fileExists(join31(current, "CLAUDE.md"))) {
14779
15726
  return current;
14780
15727
  }
14781
15728
  const parent = dirname8(current);
@@ -14843,7 +15790,7 @@ async function walkMarkdown(root, kind, maxFiles = 100) {
14843
15790
  }
14844
15791
  for (const entry of entries) {
14845
15792
  if (out.length >= maxFiles) return;
14846
- const path = join30(dir, entry.name);
15793
+ const path = join31(dir, entry.name);
14847
15794
  if (entry.isDirectory()) {
14848
15795
  await walk(path);
14849
15796
  } else if (entry.isFile() && entry.name.endsWith(".md")) {
@@ -14858,20 +15805,20 @@ async function walkMarkdown(root, kind, maxFiles = 100) {
14858
15805
  async function collectContextRoot(root) {
14859
15806
  const files = [];
14860
15807
  for (const name of ["AGENTS.md", "CLAUDE.md", "GEMINI.md"]) {
14861
- const summary = await summarizeFile(join30(root, name), "instruction-doc");
15808
+ const summary = await summarizeFile(join31(root, name), "instruction-doc");
14862
15809
  if (summary) files.push(summary);
14863
15810
  }
14864
- const evolve = join30(root, ".evolve");
15811
+ const evolve = join31(root, ".evolve");
14865
15812
  for (const name of ["skill-runs.jsonl", "governor.jsonl", "experiments.jsonl"]) {
14866
- const summary = await summarizeFile(join30(evolve, name), "evolve-jsonl");
15813
+ const summary = await summarizeFile(join31(evolve, name), "evolve-jsonl");
14867
15814
  if (summary) files.push(summary);
14868
15815
  }
14869
15816
  for (const name of ["scorecard.json"]) {
14870
- const summary = await summarizeFile(join30(evolve, name), "evolve-json");
15817
+ const summary = await summarizeFile(join31(evolve, name), "evolve-json");
14871
15818
  if (summary) files.push(summary);
14872
15819
  }
14873
- files.push(...await walkMarkdown(join30(evolve, "reflections"), "reflection"));
14874
- files.push(...await walkMarkdown(join30(evolve, "handoffs"), "handoff"));
15820
+ files.push(...await walkMarkdown(join31(evolve, "reflections"), "reflection"));
15821
+ files.push(...await walkMarkdown(join31(evolve, "handoffs"), "handoff"));
14875
15822
  return { root, files: files.sort((a, b) => a.path.localeCompare(b.path)) };
14876
15823
  }
14877
15824
  async function collectContextIndex(records) {
@@ -14917,7 +15864,7 @@ function serializeSessionIndex(index) {
14917
15864
  `;
14918
15865
  }
14919
15866
  async function writeSessionIndexFile(index, outPath) {
14920
- const path = outPath ?? join30(await mkdtemp7(join30(tmpdir8(), "traces-index-")), "session-index.json");
15867
+ const path = outPath ?? join31(await mkdtemp7(join31(tmpdir8(), "traces-index-")), "session-index.json");
14921
15868
  await writeFile8(path, serializeSessionIndex(index), "utf8");
14922
15869
  return path;
14923
15870
  }
@@ -14925,11 +15872,11 @@ async function writeSessionIndexFile(index, outPath) {
14925
15872
  // src/upload-state.ts
14926
15873
  import { mkdir as mkdir5, readFile as readFile14, writeFile as writeFile9 } from "fs/promises";
14927
15874
  import { homedir as homedir12 } from "os";
14928
- import { dirname as dirname9, join as join31 } from "path";
15875
+ import { dirname as dirname9, join as join32 } from "path";
14929
15876
  import { contentHash } from "@tangle-network/agent-eval";
14930
15877
  function stateFile() {
14931
- const base = process.env.XDG_STATE_HOME ?? join31(homedir12(), ".local", "state");
14932
- return join31(base, "tangle-traces", "uploaded.json");
15878
+ const base = process.env.XDG_STATE_HOME ?? join32(homedir12(), ".local", "state");
15879
+ return join32(base, "tangle-traces", "uploaded.json");
14933
15880
  }
14934
15881
  function uploadKey(harness, sessionId) {
14935
15882
  return `${harness}:${sessionId}`;
@@ -14961,7 +15908,7 @@ function alreadyUploaded(state, harness, sessionId, hash2) {
14961
15908
  import { readFile as readFile15 } from "fs/promises";
14962
15909
  import { createRequire as createRequire2 } from "module";
14963
15910
  import { hostname as hostname2 } from "os";
14964
- import { basename as basename15, join as join32 } from "path";
15911
+ import { basename as basename15, join as join33 } from "path";
14965
15912
  import { hostedClientFromEnv } from "@tangle-network/agent-eval/hosted";
14966
15913
  import { REDACTION_VERSION } from "@tangle-network/agent-eval/traces";
14967
15914
  var require2 = createRequire2(import.meta.url);
@@ -14985,7 +15932,7 @@ async function planUpload(opts = {}) {
14985
15932
  async function gitBranch(cwd) {
14986
15933
  if (!cwd) return void 0;
14987
15934
  try {
14988
- const head = await readFile15(join32(cwd, ".git", "HEAD"), "utf8");
15935
+ const head = await readFile15(join33(cwd, ".git", "HEAD"), "utf8");
14989
15936
  const m = head.match(/^ref:\s*refs\/heads\/(.+)$/m);
14990
15937
  return m?.[1]?.trim();
14991
15938
  } catch {
@@ -15249,9 +16196,13 @@ export {
15249
16196
  resolveCaseResources,
15250
16197
  enumerateReplayableCases,
15251
16198
  zaiChatCaller,
16199
+ clipText,
15252
16200
  buildFixPrompt,
16201
+ buildRetryFixPrompt,
16202
+ countScriptCommands,
15253
16203
  extractFixCommand,
15254
16204
  generateFixCommand,
16205
+ runFixLoop,
15255
16206
  derivedImageTag,
15256
16207
  dockerImagePreparer,
15257
16208
  seededSample,
@@ -15260,6 +16211,21 @@ export {
15260
16211
  parseReplayBatchArgs,
15261
16212
  replayBatchUsage,
15262
16213
  cmdReplayVerifyBatch,
16214
+ parseIncorrectStepsSubject,
16215
+ resolveFindingInvocation,
16216
+ replayVerifyFinding,
16217
+ findingReplayStep,
16218
+ findingTrajectoryId,
16219
+ resolveFindingReplayability,
16220
+ classifyVerdict,
16221
+ DEFAULT_SANDBOX_BASE_URL,
16222
+ assertSandboxReachable,
16223
+ verifyFindings,
16224
+ renderVerifiedFindingsSection,
16225
+ readFindingsFile,
16226
+ verifyFindingsUsage,
16227
+ parseVerifyFindingsArgs,
16228
+ cmdVerifyFindings,
15263
16229
  runCommand,
15264
16230
  commandAnalyzer,
15265
16231
  haloAnalyzer,