@tangle-network/traces 0.11.3 → 0.11.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/dist/{chunk-CK5F5TWO.js → chunk-VTYOCFGF.js} +1094 -128
- package/dist/cli.js +64 -5
- package/dist/index.d.ts +296 -5
- package/dist/index.js +36 -74
- package/docs/replay-verify.md +17 -0
- package/docs/trace-analysts.md +28 -0
- package/package.json +4 -4
|
@@ -766,8 +766,8 @@ function serializeSpans(spans) {
|
|
|
766
766
|
async function writeOtlpFile(spans, outPath) {
|
|
767
767
|
const { mkdtemp: mkdtemp8, writeFile: writeFile10 } = await import("fs/promises");
|
|
768
768
|
const { tmpdir: tmpdir9 } = await import("os");
|
|
769
|
-
const { join:
|
|
770
|
-
const path = outPath ??
|
|
769
|
+
const { join: join34 } = await import("path");
|
|
770
|
+
const path = outPath ?? join34(await mkdtemp8(join34(tmpdir9(), "traces-")), "spans.otlp.jsonl");
|
|
771
771
|
await writeFile10(path, serializeSpans(spans), "utf8");
|
|
772
772
|
return path;
|
|
773
773
|
}
|
|
@@ -5503,7 +5503,7 @@ var OpencodeAdapter = class {
|
|
|
5503
5503
|
// src/adapters/pi.ts
|
|
5504
5504
|
import { readdir as readdir12, stat as stat11 } from "fs/promises";
|
|
5505
5505
|
import { homedir as homedir10 } from "os";
|
|
5506
|
-
import { basename as basename11, join as join15 } from "path";
|
|
5506
|
+
import { basename as basename11, join as join15, resolve as resolve5 } from "path";
|
|
5507
5507
|
var SERVICE9 = "pi";
|
|
5508
5508
|
function terminalStatus(message) {
|
|
5509
5509
|
const stopReason = message?.stopReason ?? "missing";
|
|
@@ -5548,45 +5548,77 @@ function completeToolSpan(toolSpan, result) {
|
|
|
5548
5548
|
}
|
|
5549
5549
|
var PiAdapter = class {
|
|
5550
5550
|
harness = "pi";
|
|
5551
|
+
sessionsRoot;
|
|
5552
|
+
constructor(options = {}) {
|
|
5553
|
+
this.sessionsRoot = resolve5(options.sessionsRoot ?? join15(homedir10(), ".pi", "agent", "sessions"));
|
|
5554
|
+
}
|
|
5551
5555
|
root() {
|
|
5552
|
-
return
|
|
5556
|
+
return this.sessionsRoot;
|
|
5553
5557
|
}
|
|
5554
5558
|
async locate(opts = {}) {
|
|
5555
5559
|
const root = this.root();
|
|
5556
|
-
let
|
|
5560
|
+
let entries;
|
|
5557
5561
|
try {
|
|
5558
|
-
|
|
5562
|
+
entries = await readdir12(root, { withFileTypes: true });
|
|
5559
5563
|
} catch (error) {
|
|
5560
5564
|
if (isMissingPathError(error)) return [];
|
|
5561
5565
|
throw error;
|
|
5562
5566
|
}
|
|
5563
|
-
const
|
|
5564
|
-
for (const
|
|
5565
|
-
|
|
5567
|
+
const paths = [];
|
|
5568
|
+
for (const entry of entries) {
|
|
5569
|
+
if (entry.isFile() && entry.name.endsWith(".jsonl")) {
|
|
5570
|
+
paths.push(join15(root, entry.name));
|
|
5571
|
+
continue;
|
|
5572
|
+
}
|
|
5573
|
+
if (!entry.isDirectory()) continue;
|
|
5574
|
+
const dir = join15(root, entry.name);
|
|
5566
5575
|
let files;
|
|
5567
5576
|
try {
|
|
5568
|
-
files = await readdir12(
|
|
5577
|
+
files = await readdir12(dir);
|
|
5569
5578
|
} catch (error) {
|
|
5570
5579
|
if (isMissingPathError(error)) continue;
|
|
5571
5580
|
throw error;
|
|
5572
5581
|
}
|
|
5573
|
-
const
|
|
5574
|
-
|
|
5575
|
-
for (const f of files) {
|
|
5576
|
-
if (!f.endsWith(".jsonl")) continue;
|
|
5577
|
-
const path = join15(dp, f);
|
|
5578
|
-
let st;
|
|
5579
|
-
try {
|
|
5580
|
-
st = await stat11(path);
|
|
5581
|
-
} catch (error) {
|
|
5582
|
-
if (isMissingPathError(error)) continue;
|
|
5583
|
-
throw error;
|
|
5584
|
-
}
|
|
5585
|
-
if (opts.sinceMs && st.mtimeMs < opts.sinceMs) continue;
|
|
5586
|
-
const id = basename11(f, ".jsonl").replace(/^[\dTZ.-]+_/, "");
|
|
5587
|
-
refs.push({ harness: this.harness, sessionId: id, path, cwd, mtimeMs: st.mtimeMs });
|
|
5582
|
+
for (const file of files) {
|
|
5583
|
+
if (file.endsWith(".jsonl")) paths.push(join15(dir, file));
|
|
5588
5584
|
}
|
|
5589
5585
|
}
|
|
5586
|
+
const refs = [];
|
|
5587
|
+
for (const path of paths) {
|
|
5588
|
+
const f = basename11(path);
|
|
5589
|
+
let st;
|
|
5590
|
+
try {
|
|
5591
|
+
st = await stat11(path);
|
|
5592
|
+
} catch (error) {
|
|
5593
|
+
if (isMissingPathError(error)) continue;
|
|
5594
|
+
throw error;
|
|
5595
|
+
}
|
|
5596
|
+
if (opts.sinceMs && st.mtimeMs < opts.sinceMs) continue;
|
|
5597
|
+
let id = basename11(f, ".jsonl").replace(/^[\dTZ.-]+_/, "");
|
|
5598
|
+
let cwd = null;
|
|
5599
|
+
const ref = {
|
|
5600
|
+
harness: this.harness,
|
|
5601
|
+
sessionId: id,
|
|
5602
|
+
path,
|
|
5603
|
+
cwd,
|
|
5604
|
+
mtimeMs: st.mtimeMs
|
|
5605
|
+
};
|
|
5606
|
+
const [session] = await takeJsonl(path, 1, sessionJsonlOptions(ref));
|
|
5607
|
+
if (session?.type === "session") {
|
|
5608
|
+
if (session.id) id = session.id;
|
|
5609
|
+
if (session.cwd) cwd = session.cwd;
|
|
5610
|
+
}
|
|
5611
|
+
ref.sessionId = id;
|
|
5612
|
+
ref.cwd = cwd;
|
|
5613
|
+
if (ref.integrity) {
|
|
5614
|
+
ref.integrity.corruptions = ref.integrity.corruptions.map((receipt) => ({
|
|
5615
|
+
...receipt,
|
|
5616
|
+
sessionId: id
|
|
5617
|
+
}));
|
|
5618
|
+
}
|
|
5619
|
+
if (opts.cwd && (!cwd || !cwd.startsWith(opts.cwd))) continue;
|
|
5620
|
+
refs.push(ref);
|
|
5621
|
+
}
|
|
5590
5622
|
return refs.sort((a, b) => b.mtimeMs - a.mtimeMs);
|
|
5591
5623
|
}
|
|
5592
5624
|
async parse(ref, options = {}) {
|
|
@@ -6332,7 +6364,7 @@ async function stampSpanWorkdirRepoAttrs(spans) {
|
|
|
6332
6364
|
async function awaitWithAbort(operation, signal) {
|
|
6333
6365
|
if (!signal) return operation;
|
|
6334
6366
|
signal.throwIfAborted();
|
|
6335
|
-
return new Promise((
|
|
6367
|
+
return new Promise((resolve8, reject) => {
|
|
6336
6368
|
const stop = () => {
|
|
6337
6369
|
signal.removeEventListener("abort", stop);
|
|
6338
6370
|
reject(signal.reason);
|
|
@@ -6341,7 +6373,7 @@ async function awaitWithAbort(operation, signal) {
|
|
|
6341
6373
|
operation.then(
|
|
6342
6374
|
(value) => {
|
|
6343
6375
|
signal.removeEventListener("abort", stop);
|
|
6344
|
-
|
|
6376
|
+
resolve8(value);
|
|
6345
6377
|
},
|
|
6346
6378
|
(error) => {
|
|
6347
6379
|
signal.removeEventListener("abort", stop);
|
|
@@ -7387,7 +7419,7 @@ function zaiChatCaller(options) {
|
|
|
7387
7419
|
}
|
|
7388
7420
|
};
|
|
7389
7421
|
}
|
|
7390
|
-
function
|
|
7422
|
+
function clipText(text2, limit) {
|
|
7391
7423
|
if (text2.length <= limit) return text2;
|
|
7392
7424
|
const half = Math.floor(limit / 2);
|
|
7393
7425
|
return `${text2.slice(0, half)}
|
|
@@ -7396,11 +7428,11 @@ ${text2.slice(-half)}`;
|
|
|
7396
7428
|
}
|
|
7397
7429
|
function renderStep(step, marker) {
|
|
7398
7430
|
const rc = parseRecordedReturncode(step.observation);
|
|
7399
|
-
const output =
|
|
7431
|
+
const output = clipText(parseObservationOutput(step.observation).trim(), 1600);
|
|
7400
7432
|
return [
|
|
7401
7433
|
`### step ${step.step_id}${marker}`,
|
|
7402
7434
|
"```sh",
|
|
7403
|
-
|
|
7435
|
+
clipText(step.action, 2e3),
|
|
7404
7436
|
"```",
|
|
7405
7437
|
`returncode: ${rc ?? "none recorded"}`,
|
|
7406
7438
|
output.length > 0 ? `output:
|
|
@@ -7409,13 +7441,25 @@ ${output}
|
|
|
7409
7441
|
\`\`\`` : "output: (empty)"
|
|
7410
7442
|
].join("\n");
|
|
7411
7443
|
}
|
|
7412
|
-
function
|
|
7444
|
+
function promptBody(input) {
|
|
7413
7445
|
const radius = input.contextRadius ?? 3;
|
|
7414
7446
|
const target = input.steps.find((s) => s.step_id === input.k);
|
|
7415
7447
|
if (!target) throw new Error(`replay-fix: no step with step_id ${input.k}`);
|
|
7416
7448
|
const context = input.steps.filter(
|
|
7417
7449
|
(s) => s.step_id !== input.k && Math.abs(s.step_id - input.k) <= radius
|
|
7418
7450
|
);
|
|
7451
|
+
return [
|
|
7452
|
+
"## Task the agent was solving",
|
|
7453
|
+
input.taskStatement ? clipText(input.taskStatement, 4e3) : "(no task statement recorded)",
|
|
7454
|
+
"",
|
|
7455
|
+
"## Surrounding steps",
|
|
7456
|
+
...context.map((s) => renderStep(s, "")),
|
|
7457
|
+
"",
|
|
7458
|
+
"## Failing step to correct",
|
|
7459
|
+
renderStep(target, " (INCORRECT \u2014 correct this one)")
|
|
7460
|
+
];
|
|
7461
|
+
}
|
|
7462
|
+
function buildFixPrompt(input) {
|
|
7419
7463
|
const system = [
|
|
7420
7464
|
"You repair one failed shell command from a recorded coding-agent trajectory.",
|
|
7421
7465
|
"The trajectory replays inside the original docker image; every command runs as a fresh /bin/sh subshell from a fixed working directory.",
|
|
@@ -7424,19 +7468,62 @@ function buildFixPrompt(input) {
|
|
|
7424
7468
|
"The corrected command must accomplish the failing step's intent and exit 0. No prose outside the fenced block."
|
|
7425
7469
|
].join("\n");
|
|
7426
7470
|
const user = [
|
|
7427
|
-
|
|
7428
|
-
input.taskStatement ? clip(input.taskStatement, 4e3) : "(no task statement recorded)",
|
|
7471
|
+
...promptBody(input),
|
|
7429
7472
|
"",
|
|
7430
|
-
"
|
|
7431
|
-
|
|
7473
|
+
"Output the single corrected replacement for the failing step now."
|
|
7474
|
+
].join("\n");
|
|
7475
|
+
return { system, user };
|
|
7476
|
+
}
|
|
7477
|
+
function renderFailedAttempt(prior) {
|
|
7478
|
+
if (prior.command === null) {
|
|
7479
|
+
return [
|
|
7480
|
+
`### attempt ${prior.attempt}`,
|
|
7481
|
+
`model call failed before producing a command: ${prior.llmError ?? "unknown error"}`
|
|
7482
|
+
].join("\n");
|
|
7483
|
+
}
|
|
7484
|
+
const stdout = (prior.stdoutTail ?? "").trim();
|
|
7485
|
+
const stderr = (prior.stderrTail ?? "").trim();
|
|
7486
|
+
return [
|
|
7487
|
+
`### attempt ${prior.attempt}`,
|
|
7488
|
+
"```sh",
|
|
7489
|
+
clipText(prior.command, 2e3),
|
|
7490
|
+
"```",
|
|
7491
|
+
`exit code: ${prior.exitCode ?? "not executed"}`,
|
|
7492
|
+
stdout.length > 0 ? `stdout:
|
|
7493
|
+
\`\`\`
|
|
7494
|
+
${stdout}
|
|
7495
|
+
\`\`\`` : "stdout: (empty)",
|
|
7496
|
+
stderr.length > 0 ? `stderr:
|
|
7497
|
+
\`\`\`
|
|
7498
|
+
${stderr}
|
|
7499
|
+
\`\`\`` : "stderr: (empty)"
|
|
7500
|
+
].join("\n");
|
|
7501
|
+
}
|
|
7502
|
+
function buildRetryFixPrompt(input, priorAttempts, maxScriptCommands = 5) {
|
|
7503
|
+
if (priorAttempts.length === 0) {
|
|
7504
|
+
throw new Error("replay-fix: buildRetryFixPrompt requires at least one prior attempt");
|
|
7505
|
+
}
|
|
7506
|
+
const system = [
|
|
7507
|
+
"You repair one failed shell command from a recorded coding-agent trajectory.",
|
|
7508
|
+
"The trajectory replays inside the original docker image; every command runs as a fresh /bin/sh subshell from a fixed working directory.",
|
|
7509
|
+
"Earlier corrected commands were executed for real and failed; their actual output is included below.",
|
|
7510
|
+
`Reply with a corrected fix inside a single \`\`\`sh fenced block: either one command, or a short script of at most ${maxScriptCommands} commands (one per line).`,
|
|
7511
|
+
"The whole block executes as ONE /bin/sh unit from the fixed working directory and must exit 0.",
|
|
7512
|
+
"Do not repeat a command that already failed. Keep reasoning brief. No prose outside the fenced block."
|
|
7513
|
+
].join("\n");
|
|
7514
|
+
const user = [
|
|
7515
|
+
...promptBody(input),
|
|
7432
7516
|
"",
|
|
7433
|
-
"##
|
|
7434
|
-
|
|
7517
|
+
"## Previous fix attempts (executed for real \u2014 all failed)",
|
|
7518
|
+
...priorAttempts.map((prior) => renderFailedAttempt(prior)),
|
|
7435
7519
|
"",
|
|
7436
|
-
|
|
7520
|
+
`Output a corrected fix now \u2014 one \`\`\`sh block, at most ${maxScriptCommands} commands.`
|
|
7437
7521
|
].join("\n");
|
|
7438
7522
|
return { system, user };
|
|
7439
7523
|
}
|
|
7524
|
+
function countScriptCommands(script) {
|
|
7525
|
+
return script.split("\n").filter((line) => line.trim().length > 0 && !line.trim().startsWith("#")).length;
|
|
7526
|
+
}
|
|
7440
7527
|
function extractFixCommand(content) {
|
|
7441
7528
|
const blocks = [...content.matchAll(/```(?:sh|bash|shell)?\n([\s\S]*?)```/g)];
|
|
7442
7529
|
const candidate = blocks.length > 0 ? blocks[blocks.length - 1][1] : content;
|
|
@@ -7452,12 +7539,167 @@ async function generateFixCommand(caller, input) {
|
|
|
7452
7539
|
if (command === null) {
|
|
7453
7540
|
return {
|
|
7454
7541
|
succeeded: false,
|
|
7455
|
-
error: `completion carried no usable command: ${
|
|
7542
|
+
error: `completion carried no usable command: ${clipText(outcome.value.content, 300)}`
|
|
7456
7543
|
};
|
|
7457
7544
|
}
|
|
7458
7545
|
return { succeeded: true, value: { command, usage: outcome.value.usage } };
|
|
7459
7546
|
}
|
|
7460
7547
|
|
|
7548
|
+
// src/replay-fix-loop.ts
|
|
7549
|
+
function toFailedAttempt(record2) {
|
|
7550
|
+
return {
|
|
7551
|
+
attempt: record2.attempt,
|
|
7552
|
+
command: record2.command,
|
|
7553
|
+
exitCode: record2.exitCode,
|
|
7554
|
+
stdoutTail: record2.stdoutTail,
|
|
7555
|
+
stderrTail: record2.stderrTail,
|
|
7556
|
+
llmError: record2.llmError ?? record2.armBError
|
|
7557
|
+
};
|
|
7558
|
+
}
|
|
7559
|
+
async function runFixLoop(caller, input, executor, options) {
|
|
7560
|
+
if (!Number.isInteger(options.maxAttempts) || options.maxAttempts < 1) {
|
|
7561
|
+
throw new Error(
|
|
7562
|
+
`replay-fix-loop: maxAttempts must be a positive integer, got ${options.maxAttempts}`
|
|
7563
|
+
);
|
|
7564
|
+
}
|
|
7565
|
+
const scriptCap = options.maxScriptCommands ?? 5;
|
|
7566
|
+
const tailChars = options.outputTailChars ?? 1600;
|
|
7567
|
+
const attempts = [];
|
|
7568
|
+
let llmCalls = 0;
|
|
7569
|
+
let llmFailures = 0;
|
|
7570
|
+
let promptTokens = 0;
|
|
7571
|
+
let completionTokens = 0;
|
|
7572
|
+
const unexecuted = (attempt, llmError, usage, command, started) => ({
|
|
7573
|
+
attempt,
|
|
7574
|
+
command,
|
|
7575
|
+
llmError,
|
|
7576
|
+
usage,
|
|
7577
|
+
executed: false,
|
|
7578
|
+
exitCode: null,
|
|
7579
|
+
prefixExecuted: null,
|
|
7580
|
+
prefixDivergences: null,
|
|
7581
|
+
failureVanished: null,
|
|
7582
|
+
stdoutTail: null,
|
|
7583
|
+
stderrTail: null,
|
|
7584
|
+
armBError: null,
|
|
7585
|
+
wallMs: Date.now() - started
|
|
7586
|
+
});
|
|
7587
|
+
for (let attempt = 1; attempt <= options.maxAttempts; attempt++) {
|
|
7588
|
+
const started = Date.now();
|
|
7589
|
+
const prompt = attempt === 1 ? buildFixPrompt(input) : buildRetryFixPrompt(input, attempts.map(toFailedAttempt), scriptCap);
|
|
7590
|
+
llmCalls += 1;
|
|
7591
|
+
const outcome = await caller.complete(prompt.system, prompt.user);
|
|
7592
|
+
if (!outcome.succeeded) {
|
|
7593
|
+
llmFailures += 1;
|
|
7594
|
+
attempts.push(unexecuted(attempt, outcome.error, null, null, started));
|
|
7595
|
+
options.onProgress?.(
|
|
7596
|
+
`fix-loop attempt ${attempt}: LLM failed \u2014 ${outcome.error.slice(0, 160)}`
|
|
7597
|
+
);
|
|
7598
|
+
continue;
|
|
7599
|
+
}
|
|
7600
|
+
const usage = outcome.value.usage;
|
|
7601
|
+
promptTokens += usage?.promptTokens ?? 0;
|
|
7602
|
+
completionTokens += usage?.completionTokens ?? 0;
|
|
7603
|
+
const command = extractFixCommand(outcome.value.content);
|
|
7604
|
+
if (command === null) {
|
|
7605
|
+
llmFailures += 1;
|
|
7606
|
+
attempts.push(
|
|
7607
|
+
unexecuted(
|
|
7608
|
+
attempt,
|
|
7609
|
+
`completion carried no usable command: ${clipText(outcome.value.content, 300)}`,
|
|
7610
|
+
usage,
|
|
7611
|
+
null,
|
|
7612
|
+
started
|
|
7613
|
+
)
|
|
7614
|
+
);
|
|
7615
|
+
continue;
|
|
7616
|
+
}
|
|
7617
|
+
if (attempt > 1) {
|
|
7618
|
+
const commandLines = countScriptCommands(command);
|
|
7619
|
+
if (commandLines > scriptCap) {
|
|
7620
|
+
llmFailures += 1;
|
|
7621
|
+
attempts.push(
|
|
7622
|
+
unexecuted(
|
|
7623
|
+
attempt,
|
|
7624
|
+
`script exceeds ${scriptCap} command lines (${commandLines})`,
|
|
7625
|
+
usage,
|
|
7626
|
+
null,
|
|
7627
|
+
started
|
|
7628
|
+
)
|
|
7629
|
+
);
|
|
7630
|
+
options.onProgress?.(
|
|
7631
|
+
`fix-loop attempt ${attempt}: rejected script with ${commandLines} command lines`
|
|
7632
|
+
);
|
|
7633
|
+
continue;
|
|
7634
|
+
}
|
|
7635
|
+
}
|
|
7636
|
+
let execution;
|
|
7637
|
+
try {
|
|
7638
|
+
execution = await executor(command, attempt);
|
|
7639
|
+
} catch (err) {
|
|
7640
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
7641
|
+
attempts.push({
|
|
7642
|
+
...unexecuted(attempt, "", usage, command, started),
|
|
7643
|
+
llmError: null,
|
|
7644
|
+
armBError: message.slice(0, 500)
|
|
7645
|
+
});
|
|
7646
|
+
options.onProgress?.(
|
|
7647
|
+
`fix-loop attempt ${attempt}: sandbox error \u2014 ${message.slice(0, 160)}`
|
|
7648
|
+
);
|
|
7649
|
+
return {
|
|
7650
|
+
flipped: false,
|
|
7651
|
+
flippedAtAttempt: null,
|
|
7652
|
+
attempts,
|
|
7653
|
+
aborted: true,
|
|
7654
|
+
llmCalls,
|
|
7655
|
+
llmFailures,
|
|
7656
|
+
promptTokens,
|
|
7657
|
+
completionTokens
|
|
7658
|
+
};
|
|
7659
|
+
}
|
|
7660
|
+
attempts.push({
|
|
7661
|
+
attempt,
|
|
7662
|
+
command,
|
|
7663
|
+
llmError: null,
|
|
7664
|
+
usage,
|
|
7665
|
+
executed: true,
|
|
7666
|
+
exitCode: execution.exitCode,
|
|
7667
|
+
prefixExecuted: execution.prefixExecuted,
|
|
7668
|
+
prefixDivergences: execution.prefixDivergences,
|
|
7669
|
+
failureVanished: execution.failureVanished,
|
|
7670
|
+
stdoutTail: clipText(execution.stdout, tailChars),
|
|
7671
|
+
stderrTail: clipText(execution.stderr, tailChars),
|
|
7672
|
+
armBError: null,
|
|
7673
|
+
wallMs: Date.now() - started
|
|
7674
|
+
});
|
|
7675
|
+
options.onProgress?.(
|
|
7676
|
+
`fix-loop attempt ${attempt}: exit=${execution.exitCode} failureVanished=${execution.failureVanished}`
|
|
7677
|
+
);
|
|
7678
|
+
if (execution.failureVanished) {
|
|
7679
|
+
return {
|
|
7680
|
+
flipped: true,
|
|
7681
|
+
flippedAtAttempt: attempt,
|
|
7682
|
+
attempts,
|
|
7683
|
+
aborted: false,
|
|
7684
|
+
llmCalls,
|
|
7685
|
+
llmFailures,
|
|
7686
|
+
promptTokens,
|
|
7687
|
+
completionTokens
|
|
7688
|
+
};
|
|
7689
|
+
}
|
|
7690
|
+
}
|
|
7691
|
+
return {
|
|
7692
|
+
flipped: false,
|
|
7693
|
+
flippedAtAttempt: null,
|
|
7694
|
+
attempts,
|
|
7695
|
+
aborted: false,
|
|
7696
|
+
llmCalls,
|
|
7697
|
+
llmFailures,
|
|
7698
|
+
promptTokens,
|
|
7699
|
+
completionTokens
|
|
7700
|
+
};
|
|
7701
|
+
}
|
|
7702
|
+
|
|
7461
7703
|
// src/replay-batch.ts
|
|
7462
7704
|
import { execFile } from "child_process";
|
|
7463
7705
|
import { createHash as createHash8 } from "crypto";
|
|
@@ -7746,9 +7988,10 @@ async function runReplayBatch(options) {
|
|
|
7746
7988
|
}
|
|
7747
7989
|
}
|
|
7748
7990
|
let llm = null;
|
|
7749
|
-
if (options.fix
|
|
7991
|
+
if (options.fix !== "none") {
|
|
7750
7992
|
const caller = options.fixCaller;
|
|
7751
|
-
if (!caller) throw new Error(
|
|
7993
|
+
if (!caller) throw new Error(`replay-verify-batch: fix=${options.fix} requires a fixCaller`);
|
|
7994
|
+
const fixAttempts = options.fixAttempts ?? 3;
|
|
7752
7995
|
const maxFixCases = options.maxFixCases ?? 30;
|
|
7753
7996
|
const seed = options.seed ?? 17;
|
|
7754
7997
|
const eligible = rows.filter((r) => r.status === "ok" && r.replayed);
|
|
@@ -7777,20 +8020,115 @@ async function runReplayBatch(options) {
|
|
|
7777
8020
|
armBPrefixExecuted: null,
|
|
7778
8021
|
armBPrefixDivergences: null,
|
|
7779
8022
|
failureVanished: null,
|
|
7780
|
-
armBError: null
|
|
8023
|
+
armBError: null,
|
|
8024
|
+
attempts: null,
|
|
8025
|
+
flippedAtAttempt: null
|
|
7781
8026
|
}
|
|
7782
8027
|
};
|
|
7783
8028
|
continue;
|
|
7784
8029
|
}
|
|
7785
8030
|
const replayCase = selected.find((c) => c.trajId === row.trajId && c.corpus === row.corpus);
|
|
7786
8031
|
const label = `fix ${row.corpus}/${row.trajId}`;
|
|
7787
|
-
|
|
7788
|
-
|
|
7789
|
-
const
|
|
8032
|
+
const signature = verdictByTraj.get(row.trajId)?.signature ?? null;
|
|
8033
|
+
const stepTimeoutMs = options.stepTimeoutMs ?? replayCase.recordedStepTimeoutMs ?? 12e4;
|
|
8034
|
+
const promptInput = {
|
|
7790
8035
|
taskStatement: replayCase.taskStatement,
|
|
7791
8036
|
steps: replayCase.steps,
|
|
7792
8037
|
k: replayCase.k
|
|
7793
|
-
}
|
|
8038
|
+
};
|
|
8039
|
+
const caseOut = join21(options.out, caseOutDirName(row));
|
|
8040
|
+
if (options.fix === "loop") {
|
|
8041
|
+
onProgress(`${label}: fix loop (budget ${fixAttempts} attempts)`);
|
|
8042
|
+
const result = await runFixLoop(
|
|
8043
|
+
caller,
|
|
8044
|
+
promptInput,
|
|
8045
|
+
async (command, attempt) => {
|
|
8046
|
+
onProgress(
|
|
8047
|
+
`${label}: attempt ${attempt} arm B \u2014 ${command.split("\n")[0].slice(0, 120)}`
|
|
8048
|
+
);
|
|
8049
|
+
const armB = await executeArmB(
|
|
8050
|
+
replayCase,
|
|
8051
|
+
command,
|
|
8052
|
+
backendFactory(row.derivedImage),
|
|
8053
|
+
signature,
|
|
8054
|
+
stepTimeoutMs,
|
|
8055
|
+
options.prefixLimit,
|
|
8056
|
+
(message) => onProgress(`${label}: attempt ${attempt}: ${message}`)
|
|
8057
|
+
);
|
|
8058
|
+
mkdirSync2(caseOut, { recursive: true });
|
|
8059
|
+
writeFileSync2(
|
|
8060
|
+
join21(caseOut, `armB-attempt${attempt}-result.json`),
|
|
8061
|
+
`${JSON.stringify({ command, ...armB }, null, 2)}
|
|
8062
|
+
`
|
|
8063
|
+
);
|
|
8064
|
+
writeFileSync2(
|
|
8065
|
+
join21(caseOut, "armB-result.json"),
|
|
8066
|
+
`${JSON.stringify({ command, attempt, ...armB }, null, 2)}
|
|
8067
|
+
`
|
|
8068
|
+
);
|
|
8069
|
+
return armB;
|
|
8070
|
+
},
|
|
8071
|
+
{ maxAttempts: fixAttempts, onProgress: (message) => onProgress(`${label}: ${message}`) }
|
|
8072
|
+
);
|
|
8073
|
+
calls += result.llmCalls;
|
|
8074
|
+
failures += result.llmFailures;
|
|
8075
|
+
promptTokens += result.promptTokens;
|
|
8076
|
+
completionTokens += result.completionTokens;
|
|
8077
|
+
const usage = result.promptTokens + result.completionTokens > 0 ? { promptTokens: result.promptTokens, completionTokens: result.completionTokens } : null;
|
|
8078
|
+
const flippedRecord = result.flippedAtAttempt !== null ? result.attempts.find((a) => a.attempt === result.flippedAtAttempt) : null;
|
|
8079
|
+
const summary = flippedRecord ?? [...result.attempts].reverse().find((a) => a.executed) ?? null;
|
|
8080
|
+
const lastRecord = result.attempts.at(-1) ?? null;
|
|
8081
|
+
rows[index] = {
|
|
8082
|
+
...row,
|
|
8083
|
+
fix: summary ? {
|
|
8084
|
+
attempted: true,
|
|
8085
|
+
sampledOut: false,
|
|
8086
|
+
command: summary.command,
|
|
8087
|
+
llmError: null,
|
|
8088
|
+
usage,
|
|
8089
|
+
armBExit: summary.exitCode,
|
|
8090
|
+
armBPrefixExecuted: summary.prefixExecuted,
|
|
8091
|
+
armBPrefixDivergences: summary.prefixDivergences,
|
|
8092
|
+
failureVanished: summary.failureVanished,
|
|
8093
|
+
armBError: null,
|
|
8094
|
+
attempts: result.attempts,
|
|
8095
|
+
flippedAtAttempt: result.flippedAtAttempt
|
|
8096
|
+
} : result.aborted ? {
|
|
8097
|
+
attempted: true,
|
|
8098
|
+
sampledOut: false,
|
|
8099
|
+
command: lastRecord?.command ?? null,
|
|
8100
|
+
llmError: null,
|
|
8101
|
+
usage,
|
|
8102
|
+
armBExit: null,
|
|
8103
|
+
armBPrefixExecuted: null,
|
|
8104
|
+
armBPrefixDivergences: null,
|
|
8105
|
+
failureVanished: null,
|
|
8106
|
+
armBError: lastRecord?.armBError ?? "sandbox error",
|
|
8107
|
+
attempts: result.attempts,
|
|
8108
|
+
flippedAtAttempt: null
|
|
8109
|
+
} : {
|
|
8110
|
+
attempted: true,
|
|
8111
|
+
sampledOut: false,
|
|
8112
|
+
command: null,
|
|
8113
|
+
llmError: lastRecord?.llmError ?? "no attempt produced a runnable fix",
|
|
8114
|
+
usage,
|
|
8115
|
+
armBExit: null,
|
|
8116
|
+
armBPrefixExecuted: null,
|
|
8117
|
+
armBPrefixDivergences: null,
|
|
8118
|
+
failureVanished: null,
|
|
8119
|
+
armBError: null,
|
|
8120
|
+
attempts: result.attempts,
|
|
8121
|
+
flippedAtAttempt: null
|
|
8122
|
+
}
|
|
8123
|
+
};
|
|
8124
|
+
onProgress(
|
|
8125
|
+
`${label}: loop done \u2014 flipped=${result.flipped}` + (result.flippedAtAttempt !== null ? ` at attempt ${result.flippedAtAttempt}` : "") + ` (${result.llmCalls} calls, ${result.attempts.filter((a) => a.executed).length} arms)`
|
|
8126
|
+
);
|
|
8127
|
+
continue;
|
|
8128
|
+
}
|
|
8129
|
+
onProgress(`${label}: generating corrected command`);
|
|
8130
|
+
calls += 1;
|
|
8131
|
+
const generated = await generateFixCommand(caller, promptInput);
|
|
7794
8132
|
if (!generated.succeeded) {
|
|
7795
8133
|
failures += 1;
|
|
7796
8134
|
rows[index] = {
|
|
@@ -7805,7 +8143,9 @@ async function runReplayBatch(options) {
|
|
|
7805
8143
|
armBPrefixExecuted: null,
|
|
7806
8144
|
armBPrefixDivergences: null,
|
|
7807
8145
|
failureVanished: null,
|
|
7808
|
-
armBError: null
|
|
8146
|
+
armBError: null,
|
|
8147
|
+
attempts: null,
|
|
8148
|
+
flippedAtAttempt: null
|
|
7809
8149
|
}
|
|
7810
8150
|
};
|
|
7811
8151
|
onProgress(`${label}: LLM failed \u2014 ${generated.error.slice(0, 200)}`);
|
|
@@ -7813,8 +8153,6 @@ async function runReplayBatch(options) {
|
|
|
7813
8153
|
}
|
|
7814
8154
|
promptTokens += generated.value.usage?.promptTokens ?? 0;
|
|
7815
8155
|
completionTokens += generated.value.usage?.completionTokens ?? 0;
|
|
7816
|
-
const signature = verdictByTraj.get(row.trajId)?.signature ?? null;
|
|
7817
|
-
const stepTimeoutMs = options.stepTimeoutMs ?? replayCase.recordedStepTimeoutMs ?? 12e4;
|
|
7818
8156
|
onProgress(`${label}: arm B \u2014 ${generated.value.command.split("\n")[0].slice(0, 120)}`);
|
|
7819
8157
|
try {
|
|
7820
8158
|
const armB = await executeArmB(
|
|
@@ -7838,10 +8176,11 @@ async function runReplayBatch(options) {
|
|
|
7838
8176
|
armBPrefixExecuted: armB.prefixExecuted,
|
|
7839
8177
|
armBPrefixDivergences: armB.prefixDivergences,
|
|
7840
8178
|
failureVanished: armB.failureVanished,
|
|
7841
|
-
armBError: null
|
|
8179
|
+
armBError: null,
|
|
8180
|
+
attempts: null,
|
|
8181
|
+
flippedAtAttempt: armB.failureVanished ? 1 : null
|
|
7842
8182
|
}
|
|
7843
8183
|
};
|
|
7844
|
-
const caseOut = join21(options.out, caseOutDirName(row));
|
|
7845
8184
|
mkdirSync2(caseOut, { recursive: true });
|
|
7846
8185
|
writeFileSync2(
|
|
7847
8186
|
join21(caseOut, "armB-result.json"),
|
|
@@ -7863,7 +8202,9 @@ async function runReplayBatch(options) {
|
|
|
7863
8202
|
armBPrefixExecuted: null,
|
|
7864
8203
|
armBPrefixDivergences: null,
|
|
7865
8204
|
failureVanished: null,
|
|
7866
|
-
armBError: message.slice(0, 500)
|
|
8205
|
+
armBError: message.slice(0, 500),
|
|
8206
|
+
attempts: null,
|
|
8207
|
+
flippedAtAttempt: null
|
|
7867
8208
|
}
|
|
7868
8209
|
};
|
|
7869
8210
|
onProgress(`${label}: arm B error \u2014 ${message.slice(0, 200)}`);
|
|
@@ -7888,6 +8229,15 @@ async function runReplayBatch(options) {
|
|
|
7888
8229
|
(r) => r.recordedReturncodeAtK !== null && r.recordedReturncodeAtK !== 0
|
|
7889
8230
|
);
|
|
7890
8231
|
const flippedNonzeroRc = armBNonzeroRc.filter((r) => r.fix.failureVanished === true);
|
|
8232
|
+
const attempt1Executed = rows.filter(
|
|
8233
|
+
(r) => r.fix?.attempts?.find((a) => a.attempt === 1)?.executed === true
|
|
8234
|
+
);
|
|
8235
|
+
const flippedAt1 = rows.filter((r) => r.fix?.flippedAtAttempt === 1);
|
|
8236
|
+
const flipsByAttempt = {};
|
|
8237
|
+
for (const row of rows) {
|
|
8238
|
+
const at = row.fix?.flippedAtAttempt;
|
|
8239
|
+
if (typeof at === "number") flipsByAttempt[String(at)] = (flipsByAttempt[String(at)] ?? 0) + 1;
|
|
8240
|
+
}
|
|
7891
8241
|
const excludedByReason = {};
|
|
7892
8242
|
for (const excluded of enumeration.excluded) {
|
|
7893
8243
|
excludedByReason[excluded.reason] = (excludedByReason[excluded.reason] ?? 0) + 1;
|
|
@@ -7927,16 +8277,22 @@ async function runReplayBatch(options) {
|
|
|
7927
8277
|
denominator: selected.length,
|
|
7928
8278
|
value: rate(signatureStrict.length, selected.length)
|
|
7929
8279
|
},
|
|
7930
|
-
fixFlipRate: options.fix
|
|
8280
|
+
fixFlipRate: options.fix !== "none" ? {
|
|
7931
8281
|
numerator: flipped.length,
|
|
7932
8282
|
denominator: armBExecuted.length,
|
|
7933
8283
|
value: rate(flipped.length, armBExecuted.length)
|
|
7934
8284
|
} : null,
|
|
7935
|
-
fixFlipRateNonzeroRc: options.fix
|
|
8285
|
+
fixFlipRateNonzeroRc: options.fix !== "none" ? {
|
|
7936
8286
|
numerator: flippedNonzeroRc.length,
|
|
7937
8287
|
denominator: armBNonzeroRc.length,
|
|
7938
8288
|
value: rate(flippedNonzeroRc.length, armBNonzeroRc.length)
|
|
7939
|
-
} : null
|
|
8289
|
+
} : null,
|
|
8290
|
+
fixFlipAttempt1: options.fix === "loop" ? {
|
|
8291
|
+
numerator: flippedAt1.length,
|
|
8292
|
+
denominator: attempt1Executed.length,
|
|
8293
|
+
value: rate(flippedAt1.length, attempt1Executed.length)
|
|
8294
|
+
} : null,
|
|
8295
|
+
flipsByAttempt: options.fix === "loop" ? flipsByAttempt : null
|
|
7940
8296
|
},
|
|
7941
8297
|
llm,
|
|
7942
8298
|
excluded: enumeration.excluded,
|
|
@@ -7980,6 +8336,15 @@ function renderBatchReport(report) {
|
|
|
7980
8336
|
`- Fix-flip rate on recorded-rc\u22600 cases: ${pct(headline.fixFlipRateNonzeroRc.value)} (${headline.fixFlipRateNonzeroRc.numerator}/${headline.fixFlipRateNonzeroRc.denominator}; real recorded failures \u2014 a gold step recorded with rc 0 flips vacuously).`
|
|
7981
8337
|
);
|
|
7982
8338
|
}
|
|
8339
|
+
if (headline.fixFlipAttempt1) {
|
|
8340
|
+
lines.push(
|
|
8341
|
+
`- Fix-flip@1: ${pct(headline.fixFlipAttempt1.value)} (${headline.fixFlipAttempt1.numerator}/${headline.fixFlipAttempt1.denominator} cases whose attempt 1 executed \u2014 the one-shot-comparable number).`
|
|
8342
|
+
);
|
|
8343
|
+
}
|
|
8344
|
+
if (headline.flipsByAttempt && Object.keys(headline.flipsByAttempt).length > 0) {
|
|
8345
|
+
const parts = Object.entries(headline.flipsByAttempt).sort((a, b) => Number(a[0]) - Number(b[0])).map(([attempt, count2]) => `attempt ${attempt}: ${count2}`);
|
|
8346
|
+
lines.push(`- Flips by attempt: ${parts.join(", ")}.`);
|
|
8347
|
+
}
|
|
7983
8348
|
lines.push("");
|
|
7984
8349
|
lines.push("## Enumeration");
|
|
7985
8350
|
lines.push("");
|
|
@@ -8028,7 +8393,7 @@ function renderBatchReport(report) {
|
|
|
8028
8393
|
);
|
|
8029
8394
|
for (const row of report.cases) {
|
|
8030
8395
|
const fix = row.fix;
|
|
8031
|
-
const fixCell = !fix ? "\u2014" : fix.sampledOut ? "sampled-out" : fix.llmError ? "llm-failed" : fix.armBError ? "armB-error" : "generated";
|
|
8396
|
+
const fixCell = !fix ? "\u2014" : fix.sampledOut ? "sampled-out" : fix.llmError ? "llm-failed" : fix.armBError ? "armB-error" : fix.attempts ? fix.flippedAtAttempt !== null ? `flip@${fix.flippedAtAttempt}` : `exhausted(${fix.attempts.length})` : "generated";
|
|
8032
8397
|
lines.push(
|
|
8033
8398
|
[
|
|
8034
8399
|
row.corpus,
|
|
@@ -8090,13 +8455,20 @@ function parseReplayBatchArgs(argv) {
|
|
|
8090
8455
|
return n;
|
|
8091
8456
|
};
|
|
8092
8457
|
const fix = values.get("--fix") ?? "none";
|
|
8093
|
-
if (fix !== "none" && fix !== "generate") {
|
|
8094
|
-
throw new Error(`replay-verify-batch: --fix must be none or
|
|
8458
|
+
if (fix !== "none" && fix !== "generate" && fix !== "loop") {
|
|
8459
|
+
throw new Error(`replay-verify-batch: --fix must be none, generate, or loop, got ${fix}`);
|
|
8460
|
+
}
|
|
8461
|
+
const fixAttempts = optionalNumber("--fix-attempts") ?? 3;
|
|
8462
|
+
if (!Number.isInteger(fixAttempts) || fixAttempts < 1) {
|
|
8463
|
+
throw new Error(
|
|
8464
|
+
`replay-verify-batch: --fix-attempts must be a positive integer, got ${values.get("--fix-attempts")}`
|
|
8465
|
+
);
|
|
8095
8466
|
}
|
|
8096
8467
|
return {
|
|
8097
8468
|
corpora,
|
|
8098
8469
|
out: out ?? ".",
|
|
8099
8470
|
fix,
|
|
8471
|
+
fixAttempts,
|
|
8100
8472
|
fixModel: values.get("--fix-model") ?? "glm-5.2",
|
|
8101
8473
|
fixBaseUrl: values.get("--fix-base-url") ?? "https://api.z.ai/api/coding/paas/v4",
|
|
8102
8474
|
fixApiKeyEnv: values.get("--fix-api-key-env") ?? "ZAI_GLM_API_KEY",
|
|
@@ -8118,7 +8490,7 @@ function replayBatchUsage() {
|
|
|
8118
8490
|
Usage:
|
|
8119
8491
|
traces replay-verify-batch \\
|
|
8120
8492
|
--corpus NAME=<labels.json>::<prepared-dir> [--corpus ...] \\
|
|
8121
|
-
--out DIR [--fix none|generate] [--enumerate-only] \\
|
|
8493
|
+
--out DIR [--fix none|generate|loop] [--fix-attempts 3] [--enumerate-only] \\
|
|
8122
8494
|
[--fix-model glm-5.2] [--fix-base-url URL] [--fix-api-key-env ZAI_GLM_API_KEY] \\
|
|
8123
8495
|
[--max-fix-cases 30] [--seed 17] [--step-timeout MS] [--prefix-limit N] \\
|
|
8124
8496
|
[--case-filter SUBSTRING] [--case-limit N] \\
|
|
@@ -8137,6 +8509,13 @@ Usage:
|
|
|
8137
8509
|
--max-fix-cases, seeded sample beyond it) and executes the corrected command
|
|
8138
8510
|
as arm B in a fresh sandbox.
|
|
8139
8511
|
|
|
8512
|
+
--fix loop iterates: attempt 1 is the one-shot prompt; when an arm fails
|
|
8513
|
+
(nonzero exit or the failure signature persists) or the model call fails,
|
|
8514
|
+
the next prompt carries every prior command with its REAL executed
|
|
8515
|
+
stdout/stderr, up to --fix-attempts attempts (default 3). Retries may
|
|
8516
|
+
answer with a short script (<=5 commands) executed as one /bin/sh unit.
|
|
8517
|
+
Every attempt runs in its own fresh sandbox with the same replayed prefix.
|
|
8518
|
+
|
|
8140
8519
|
Outputs batch-report.json, batch-report.md, cases.jsonl (incremental), and one
|
|
8141
8520
|
directory per case with the full replay-verify artifacts.
|
|
8142
8521
|
See docs/replay-verify.md for orchestrator setup and honest limits.
|
|
@@ -8181,7 +8560,7 @@ async function cmdReplayVerifyBatch(argv) {
|
|
|
8181
8560
|
);
|
|
8182
8561
|
}
|
|
8183
8562
|
let fixCaller;
|
|
8184
|
-
if (parsed.fix
|
|
8563
|
+
if (parsed.fix !== "none") {
|
|
8185
8564
|
const fixApiKey = process.env[parsed.fixApiKeyEnv];
|
|
8186
8565
|
if (!fixApiKey) {
|
|
8187
8566
|
throw new Error(
|
|
@@ -8200,6 +8579,7 @@ async function cmdReplayVerifyBatch(argv) {
|
|
|
8200
8579
|
baseUrl: parsed.baseUrl,
|
|
8201
8580
|
apiKey,
|
|
8202
8581
|
fix: parsed.fix,
|
|
8582
|
+
fixAttempts: parsed.fixAttempts,
|
|
8203
8583
|
fixCaller,
|
|
8204
8584
|
fixModelLabel: parsed.fixModel,
|
|
8205
8585
|
maxFixCases: parsed.maxFixCases,
|
|
@@ -8219,6 +8599,573 @@ async function cmdReplayVerifyBatch(argv) {
|
|
|
8219
8599
|
);
|
|
8220
8600
|
}
|
|
8221
8601
|
|
|
8602
|
+
// src/replay-wire.ts
|
|
8603
|
+
var SUBJECT_PATTERN = /^incorrect-steps-(\d+)-(\d+)-(escaped|unescaped)-consequence-(\d+)$/;
|
|
8604
|
+
function parseIncorrectStepsSubject(subject) {
|
|
8605
|
+
const match = SUBJECT_PATTERN.exec(subject);
|
|
8606
|
+
if (!match) return null;
|
|
8607
|
+
return {
|
|
8608
|
+
firstStep: Number(match[1]),
|
|
8609
|
+
lastStep: Number(match[2]),
|
|
8610
|
+
escapeStatus: match[3],
|
|
8611
|
+
consequenceStep: Number(match[4])
|
|
8612
|
+
};
|
|
8613
|
+
}
|
|
8614
|
+
function resolveFindingInvocation(finding2, corpora) {
|
|
8615
|
+
const subject = parseIncorrectStepsSubject(finding2.subject);
|
|
8616
|
+
if (!subject) {
|
|
8617
|
+
throw new Error(
|
|
8618
|
+
`replay-wire: subject '${finding2.subject}' is not incorrect-steps-<first>-<last>-<escaped|unescaped>-consequence-<step>`
|
|
8619
|
+
);
|
|
8620
|
+
}
|
|
8621
|
+
const failures = [];
|
|
8622
|
+
for (const corpus of corpora) {
|
|
8623
|
+
const resolution = resolveCaseResources(corpus, finding2.trajId);
|
|
8624
|
+
if (resolution.resolved) {
|
|
8625
|
+
const at = subject.firstStep;
|
|
8626
|
+
const step = resolution.resources.steps.find((s) => s.step_id === at);
|
|
8627
|
+
if (!step) {
|
|
8628
|
+
throw new Error(
|
|
8629
|
+
`replay-wire: finding step ${at} is outside ${finding2.trajId} (${resolution.resources.steps.length} steps)`
|
|
8630
|
+
);
|
|
8631
|
+
}
|
|
8632
|
+
return { resources: resolution.resources, subject, at };
|
|
8633
|
+
}
|
|
8634
|
+
failures.push(`${corpus.name}: ${resolution.reason}`);
|
|
8635
|
+
}
|
|
8636
|
+
throw new Error(
|
|
8637
|
+
`replay-wire: trajectory ${finding2.trajId} is not replayable in any corpus \u2014 ${failures.join("; ")}`
|
|
8638
|
+
);
|
|
8639
|
+
}
|
|
8640
|
+
async function replayVerifyFinding(finding2, options) {
|
|
8641
|
+
if (options.fixCaller && options.fixCommand !== void 0) {
|
|
8642
|
+
throw new Error("replay-wire: pass fixCaller or fixCommand, not both");
|
|
8643
|
+
}
|
|
8644
|
+
const invocation = resolveFindingInvocation(finding2, options.corpora);
|
|
8645
|
+
let fixCommand = options.fixCommand ?? null;
|
|
8646
|
+
if (options.fixCaller) {
|
|
8647
|
+
const generated = await generateFixCommand(options.fixCaller, {
|
|
8648
|
+
taskStatement: invocation.resources.taskStatement,
|
|
8649
|
+
steps: invocation.resources.steps,
|
|
8650
|
+
k: invocation.at
|
|
8651
|
+
});
|
|
8652
|
+
if (!generated.succeeded) {
|
|
8653
|
+
throw new Error(`replay-wire: fix generation failed \u2014 ${generated.error}`);
|
|
8654
|
+
}
|
|
8655
|
+
fixCommand = generated.value.command;
|
|
8656
|
+
}
|
|
8657
|
+
const verdict = await replayVerify({
|
|
8658
|
+
stepsPath: invocation.resources.stepsPath,
|
|
8659
|
+
image: invocation.resources.image,
|
|
8660
|
+
at: invocation.at,
|
|
8661
|
+
fixCommand: fixCommand ?? void 0,
|
|
8662
|
+
cwd: invocation.resources.cwd,
|
|
8663
|
+
out: options.out,
|
|
8664
|
+
caseId: invocation.resources.trajId,
|
|
8665
|
+
stepTimeoutMs: options.stepTimeoutMs ?? invocation.resources.recordedStepTimeoutMs ?? void 0,
|
|
8666
|
+
prefixLimit: options.prefixLimit,
|
|
8667
|
+
backend: options.backend,
|
|
8668
|
+
apiKey: options.apiKey,
|
|
8669
|
+
baseUrl: options.baseUrl,
|
|
8670
|
+
onProgress: options.onProgress
|
|
8671
|
+
});
|
|
8672
|
+
return { invocation, fixCommand, verdict };
|
|
8673
|
+
}
|
|
8674
|
+
|
|
8675
|
+
// src/analyze-verify.ts
|
|
8676
|
+
import { mkdirSync as mkdirSync3, readFileSync as readFileSync4, writeFileSync as writeFileSync3 } from "fs";
|
|
8677
|
+
import { join as join22 } from "path";
|
|
8678
|
+
var SINGLE_STEP_SUBJECT = /^incorrect-step-(\d+)$/;
|
|
8679
|
+
function findingReplayStep(finding2) {
|
|
8680
|
+
const fromMetadata = finding2.metadata?.block_first_step;
|
|
8681
|
+
if (typeof fromMetadata === "number" && Number.isInteger(fromMetadata) && fromMetadata >= 1) {
|
|
8682
|
+
return fromMetadata;
|
|
8683
|
+
}
|
|
8684
|
+
const subject = finding2.subject ?? "";
|
|
8685
|
+
const wire = parseIncorrectStepsSubject(subject);
|
|
8686
|
+
if (wire) return wire.firstStep;
|
|
8687
|
+
const single = SINGLE_STEP_SUBJECT.exec(subject);
|
|
8688
|
+
if (single) return Number(single[1]);
|
|
8689
|
+
return null;
|
|
8690
|
+
}
|
|
8691
|
+
var TRACE_EVIDENCE_URI = /^trace:\/\/([^/]+)\//;
|
|
8692
|
+
function findingTrajectoryId(finding2) {
|
|
8693
|
+
for (const ref of finding2.evidence_refs ?? []) {
|
|
8694
|
+
if (typeof ref.uri !== "string") continue;
|
|
8695
|
+
const match = TRACE_EVIDENCE_URI.exec(ref.uri);
|
|
8696
|
+
if (match) return match[1];
|
|
8697
|
+
}
|
|
8698
|
+
return null;
|
|
8699
|
+
}
|
|
8700
|
+
function checkStep(steps, at) {
|
|
8701
|
+
const step = steps.find((s) => s.step_id === at);
|
|
8702
|
+
if (!step) {
|
|
8703
|
+
return { ok: false, reason: `step ${at} is outside the trajectory (${steps.length} steps)` };
|
|
8704
|
+
}
|
|
8705
|
+
if (isSubmitAction(step.action)) {
|
|
8706
|
+
return {
|
|
8707
|
+
ok: false,
|
|
8708
|
+
reason: `step ${at} is the submit action \u2014 a submit decision has no executable failure to replay`
|
|
8709
|
+
};
|
|
8710
|
+
}
|
|
8711
|
+
const recordedReturncode = parseRecordedReturncode(step.observation);
|
|
8712
|
+
if (recordedReturncode === null) {
|
|
8713
|
+
return {
|
|
8714
|
+
ok: false,
|
|
8715
|
+
reason: `step ${at} recorded no returncode \u2014 there is no executable failure signature to reproduce`
|
|
8716
|
+
};
|
|
8717
|
+
}
|
|
8718
|
+
return { ok: true, recordedReturncode };
|
|
8719
|
+
}
|
|
8720
|
+
function resolveFindingReplayability(finding2, source) {
|
|
8721
|
+
const at = findingReplayStep(finding2);
|
|
8722
|
+
if (at === null) {
|
|
8723
|
+
return {
|
|
8724
|
+
replayable: false,
|
|
8725
|
+
reason: `subject '${finding2.subject ?? "(none)"}' names no trajectory step (expected incorrect-step-<n>, incorrect-steps-<f>-<l>-\u2026, or metadata.block_first_step)`
|
|
8726
|
+
};
|
|
8727
|
+
}
|
|
8728
|
+
if (source.kind === "direct") {
|
|
8729
|
+
const trajId2 = findingTrajectoryId(finding2);
|
|
8730
|
+
if (trajId2 && source.caseId && trajId2 !== source.caseId) {
|
|
8731
|
+
return {
|
|
8732
|
+
replayable: false,
|
|
8733
|
+
reason: `finding cites trajectory '${trajId2}' but the supplied steps are case '${source.caseId}'`
|
|
8734
|
+
};
|
|
8735
|
+
}
|
|
8736
|
+
let steps;
|
|
8737
|
+
try {
|
|
8738
|
+
steps = JSON.parse(readFileSync4(source.stepsPath, "utf8"));
|
|
8739
|
+
} catch (err) {
|
|
8740
|
+
throw new Error(
|
|
8741
|
+
`verify-findings: cannot read steps file ${source.stepsPath} \u2014 ${err instanceof Error ? err.message : String(err)}`
|
|
8742
|
+
);
|
|
8743
|
+
}
|
|
8744
|
+
if (!Array.isArray(steps) || steps.length === 0) {
|
|
8745
|
+
throw new Error(`verify-findings: ${source.stepsPath} is not a non-empty steps array`);
|
|
8746
|
+
}
|
|
8747
|
+
const step = checkStep(steps, at);
|
|
8748
|
+
if (!step.ok) return { replayable: false, reason: step.reason };
|
|
8749
|
+
return {
|
|
8750
|
+
replayable: true,
|
|
8751
|
+
resolved: {
|
|
8752
|
+
caseId: source.caseId ?? trajId2 ?? source.stepsPath,
|
|
8753
|
+
stepsPath: source.stepsPath,
|
|
8754
|
+
image: source.image,
|
|
8755
|
+
cwd: source.cwd,
|
|
8756
|
+
at,
|
|
8757
|
+
recordedReturncode: step.recordedReturncode,
|
|
8758
|
+
recordedStepTimeoutMs: null
|
|
8759
|
+
}
|
|
8760
|
+
};
|
|
8761
|
+
}
|
|
8762
|
+
const trajId = findingTrajectoryId(finding2);
|
|
8763
|
+
if (!trajId) {
|
|
8764
|
+
return {
|
|
8765
|
+
replayable: false,
|
|
8766
|
+
reason: "finding carries no trace://<trajectory>/ evidence ref naming its trajectory"
|
|
8767
|
+
};
|
|
8768
|
+
}
|
|
8769
|
+
const failures = [];
|
|
8770
|
+
for (const corpus of source.corpora) {
|
|
8771
|
+
const resolution = resolveCaseResources(corpus, trajId);
|
|
8772
|
+
if (!resolution.resolved) {
|
|
8773
|
+
failures.push(`${corpus.name}: ${resolution.reason}${resolution.detail ? ` (${resolution.detail})` : ""}`);
|
|
8774
|
+
continue;
|
|
8775
|
+
}
|
|
8776
|
+
const step = checkStep(resolution.resources.steps, at);
|
|
8777
|
+
if (!step.ok) return { replayable: false, reason: step.reason };
|
|
8778
|
+
return {
|
|
8779
|
+
replayable: true,
|
|
8780
|
+
resolved: {
|
|
8781
|
+
caseId: trajId,
|
|
8782
|
+
stepsPath: resolution.resources.stepsPath,
|
|
8783
|
+
image: resolution.resources.image,
|
|
8784
|
+
cwd: resolution.resources.cwd,
|
|
8785
|
+
at,
|
|
8786
|
+
recordedReturncode: step.recordedReturncode,
|
|
8787
|
+
recordedStepTimeoutMs: resolution.resources.recordedStepTimeoutMs
|
|
8788
|
+
}
|
|
8789
|
+
};
|
|
8790
|
+
}
|
|
8791
|
+
return {
|
|
8792
|
+
replayable: false,
|
|
8793
|
+
reason: `trajectory ${trajId} is not replayable in any corpus \u2014 ${failures.join("; ")}`
|
|
8794
|
+
};
|
|
8795
|
+
}
|
|
8796
|
+
function classifyVerdict(verdict) {
|
|
8797
|
+
if (!verdict.armA.failureSignatureMatch) return "divergent";
|
|
8798
|
+
if (verdict.armB?.failureVanished) return "fix-flipped";
|
|
8799
|
+
return "reproduced";
|
|
8800
|
+
}
|
|
8801
|
+
var DEFAULT_SANDBOX_BASE_URL = "http://127.0.0.1:4097";
|
|
8802
|
+
async function assertSandboxReachable(baseUrl) {
|
|
8803
|
+
let response;
|
|
8804
|
+
try {
|
|
8805
|
+
response = await fetch(new URL("/health", baseUrl), { signal: AbortSignal.timeout(5e3) });
|
|
8806
|
+
} catch (err) {
|
|
8807
|
+
throw new Error(
|
|
8808
|
+
`verify-findings: sandbox API unreachable at ${baseUrl} \u2014 ${err instanceof Error ? err.message : String(err)}. Executing findings requires a running sandbox orchestrator + SDK adapter (see docs/replay-verify.md); start them or pass --base-url.`
|
|
8809
|
+
);
|
|
8810
|
+
}
|
|
8811
|
+
if (!response.ok) {
|
|
8812
|
+
throw new Error(
|
|
8813
|
+
`verify-findings: sandbox API at ${baseUrl} answered /health with HTTP ${response.status} \u2014 refusing to run proofs against degraded infrastructure`
|
|
8814
|
+
);
|
|
8815
|
+
}
|
|
8816
|
+
}
|
|
8817
|
+
function receiptExecution(verdict) {
|
|
8818
|
+
return {
|
|
8819
|
+
image: verdict.image,
|
|
8820
|
+
cwd: verdict.cwd,
|
|
8821
|
+
recordedReturncode: verdict.recordedReturncode,
|
|
8822
|
+
signature: verdict.signature,
|
|
8823
|
+
signatureBasis: verdict.signatureBasis,
|
|
8824
|
+
armA: {
|
|
8825
|
+
command: verdict.armA.command,
|
|
8826
|
+
exitCode: verdict.armA.exitCode,
|
|
8827
|
+
wallMs: verdict.armA.wallMs,
|
|
8828
|
+
failureSignatureMatch: verdict.armA.failureSignatureMatch
|
|
8829
|
+
},
|
|
8830
|
+
armB: verdict.armB ? {
|
|
8831
|
+
command: verdict.armB.command,
|
|
8832
|
+
exitCode: verdict.armB.exitCode,
|
|
8833
|
+
wallMs: verdict.armB.wallMs,
|
|
8834
|
+
failureVanished: verdict.armB.failureVanished
|
|
8835
|
+
} : null,
|
|
8836
|
+
prefixExecuted: verdict.prefixExecuted,
|
|
8837
|
+
prefixDivergences: verdict.prefixDivergences.length,
|
|
8838
|
+
totalMs: verdict.timings.totalMs
|
|
8839
|
+
};
|
|
8840
|
+
}
|
|
8841
|
+
function writeReceipt(receiptDir, finding2, verification, execution) {
|
|
8842
|
+
const receipt = {
|
|
8843
|
+
schema_version: "1.0.0",
|
|
8844
|
+
produced_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
8845
|
+
finding_id: verification.finding_id,
|
|
8846
|
+
analyst_id: finding2.analyst_id ?? null,
|
|
8847
|
+
subject: verification.subject,
|
|
8848
|
+
claim: typeof finding2.claim === "string" ? finding2.claim.slice(0, 600) : null,
|
|
8849
|
+
trajectory_id: verification.trajectory_id,
|
|
8850
|
+
step: verification.step,
|
|
8851
|
+
verified: verification.verified,
|
|
8852
|
+
reason: verification.reason,
|
|
8853
|
+
execution,
|
|
8854
|
+
verdict_path: verification.verdict_path,
|
|
8855
|
+
deduplicated_with: verification.deduplicated_with
|
|
8856
|
+
};
|
|
8857
|
+
writeFileSync3(join22(receiptDir, "receipt.json"), `${JSON.stringify(receipt, null, 2)}
|
|
8858
|
+
`);
|
|
8859
|
+
}
|
|
8860
|
+
function receiptDirName(index, finding2) {
|
|
8861
|
+
const id = typeof finding2.finding_id === "string" && finding2.finding_id.length > 0 ? finding2.finding_id.replace(/[^A-Za-z0-9_-]/g, "_") : "finding";
|
|
8862
|
+
return `${String(index + 1).padStart(3, "0")}-${id}`;
|
|
8863
|
+
}
|
|
8864
|
+
async function verifyFindings(findings, options) {
|
|
8865
|
+
if (findings.length === 0) throw new Error("verify-findings: no findings to verify");
|
|
8866
|
+
mkdirSync3(options.out, { recursive: true });
|
|
8867
|
+
const resolutions = findings.map((finding2) => resolveFindingReplayability(finding2, options.source));
|
|
8868
|
+
const baseUrl = options.baseUrl ?? DEFAULT_SANDBOX_BASE_URL;
|
|
8869
|
+
if (resolutions.some((resolution) => resolution.replayable) && !options.backend) {
|
|
8870
|
+
if (!options.apiKey) {
|
|
8871
|
+
throw new Error(
|
|
8872
|
+
"verify-findings: replayable findings need a sandbox API key \u2014 export SANDBOX_API_KEY or pass --api-key-env"
|
|
8873
|
+
);
|
|
8874
|
+
}
|
|
8875
|
+
await assertSandboxReachable(baseUrl);
|
|
8876
|
+
}
|
|
8877
|
+
const preparer = options.source.kind === "corpus" && !options.backend ? options.source.preparer ?? dockerImagePreparer() : null;
|
|
8878
|
+
const preparedImages = /* @__PURE__ */ new Map();
|
|
8879
|
+
const executedByKey = /* @__PURE__ */ new Map();
|
|
8880
|
+
const verifications = [];
|
|
8881
|
+
let executions = 0;
|
|
8882
|
+
for (let index = 0; index < findings.length; index++) {
|
|
8883
|
+
const finding2 = findings[index];
|
|
8884
|
+
const resolution = resolutions[index];
|
|
8885
|
+
const receiptDir = join22(options.out, receiptDirName(index, finding2));
|
|
8886
|
+
mkdirSync3(receiptDir, { recursive: true });
|
|
8887
|
+
const identity = {
|
|
8888
|
+
finding_id: finding2.finding_id ?? null,
|
|
8889
|
+
subject: finding2.subject ?? null,
|
|
8890
|
+
trajectory_id: findingTrajectoryId(finding2)
|
|
8891
|
+
};
|
|
8892
|
+
if (!resolution.replayable) {
|
|
8893
|
+
const verification2 = {
|
|
8894
|
+
...identity,
|
|
8895
|
+
step: findingReplayStep(finding2),
|
|
8896
|
+
verified: "not-replayable",
|
|
8897
|
+
reason: resolution.reason,
|
|
8898
|
+
receipt: receiptDir,
|
|
8899
|
+
verdict_path: null,
|
|
8900
|
+
deduplicated_with: null
|
|
8901
|
+
};
|
|
8902
|
+
writeReceipt(receiptDir, finding2, verification2, null);
|
|
8903
|
+
verifications.push(verification2);
|
|
8904
|
+
options.onProgress?.(`${identity.finding_id ?? `finding ${index + 1}`}: not-replayable \u2014 ${resolution.reason}`);
|
|
8905
|
+
continue;
|
|
8906
|
+
}
|
|
8907
|
+
const resolved = resolution.resolved;
|
|
8908
|
+
const dedupeKey = `${resolved.caseId}::${resolved.at}::${options.fixCommand ?? ""}`;
|
|
8909
|
+
const prior = executedByKey.get(dedupeKey);
|
|
8910
|
+
if (prior) {
|
|
8911
|
+
const verification2 = {
|
|
8912
|
+
...identity,
|
|
8913
|
+
step: resolved.at,
|
|
8914
|
+
verified: prior.verified,
|
|
8915
|
+
reason: prior.reason,
|
|
8916
|
+
receipt: receiptDir,
|
|
8917
|
+
verdict_path: prior.verdict_path,
|
|
8918
|
+
deduplicated_with: prior.receipt
|
|
8919
|
+
};
|
|
8920
|
+
const priorVerdict = prior.verdict_path ? JSON.parse(readFileSync4(prior.verdict_path, "utf8")) : null;
|
|
8921
|
+
writeReceipt(receiptDir, finding2, verification2, priorVerdict ? receiptExecution(priorVerdict) : null);
|
|
8922
|
+
verifications.push(verification2);
|
|
8923
|
+
options.onProgress?.(
|
|
8924
|
+
`${identity.finding_id ?? `finding ${index + 1}`}: ${prior.verified} (shares proof with ${prior.finding_id ?? prior.receipt})`
|
|
8925
|
+
);
|
|
8926
|
+
continue;
|
|
8927
|
+
}
|
|
8928
|
+
let image = resolved.image;
|
|
8929
|
+
if (preparer) {
|
|
8930
|
+
const preparationKey = `${resolved.image}::${resolved.cwd}`;
|
|
8931
|
+
let preparation = preparedImages.get(preparationKey);
|
|
8932
|
+
if (!preparation) {
|
|
8933
|
+
const ensured = await preparer.ensure(resolved.image, resolved.cwd);
|
|
8934
|
+
preparation = ensured.succeeded ? { succeeded: true, image: ensured.value.derivedImage } : { succeeded: false, error: ensured.error };
|
|
8935
|
+
preparedImages.set(preparationKey, preparation);
|
|
8936
|
+
}
|
|
8937
|
+
if (!preparation.succeeded) {
|
|
8938
|
+
const verification2 = {
|
|
8939
|
+
...identity,
|
|
8940
|
+
step: resolved.at,
|
|
8941
|
+
verified: "not-replayable",
|
|
8942
|
+
reason: `replay image could not be prepared \u2014 ${preparation.error}`,
|
|
8943
|
+
receipt: receiptDir,
|
|
8944
|
+
verdict_path: null,
|
|
8945
|
+
deduplicated_with: null
|
|
8946
|
+
};
|
|
8947
|
+
writeReceipt(receiptDir, finding2, verification2, null);
|
|
8948
|
+
verifications.push(verification2);
|
|
8949
|
+
options.onProgress?.(`${identity.finding_id ?? `finding ${index + 1}`}: not-replayable \u2014 image preparation failed`);
|
|
8950
|
+
continue;
|
|
8951
|
+
}
|
|
8952
|
+
image = preparation.image;
|
|
8953
|
+
}
|
|
8954
|
+
options.onProgress?.(
|
|
8955
|
+
`${identity.finding_id ?? `finding ${index + 1}`}: executing arm A at step ${resolved.at} of ${resolved.caseId} on ${image}`
|
|
8956
|
+
);
|
|
8957
|
+
const verdict = await replayVerify({
|
|
8958
|
+
stepsPath: resolved.stepsPath,
|
|
8959
|
+
image,
|
|
8960
|
+
at: resolved.at,
|
|
8961
|
+
fixCommand: options.fixCommand,
|
|
8962
|
+
cwd: resolved.cwd,
|
|
8963
|
+
out: receiptDir,
|
|
8964
|
+
caseId: resolved.caseId,
|
|
8965
|
+
stepTimeoutMs: options.stepTimeoutMs ?? resolved.recordedStepTimeoutMs ?? void 0,
|
|
8966
|
+
prefixLimit: options.prefixLimit,
|
|
8967
|
+
backend: options.backend,
|
|
8968
|
+
apiKey: options.apiKey,
|
|
8969
|
+
baseUrl,
|
|
8970
|
+
maxLifetimeSeconds: options.maxLifetimeSeconds,
|
|
8971
|
+
onProgress: options.onProgress
|
|
8972
|
+
});
|
|
8973
|
+
executions += 1;
|
|
8974
|
+
const verification = {
|
|
8975
|
+
...identity,
|
|
8976
|
+
step: resolved.at,
|
|
8977
|
+
verified: classifyVerdict(verdict),
|
|
8978
|
+
reason: null,
|
|
8979
|
+
receipt: receiptDir,
|
|
8980
|
+
verdict_path: join22(receiptDir, "replay-verdict.json"),
|
|
8981
|
+
deduplicated_with: null
|
|
8982
|
+
};
|
|
8983
|
+
executedByKey.set(dedupeKey, verification);
|
|
8984
|
+
writeReceipt(receiptDir, finding2, verification, receiptExecution(verdict));
|
|
8985
|
+
verifications.push(verification);
|
|
8986
|
+
options.onProgress?.(`${identity.finding_id ?? `finding ${index + 1}`}: ${verification.verified}`);
|
|
8987
|
+
}
|
|
8988
|
+
const counts = {
|
|
8989
|
+
reproduced: 0,
|
|
8990
|
+
"fix-flipped": 0,
|
|
8991
|
+
divergent: 0,
|
|
8992
|
+
"not-replayable": 0
|
|
8993
|
+
};
|
|
8994
|
+
for (const verification of verifications) counts[verification.verified] += 1;
|
|
8995
|
+
const run = { out: options.out, verifications, counts, executions };
|
|
8996
|
+
writeFileSync3(
|
|
8997
|
+
join22(options.out, "verifications.json"),
|
|
8998
|
+
`${JSON.stringify({ schema_version: "1.0.0", ...run }, null, 2)}
|
|
8999
|
+
`
|
|
9000
|
+
);
|
|
9001
|
+
return run;
|
|
9002
|
+
}
|
|
9003
|
+
function verdictCell(verification) {
|
|
9004
|
+
switch (verification.verified) {
|
|
9005
|
+
case "reproduced":
|
|
9006
|
+
return "**VERIFIED** \u2014 reproduced";
|
|
9007
|
+
case "fix-flipped":
|
|
9008
|
+
return "**VERIFIED** \u2014 fix-flipped";
|
|
9009
|
+
case "divergent":
|
|
9010
|
+
return "DIVERGENT \u2014 recorded failure did not reproduce";
|
|
9011
|
+
case "not-replayable":
|
|
9012
|
+
return `UNVERIFIABLE \u2014 ${verification.reason ?? "no reason recorded"}`;
|
|
9013
|
+
}
|
|
9014
|
+
}
|
|
9015
|
+
function renderVerifiedFindingsSection(run) {
|
|
9016
|
+
const lines = ["## Verified findings (executed replay)", ""];
|
|
9017
|
+
const total = run.verifications.length;
|
|
9018
|
+
lines.push(
|
|
9019
|
+
`${total} finding(s) \u2192 ${run.counts.reproduced} reproduced, ${run.counts["fix-flipped"]} fix-flipped, ${run.counts.divergent} divergent, ${run.counts["not-replayable"]} not replayable (${run.executions} sandbox execution(s); findings accusing the same step share one proof).`
|
|
9020
|
+
);
|
|
9021
|
+
lines.push("");
|
|
9022
|
+
lines.push("| Finding | Subject | Step | Verdict | Receipt |");
|
|
9023
|
+
lines.push("|---|---|---:|---|---|");
|
|
9024
|
+
for (const verification of run.verifications) {
|
|
9025
|
+
const shared = verification.deduplicated_with ? " (shared proof)" : "";
|
|
9026
|
+
lines.push(
|
|
9027
|
+
`| \`${verification.finding_id ?? "\u2014"}\` | \`${verification.subject ?? "\u2014"}\` | ${verification.step ?? "\u2014"} | ${verdictCell(verification)} | \`${verification.receipt}\`${shared} |`
|
|
9028
|
+
);
|
|
9029
|
+
}
|
|
9030
|
+
lines.push("");
|
|
9031
|
+
lines.push(
|
|
9032
|
+
"VERIFIED = the accused step was re-executed in a sandbox after replaying the trajectory prefix, and the recorded failure signature reproduced (fix-flipped: a corrected command additionally made it vanish). Each receipt directory carries receipt.json and, when executed, replay-verdict.json + report.md with real stdout/stderr."
|
|
9033
|
+
);
|
|
9034
|
+
lines.push("");
|
|
9035
|
+
return lines.join("\n");
|
|
9036
|
+
}
|
|
9037
|
+
function readFindingsFile(path) {
|
|
9038
|
+
const parsed = JSON.parse(readFileSync4(path, "utf8"));
|
|
9039
|
+
const array = Array.isArray(parsed) ? parsed : parsed && typeof parsed === "object" && Array.isArray(parsed.findings) ? parsed.findings : null;
|
|
9040
|
+
if (!array) {
|
|
9041
|
+
throw new Error(`${path} is neither a findings array nor an object with a findings array`);
|
|
9042
|
+
}
|
|
9043
|
+
for (const entry of array) {
|
|
9044
|
+
if (!entry || typeof entry !== "object") {
|
|
9045
|
+
throw new Error(`${path}: every finding must be an object, got ${JSON.stringify(entry)}`);
|
|
9046
|
+
}
|
|
9047
|
+
}
|
|
9048
|
+
return array;
|
|
9049
|
+
}
|
|
9050
|
+
function verifyFindingsUsage() {
|
|
9051
|
+
return `traces verify-findings \u2014 execute analyst findings as sandbox replay proofs
|
|
9052
|
+
|
|
9053
|
+
Usage:
|
|
9054
|
+
traces verify-findings --findings FINDINGS.json --out DIR \\
|
|
9055
|
+
( --steps STEPS.json --image IMG --cwd DIR [--case ID]
|
|
9056
|
+
| --corpus name=<labels.json>::<preparedDir> [--corpus ...] ) \\
|
|
9057
|
+
[--fix-command CMD] [--step-timeout MS] [--prefix-limit N] \\
|
|
9058
|
+
[--base-url URL] [--api-key-env VAR] [--max-lifetime SECONDS]
|
|
9059
|
+
|
|
9060
|
+
--findings JSON array of analyst findings (or an object with a findings
|
|
9061
|
+
array) \u2014 the shape agent-eval analysts emit: subject
|
|
9062
|
+
incorrect-step-<n>, metadata.block_first_step, trace:// evidence
|
|
9063
|
+
--out receipt root; one directory per finding + verifications.json
|
|
9064
|
+
--steps/--image/--cwd
|
|
9065
|
+
verify against one trajectory; the image must be replay-ready
|
|
9066
|
+
(uid-1000 derived), exactly like traces replay-verify
|
|
9067
|
+
--case trajectory id of --steps; findings citing other trajectories
|
|
9068
|
+
are honestly marked not-replayable
|
|
9069
|
+
--corpus resolve each finding's trajectory in CodeTraceBench corpora;
|
|
9070
|
+
images are derived via the batch uid-1000 preparer (docker)
|
|
9071
|
+
--fix-command corrected step for arm B; a reproduced finding whose fix
|
|
9072
|
+
flips becomes fix-flipped
|
|
9073
|
+
--base-url sandbox API url (default $SANDBOX_API_URL or ${DEFAULT_SANDBOX_BASE_URL})
|
|
9074
|
+
--api-key-env env var holding the sandbox API key (default SANDBOX_API_KEY)
|
|
9075
|
+
|
|
9076
|
+
Every finding gets a verdict: reproduced | fix-flipped | divergent | not-replayable,
|
|
9077
|
+
with a receipt directory carrying the executed evidence or the precise reason.
|
|
9078
|
+
Requires a running sandbox orchestrator when any finding is replayable \u2014 infra
|
|
9079
|
+
absence is an error, never a silent skip. See docs/replay-verify.md for setup.
|
|
9080
|
+
`;
|
|
9081
|
+
}
|
|
9082
|
+
function parseVerifyFindingsArgs(argv) {
|
|
9083
|
+
const values = /* @__PURE__ */ new Map();
|
|
9084
|
+
const corpora = [];
|
|
9085
|
+
for (let i = 0; i < argv.length; i++) {
|
|
9086
|
+
const arg = argv[i];
|
|
9087
|
+
if (arg === "--help" || arg === "-h") return "help";
|
|
9088
|
+
if (!arg.startsWith("--")) throw new Error(`unexpected positional argument: ${arg}`);
|
|
9089
|
+
const value = argv[++i];
|
|
9090
|
+
if (value === void 0) throw new Error(`missing value for ${arg}`);
|
|
9091
|
+
if (arg === "--corpus") {
|
|
9092
|
+
corpora.push(parseCorpusFlag(value));
|
|
9093
|
+
continue;
|
|
9094
|
+
}
|
|
9095
|
+
values.set(arg, value);
|
|
9096
|
+
}
|
|
9097
|
+
const required = (flag) => {
|
|
9098
|
+
const v = values.get(flag);
|
|
9099
|
+
if (v === void 0) throw new Error(`verify-findings: ${flag} is required`);
|
|
9100
|
+
return v;
|
|
9101
|
+
};
|
|
9102
|
+
const optionalNumber = (flag) => {
|
|
9103
|
+
const v = values.get(flag);
|
|
9104
|
+
if (v === void 0) return void 0;
|
|
9105
|
+
const n = Number(v);
|
|
9106
|
+
if (!Number.isFinite(n)) throw new Error(`verify-findings: ${flag} must be a number, got ${v}`);
|
|
9107
|
+
return n;
|
|
9108
|
+
};
|
|
9109
|
+
const findingsPath = required("--findings");
|
|
9110
|
+
const out = required("--out");
|
|
9111
|
+
const direct = values.has("--steps") || values.has("--image") || values.has("--cwd");
|
|
9112
|
+
if (direct && corpora.length > 0) {
|
|
9113
|
+
throw new Error("verify-findings: pass --steps/--image/--cwd or --corpus, not both");
|
|
9114
|
+
}
|
|
9115
|
+
let source;
|
|
9116
|
+
if (direct) {
|
|
9117
|
+
source = {
|
|
9118
|
+
kind: "direct",
|
|
9119
|
+
stepsPath: required("--steps"),
|
|
9120
|
+
image: required("--image"),
|
|
9121
|
+
cwd: required("--cwd"),
|
|
9122
|
+
...values.has("--case") ? { caseId: values.get("--case") } : {}
|
|
9123
|
+
};
|
|
9124
|
+
} else if (corpora.length > 0) {
|
|
9125
|
+
source = { kind: "corpus", corpora };
|
|
9126
|
+
} else {
|
|
9127
|
+
throw new Error("verify-findings: a replay source is required \u2014 --steps/--image/--cwd or --corpus");
|
|
9128
|
+
}
|
|
9129
|
+
return {
|
|
9130
|
+
findingsPath,
|
|
9131
|
+
source,
|
|
9132
|
+
out,
|
|
9133
|
+
fixCommand: values.get("--fix-command"),
|
|
9134
|
+
stepTimeoutMs: optionalNumber("--step-timeout"),
|
|
9135
|
+
prefixLimit: optionalNumber("--prefix-limit"),
|
|
9136
|
+
baseUrl: values.get("--base-url") ?? process.env.SANDBOX_API_URL ?? DEFAULT_SANDBOX_BASE_URL,
|
|
9137
|
+
apiKeyEnv: values.get("--api-key-env") ?? "SANDBOX_API_KEY",
|
|
9138
|
+
maxLifetimeSeconds: optionalNumber("--max-lifetime")
|
|
9139
|
+
};
|
|
9140
|
+
}
|
|
9141
|
+
async function cmdVerifyFindings(argv) {
|
|
9142
|
+
const parsed = parseVerifyFindingsArgs(argv);
|
|
9143
|
+
if (parsed === "help") {
|
|
9144
|
+
process.stdout.write(verifyFindingsUsage());
|
|
9145
|
+
return;
|
|
9146
|
+
}
|
|
9147
|
+
const findings = readFindingsFile(parsed.findingsPath);
|
|
9148
|
+
const apiKey = process.env[parsed.apiKeyEnv];
|
|
9149
|
+
const run = await verifyFindings(findings, {
|
|
9150
|
+
source: parsed.source,
|
|
9151
|
+
out: parsed.out,
|
|
9152
|
+
fixCommand: parsed.fixCommand,
|
|
9153
|
+
stepTimeoutMs: parsed.stepTimeoutMs,
|
|
9154
|
+
prefixLimit: parsed.prefixLimit,
|
|
9155
|
+
apiKey,
|
|
9156
|
+
baseUrl: parsed.baseUrl,
|
|
9157
|
+
maxLifetimeSeconds: parsed.maxLifetimeSeconds,
|
|
9158
|
+
onProgress: (message) => process.stderr.write(`${message}
|
|
9159
|
+
`)
|
|
9160
|
+
});
|
|
9161
|
+
process.stdout.write(`${JSON.stringify(run, null, 2)}
|
|
9162
|
+
`);
|
|
9163
|
+
process.stderr.write(
|
|
9164
|
+
`verify-findings: ${run.verifications.length} finding(s) \u2192 ${run.counts.reproduced} reproduced, ${run.counts["fix-flipped"]} fix-flipped, ${run.counts.divergent} divergent, ${run.counts["not-replayable"]} not-replayable (${run.executions} execution(s)) \u2192 ${run.out}
|
|
9165
|
+
`
|
|
9166
|
+
);
|
|
9167
|
+
}
|
|
9168
|
+
|
|
8222
9169
|
// src/external.ts
|
|
8223
9170
|
import { spawn } from "child_process";
|
|
8224
9171
|
import { readdir as readdir14, readFile as readFile7 } from "fs/promises";
|
|
@@ -8497,13 +9444,13 @@ function terminateWindowsProcessTree(child) {
|
|
|
8497
9444
|
forceKill(child);
|
|
8498
9445
|
return Promise.resolve();
|
|
8499
9446
|
}
|
|
8500
|
-
return new Promise((
|
|
9447
|
+
return new Promise((resolve8) => {
|
|
8501
9448
|
let finished = false;
|
|
8502
9449
|
const finish = () => {
|
|
8503
9450
|
if (finished) return;
|
|
8504
9451
|
finished = true;
|
|
8505
9452
|
forceKill(child);
|
|
8506
|
-
|
|
9453
|
+
resolve8();
|
|
8507
9454
|
};
|
|
8508
9455
|
let killer;
|
|
8509
9456
|
try {
|
|
@@ -8530,7 +9477,7 @@ function signalProcess(pid, signal) {
|
|
|
8530
9477
|
}
|
|
8531
9478
|
}
|
|
8532
9479
|
function readProcessTableWithPs() {
|
|
8533
|
-
return new Promise((
|
|
9480
|
+
return new Promise((resolve8, reject) => {
|
|
8534
9481
|
let child;
|
|
8535
9482
|
try {
|
|
8536
9483
|
child = spawn("ps", ["-A", "-o", "pid=", "-o", "ppid="], {
|
|
@@ -8584,7 +9531,7 @@ function readProcessTableWithPs() {
|
|
|
8584
9531
|
reject(new Error("ps: returned no process records"));
|
|
8585
9532
|
return;
|
|
8586
9533
|
}
|
|
8587
|
-
|
|
9534
|
+
resolve8(rows);
|
|
8588
9535
|
});
|
|
8589
9536
|
});
|
|
8590
9537
|
});
|
|
@@ -8719,7 +9666,7 @@ function runCommand(command, args = [], opts = {}) {
|
|
|
8719
9666
|
throw new RangeError("maxBuffer must be a positive safe integer");
|
|
8720
9667
|
}
|
|
8721
9668
|
if (opts.signal?.aborted) return Promise.reject(abortError(opts.signal));
|
|
8722
|
-
return new Promise((
|
|
9669
|
+
return new Promise((resolve8, reject) => {
|
|
8723
9670
|
let child;
|
|
8724
9671
|
try {
|
|
8725
9672
|
child = spawn(command, [...args], { detached: process.platform !== "win32" });
|
|
@@ -8751,7 +9698,7 @@ function runCommand(command, args = [], opts = {}) {
|
|
|
8751
9698
|
} else if (spawnError) {
|
|
8752
9699
|
reject(spawnError);
|
|
8753
9700
|
} else {
|
|
8754
|
-
|
|
9701
|
+
resolve8({ stdout: out, stderr: err, code: closeCode });
|
|
8755
9702
|
}
|
|
8756
9703
|
};
|
|
8757
9704
|
function requestTermination(error) {
|
|
@@ -8983,18 +9930,18 @@ function commandRedactor(spec) {
|
|
|
8983
9930
|
import { createHash as createHash9 } from "crypto";
|
|
8984
9931
|
import { mkdir as mkdir3, mkdtemp as mkdtemp3, readFile as readFile8, readdir as readdir15, writeFile as writeFile4 } from "fs/promises";
|
|
8985
9932
|
import { endianness, tmpdir as tmpdir4 } from "os";
|
|
8986
|
-
import { join as
|
|
9933
|
+
import { join as join23, resolve as resolve6 } from "path";
|
|
8987
9934
|
var HODOSCOPE_VERSION = "0.2.4";
|
|
8988
9935
|
async function writeHodoscopeInput(otlpPath, options = {}) {
|
|
8989
|
-
const explicitDirectory = options.directory ?
|
|
8990
|
-
const directory = explicitDirectory ?? await mkdtemp3(
|
|
9936
|
+
const explicitDirectory = options.directory ? resolve6(options.directory) : void 0;
|
|
9937
|
+
const directory = explicitDirectory ?? await mkdtemp3(join23(tmpdir4(), "traces-hodoscope-input-"));
|
|
8991
9938
|
if (explicitDirectory) {
|
|
8992
9939
|
await mkdir3(directory, { recursive: true });
|
|
8993
9940
|
if ((await readdir15(directory)).length > 0) {
|
|
8994
9941
|
throw new Error(`Hodoscope explicit directory must be empty: ${directory}`);
|
|
8995
9942
|
}
|
|
8996
9943
|
}
|
|
8997
|
-
const samples =
|
|
9944
|
+
const samples = join23(directory, "samples");
|
|
8998
9945
|
await mkdir3(samples);
|
|
8999
9946
|
const rows = [];
|
|
9000
9947
|
for await (const value of readJsonl(otlpPath)) rows.push(parseOpenInferenceRow(value));
|
|
@@ -9020,7 +9967,7 @@ async function writeHodoscopeInput(otlpPath, options = {}) {
|
|
|
9020
9967
|
}
|
|
9021
9968
|
actionCount += actions;
|
|
9022
9969
|
const file = `${createHash9("sha256").update(traceId).digest("hex").slice(0, 24)}.json`;
|
|
9023
|
-
await writeFile4(
|
|
9970
|
+
await writeFile4(join23(samples, file), `${JSON.stringify(trajectory, null, 2)}
|
|
9024
9971
|
`, "utf8");
|
|
9025
9972
|
}
|
|
9026
9973
|
return {
|
|
@@ -9042,12 +9989,12 @@ function hodoscopeAnalyzer(options = {}) {
|
|
|
9042
9989
|
return {
|
|
9043
9990
|
name: "hodoscope",
|
|
9044
9991
|
async analyze(otlpPath, runOptions2 = {}) {
|
|
9045
|
-
const outputRoot = options.outputDir ?
|
|
9992
|
+
const outputRoot = options.outputDir ? resolve6(options.outputDir) : tmpdir4();
|
|
9046
9993
|
await mkdir3(outputRoot, { recursive: true });
|
|
9047
|
-
const directory = await mkdtemp3(
|
|
9048
|
-
const input =
|
|
9049
|
-
const analysisFile =
|
|
9050
|
-
const samplesFile =
|
|
9994
|
+
const directory = await mkdtemp3(join23(outputRoot, "traces-hodoscope-"));
|
|
9995
|
+
const input = join23(directory, "input");
|
|
9996
|
+
const analysisFile = join23(directory, "analysis.hodoscope.json");
|
|
9997
|
+
const samplesFile = join23(directory, "samples.json");
|
|
9051
9998
|
try {
|
|
9052
9999
|
const prepared = await writeHodoscopeInput(otlpPath, {
|
|
9053
10000
|
directory: input,
|
|
@@ -9246,7 +10193,7 @@ async function readHodoscopeAnalysis(path, expected) {
|
|
|
9246
10193
|
throw new Error("Hodoscope analysis.created_at must be a valid date-time");
|
|
9247
10194
|
}
|
|
9248
10195
|
const source = requiredString2(value.source, "analysis.source");
|
|
9249
|
-
if (
|
|
10196
|
+
if (resolve6(source) !== resolve6(expected.source)) {
|
|
9250
10197
|
throw new Error(
|
|
9251
10198
|
`Hodoscope analysis.source '${source}' does not match requested input '${expected.source}'`
|
|
9252
10199
|
);
|
|
@@ -9759,7 +10706,7 @@ async function applyRedactor(spans, redactor) {
|
|
|
9759
10706
|
import { createHash as createHash10 } from "crypto";
|
|
9760
10707
|
import { mkdtemp as mkdtemp4, readFile as readFile9, writeFile as writeFile5 } from "fs/promises";
|
|
9761
10708
|
import { tmpdir as tmpdir5 } from "os";
|
|
9762
|
-
import { join as
|
|
10709
|
+
import { join as join24 } from "path";
|
|
9763
10710
|
import {
|
|
9764
10711
|
firstNumberAttr,
|
|
9765
10712
|
LLM_CACHED_TOKEN_ATTR_KEYS,
|
|
@@ -10355,7 +11302,7 @@ async function exportTraceEvidenceFile(inputPath, opts = {}) {
|
|
|
10355
11302
|
}
|
|
10356
11303
|
async function writeTraceEvidenceExportFile(inputPath, outPath, opts = {}) {
|
|
10357
11304
|
const result = await exportTraceEvidenceFile(inputPath, opts);
|
|
10358
|
-
const path = outPath ??
|
|
11305
|
+
const path = outPath ?? join24(await mkdtemp4(join24(tmpdir5(), "traces-export-")), "spans.openinference.jsonl");
|
|
10359
11306
|
await writeFile5(path, serializeSpans(result.spans), "utf8");
|
|
10360
11307
|
return { ...result, path };
|
|
10361
11308
|
}
|
|
@@ -10363,7 +11310,7 @@ async function writeTraceEvidenceExportFile(inputPath, outPath, opts = {}) {
|
|
|
10363
11310
|
// src/inspect.ts
|
|
10364
11311
|
import { mkdtemp as mkdtemp5, readFile as readFile10, writeFile as writeFile6 } from "fs/promises";
|
|
10365
11312
|
import { tmpdir as tmpdir6 } from "os";
|
|
10366
|
-
import { basename as basename13, join as
|
|
11313
|
+
import { basename as basename13, join as join25 } from "path";
|
|
10367
11314
|
var severityRank = {
|
|
10368
11315
|
high: 0,
|
|
10369
11316
|
medium: 1,
|
|
@@ -10652,14 +11599,14 @@ async function readSessionIndexFile(path) {
|
|
|
10652
11599
|
return parsed;
|
|
10653
11600
|
}
|
|
10654
11601
|
async function writeInspectionReportFile(report, outPath) {
|
|
10655
|
-
const path = outPath ??
|
|
11602
|
+
const path = outPath ?? join25(await mkdtemp5(join25(tmpdir6(), "traces-inspect-")), "inspection-report.md");
|
|
10656
11603
|
await writeFile6(path, renderInspectionReport(report), "utf8");
|
|
10657
11604
|
return path;
|
|
10658
11605
|
}
|
|
10659
11606
|
|
|
10660
11607
|
// src/adoption.ts
|
|
10661
11608
|
import { readFile as readFile11 } from "fs/promises";
|
|
10662
|
-
import { join as
|
|
11609
|
+
import { join as join26 } from "path";
|
|
10663
11610
|
function toolName2(s) {
|
|
10664
11611
|
const n = s.attributes["tool.name"];
|
|
10665
11612
|
if (typeof n === "string") return n;
|
|
@@ -10779,7 +11726,7 @@ async function readLoopRuns(cwds, sessionIds) {
|
|
|
10779
11726
|
seenCwds.add(cwd);
|
|
10780
11727
|
const firstSourceByRun = /* @__PURE__ */ new Map();
|
|
10781
11728
|
for (const artifactDir of [".agent", ".evolve"]) {
|
|
10782
|
-
const path =
|
|
11729
|
+
const path = join26(cwd, artifactDir, "skill-runs.jsonl");
|
|
10783
11730
|
let raw;
|
|
10784
11731
|
try {
|
|
10785
11732
|
raw = await readFile11(path, "utf8");
|
|
@@ -12191,7 +13138,7 @@ function renderAdoption(ar) {
|
|
|
12191
13138
|
// src/improvement.ts
|
|
12192
13139
|
import { access, mkdir as mkdir4, mkdtemp as mkdtemp6, writeFile as writeFile7 } from "fs/promises";
|
|
12193
13140
|
import { tmpdir as tmpdir7 } from "os";
|
|
12194
|
-
import { dirname as dirname7, join as
|
|
13141
|
+
import { dirname as dirname7, join as join27, resolve as resolve7 } from "path";
|
|
12195
13142
|
import { pathToFileURL } from "url";
|
|
12196
13143
|
import {
|
|
12197
13144
|
AnalystRegistry,
|
|
@@ -12549,7 +13496,7 @@ function registryFromConfig(config) {
|
|
|
12549
13496
|
async function existingConfigPath(path) {
|
|
12550
13497
|
const candidates = path ? [path] : ["traces.config.mjs", "traces.config.js", "traces.config.cjs"];
|
|
12551
13498
|
for (const candidate of candidates) {
|
|
12552
|
-
const resolved =
|
|
13499
|
+
const resolved = resolve7(candidate);
|
|
12553
13500
|
try {
|
|
12554
13501
|
await access(resolved);
|
|
12555
13502
|
return resolved;
|
|
@@ -12698,13 +13645,13 @@ async function runTraceStoreInvestigation(opts) {
|
|
|
12698
13645
|
};
|
|
12699
13646
|
}
|
|
12700
13647
|
async function writeTraceImprovementArtifacts(result, outDir) {
|
|
12701
|
-
const directory = outDir ?
|
|
13648
|
+
const directory = outDir ? resolve7(outDir) : await mkdtemp6(join27(tmpdir7(), "traces-improvement-"));
|
|
12702
13649
|
await mkdir4(directory, { recursive: true });
|
|
12703
13650
|
const paths = {
|
|
12704
13651
|
directory,
|
|
12705
|
-
result:
|
|
12706
|
-
evidence:
|
|
12707
|
-
report:
|
|
13652
|
+
result: join27(directory, "result.json"),
|
|
13653
|
+
evidence: join27(directory, "evidence.jsonl"),
|
|
13654
|
+
report: join27(directory, "report.md"),
|
|
12708
13655
|
traces: result.otlpPath
|
|
12709
13656
|
};
|
|
12710
13657
|
const { report, ...machineResult } = result;
|
|
@@ -12716,11 +13663,11 @@ async function writeTraceImprovementArtifacts(result, outDir) {
|
|
|
12716
13663
|
return paths;
|
|
12717
13664
|
}
|
|
12718
13665
|
async function runTraceImprovement(opts) {
|
|
12719
|
-
const directory = opts.outDir ?
|
|
13666
|
+
const directory = opts.outDir ? resolve7(opts.outDir) : await mkdtemp6(join27(tmpdir7(), "traces-improvement-"));
|
|
12720
13667
|
await mkdir4(directory, { recursive: true });
|
|
12721
13668
|
const investigation = await runTraceInvestigation({
|
|
12722
13669
|
...opts,
|
|
12723
|
-
otlpOutPath: opts.otlpOutPath ??
|
|
13670
|
+
otlpOutPath: opts.otlpOutPath ?? join27(directory, "traces.otlp.jsonl")
|
|
12724
13671
|
});
|
|
12725
13672
|
const result = {
|
|
12726
13673
|
...investigation,
|
|
@@ -12732,7 +13679,7 @@ async function runTraceImprovement(opts) {
|
|
|
12732
13679
|
};
|
|
12733
13680
|
}
|
|
12734
13681
|
async function saveReport(path, report) {
|
|
12735
|
-
await mkdir4(dirname7(
|
|
13682
|
+
await mkdir4(dirname7(resolve7(path)), { recursive: true });
|
|
12736
13683
|
await writeFile7(path, report, "utf8");
|
|
12737
13684
|
}
|
|
12738
13685
|
|
|
@@ -12741,18 +13688,18 @@ import { createHash as createHash11 } from "crypto";
|
|
|
12741
13688
|
var VERIFY_RE = /\b(test|tests|vitest|jest|pytest|go test|cargo test|tsc|typecheck|lint|biome|eslint|build|check:invariants|preflight)\b/i;
|
|
12742
13689
|
var CHANGE_RE = /\b(apply_patch|edit|write|create|delete|rename|replace|update|git apply|sed -i|perl -pi|tee\s|cat\s+>)/i;
|
|
12743
13690
|
var CLAIM_DONE_RE = /\b(done|fixed|complete|completed|implemented|merged|verified|all green|tests pass|ready|works now)\b/i;
|
|
12744
|
-
var sleep = (ms2, signal) => new Promise((
|
|
13691
|
+
var sleep = (ms2, signal) => new Promise((resolve8) => {
|
|
12745
13692
|
if (signal?.aborted) {
|
|
12746
|
-
|
|
13693
|
+
resolve8();
|
|
12747
13694
|
return;
|
|
12748
13695
|
}
|
|
12749
13696
|
const onAbort = () => {
|
|
12750
13697
|
clearTimeout(t);
|
|
12751
|
-
|
|
13698
|
+
resolve8();
|
|
12752
13699
|
};
|
|
12753
13700
|
const t = setTimeout(() => {
|
|
12754
13701
|
signal?.removeEventListener("abort", onAbort);
|
|
12755
|
-
|
|
13702
|
+
resolve8();
|
|
12756
13703
|
}, ms2);
|
|
12757
13704
|
signal?.addEventListener("abort", onAbort, { once: true });
|
|
12758
13705
|
});
|
|
@@ -13163,7 +14110,7 @@ async function streamSessions(opts) {
|
|
|
13163
14110
|
|
|
13164
14111
|
// src/supervisor-run-context.ts
|
|
13165
14112
|
import { readFile as readFile12, stat as stat14 } from "fs/promises";
|
|
13166
|
-
import { basename as basename14, join as
|
|
14113
|
+
import { basename as basename14, join as join28 } from "path";
|
|
13167
14114
|
import { materializeTreeView } from "@tangle-network/agent-runtime/kernel";
|
|
13168
14115
|
import {
|
|
13169
14116
|
NO_SOURCE_LIMITS
|
|
@@ -13324,10 +14271,10 @@ function parseInstant(v) {
|
|
|
13324
14271
|
return Number.isFinite(t) ? t : null;
|
|
13325
14272
|
}
|
|
13326
14273
|
async function readFileRunContext(runDir, opts = {}) {
|
|
13327
|
-
const journalText = await readMaybe(
|
|
14274
|
+
const journalText = await readMaybe(join28(runDir, SPAWN_JOURNAL_FILE));
|
|
13328
14275
|
const journal = readRunContextJournal(journalText);
|
|
13329
|
-
const coordination = readCoordinationLog(await readMaybe(
|
|
13330
|
-
const result = await readMaybe(
|
|
14276
|
+
const coordination = readCoordinationLog(await readMaybe(join28(runDir, COORDINATION_LOG_FILE)));
|
|
14277
|
+
const result = await readMaybe(join28(runDir, RESULT_FILE));
|
|
13331
14278
|
return {
|
|
13332
14279
|
runRef: runDir,
|
|
13333
14280
|
instanceId: opts.instanceId !== void 0 ? opts.instanceId : journal.root,
|
|
@@ -13350,7 +14297,7 @@ async function readFileRunContext(runDir, opts = {}) {
|
|
|
13350
14297
|
harnessWorkerTokens: null,
|
|
13351
14298
|
harnessMissingReason: "the file run context journals each child\u2019s own settled spend, so no external harness join is used",
|
|
13352
14299
|
limits: sourceLimits(journal, coordination),
|
|
13353
|
-
rootTranscriptRef: journal.present ?
|
|
14300
|
+
rootTranscriptRef: journal.present ? join28(runDir, SPAWN_JOURNAL_FILE) : null,
|
|
13354
14301
|
traceCommand: `traces watch ${runDir}`
|
|
13355
14302
|
};
|
|
13356
14303
|
}
|
|
@@ -13358,7 +14305,7 @@ function fileRunContextSupervisorRunReader(runDir, opts = {}) {
|
|
|
13358
14305
|
return { runRef: runDir, read: () => readFileRunContext(runDir, opts) };
|
|
13359
14306
|
}
|
|
13360
14307
|
async function isFileRunContextDir(runDir) {
|
|
13361
|
-
return stat14(
|
|
14308
|
+
return stat14(join28(runDir, SPAWN_JOURNAL_FILE)).then((s) => s.isFile()).catch(() => false);
|
|
13362
14309
|
}
|
|
13363
14310
|
function flattenJournal(journal) {
|
|
13364
14311
|
return journal.events.map((event) => {
|
|
@@ -13504,9 +14451,9 @@ function mergeRolls(a, b) {
|
|
|
13504
14451
|
};
|
|
13505
14452
|
}
|
|
13506
14453
|
async function readRunContextSnapshot(runDir) {
|
|
13507
|
-
const journal = readRunContextJournal(await readMaybe(
|
|
13508
|
-
const coordination = readCoordinationLog(await readMaybe(
|
|
13509
|
-
const resultText = await readMaybe(
|
|
14454
|
+
const journal = readRunContextJournal(await readMaybe(join28(runDir, SPAWN_JOURNAL_FILE)));
|
|
14455
|
+
const coordination = readCoordinationLog(await readMaybe(join28(runDir, COORDINATION_LOG_FILE)));
|
|
14456
|
+
const resultText = await readMaybe(join28(runDir, RESULT_FILE));
|
|
13510
14457
|
const result = parseResult(resultText);
|
|
13511
14458
|
const view = safeTreeView(journal.events);
|
|
13512
14459
|
const nodes = view === null ? [] : orderNodes(view, journal);
|
|
@@ -13689,7 +14636,7 @@ function wrap(text2, indent) {
|
|
|
13689
14636
|
|
|
13690
14637
|
// src/run-span-tree.ts
|
|
13691
14638
|
import { readdir as readdir16 } from "fs/promises";
|
|
13692
|
-
import { join as
|
|
14639
|
+
import { join as join29 } from "path";
|
|
13693
14640
|
import {
|
|
13694
14641
|
firstNumberAttr as firstNumberAttr3,
|
|
13695
14642
|
LLM_COST_ATTR_KEYS as LLM_COST_ATTR_KEYS2,
|
|
@@ -13962,7 +14909,7 @@ async function findSpanSourceFiles(dir) {
|
|
|
13962
14909
|
const found = [];
|
|
13963
14910
|
const entries = await readdir16(dir, { withFileTypes: true }).catch(() => []);
|
|
13964
14911
|
for (const entry of entries) {
|
|
13965
|
-
const child =
|
|
14912
|
+
const child = join29(dir, entry.name);
|
|
13966
14913
|
if (entry.isFile()) {
|
|
13967
14914
|
if (SPAN_FILE_PATTERN.test(entry.name)) found.push(child);
|
|
13968
14915
|
continue;
|
|
@@ -13970,7 +14917,7 @@ async function findSpanSourceFiles(dir) {
|
|
|
13970
14917
|
if (!entry.isDirectory()) continue;
|
|
13971
14918
|
const nested = await readdir16(child, { withFileTypes: true }).catch(() => []);
|
|
13972
14919
|
for (const inner of nested) {
|
|
13973
|
-
if (inner.isFile() && SPAN_FILE_PATTERN.test(inner.name)) found.push(
|
|
14920
|
+
if (inner.isFile() && SPAN_FILE_PATTERN.test(inner.name)) found.push(join29(child, inner.name));
|
|
13974
14921
|
}
|
|
13975
14922
|
}
|
|
13976
14923
|
return found.sort();
|
|
@@ -13978,7 +14925,7 @@ async function findSpanSourceFiles(dir) {
|
|
|
13978
14925
|
|
|
13979
14926
|
// src/supervisor-run-watch.ts
|
|
13980
14927
|
import { readdir as readdir17 } from "fs/promises";
|
|
13981
|
-
import { join as
|
|
14928
|
+
import { join as join30 } from "path";
|
|
13982
14929
|
function budget(node) {
|
|
13983
14930
|
const parts = [`${int(node.budget.maxIterations)}it`, `${int(node.budget.maxTokens)}tok`];
|
|
13984
14931
|
if (node.budget.maxUsd !== void 0) parts.push(`$${node.budget.maxUsd.toFixed(2)}`);
|
|
@@ -14122,7 +15069,7 @@ async function resolveRunContextDir(dir) {
|
|
|
14122
15069
|
const candidates = [];
|
|
14123
15070
|
for (const entry of entries) {
|
|
14124
15071
|
if (!entry.isDirectory()) continue;
|
|
14125
|
-
const child =
|
|
15072
|
+
const child = join30(dir, entry.name);
|
|
14126
15073
|
if (await isFileRunContextDir(child)) candidates.push(child);
|
|
14127
15074
|
}
|
|
14128
15075
|
if (candidates.length === 1) return candidates[0];
|
|
@@ -14227,13 +15174,13 @@ function aborted(signal) {
|
|
|
14227
15174
|
return signal !== void 0 && signal.aborted;
|
|
14228
15175
|
}
|
|
14229
15176
|
function sleep2(msDelay, signal) {
|
|
14230
|
-
return new Promise((
|
|
14231
|
-
const timer = setTimeout(
|
|
15177
|
+
return new Promise((resolve8) => {
|
|
15178
|
+
const timer = setTimeout(resolve8, msDelay);
|
|
14232
15179
|
signal?.addEventListener(
|
|
14233
15180
|
"abort",
|
|
14234
15181
|
() => {
|
|
14235
15182
|
clearTimeout(timer);
|
|
14236
|
-
|
|
15183
|
+
resolve8();
|
|
14237
15184
|
},
|
|
14238
15185
|
{ once: true }
|
|
14239
15186
|
);
|
|
@@ -14702,7 +15649,7 @@ async function collectSessionSelection(groups, options = {}) {
|
|
|
14702
15649
|
// src/session-index.ts
|
|
14703
15650
|
import { mkdtemp as mkdtemp7, readdir as readdir18, readFile as readFile13, stat as stat16, writeFile as writeFile8 } from "fs/promises";
|
|
14704
15651
|
import { tmpdir as tmpdir8 } from "os";
|
|
14705
|
-
import { dirname as dirname8, join as
|
|
15652
|
+
import { dirname as dirname8, join as join31 } from "path";
|
|
14706
15653
|
function indexRow(record2) {
|
|
14707
15654
|
return {
|
|
14708
15655
|
session: record2.session,
|
|
@@ -14775,7 +15722,7 @@ async function findContextRoot(cwd) {
|
|
|
14775
15722
|
if (!s) return null;
|
|
14776
15723
|
if (!s.isDirectory()) current = dirname8(current);
|
|
14777
15724
|
for (; ; ) {
|
|
14778
|
-
if (await pathStat2(
|
|
15725
|
+
if (await pathStat2(join31(current, ".git")) || await dirExists(join31(current, ".evolve")) || await fileExists(join31(current, "AGENTS.md")) || await fileExists(join31(current, "CLAUDE.md"))) {
|
|
14779
15726
|
return current;
|
|
14780
15727
|
}
|
|
14781
15728
|
const parent = dirname8(current);
|
|
@@ -14843,7 +15790,7 @@ async function walkMarkdown(root, kind, maxFiles = 100) {
|
|
|
14843
15790
|
}
|
|
14844
15791
|
for (const entry of entries) {
|
|
14845
15792
|
if (out.length >= maxFiles) return;
|
|
14846
|
-
const path =
|
|
15793
|
+
const path = join31(dir, entry.name);
|
|
14847
15794
|
if (entry.isDirectory()) {
|
|
14848
15795
|
await walk(path);
|
|
14849
15796
|
} else if (entry.isFile() && entry.name.endsWith(".md")) {
|
|
@@ -14858,20 +15805,20 @@ async function walkMarkdown(root, kind, maxFiles = 100) {
|
|
|
14858
15805
|
async function collectContextRoot(root) {
|
|
14859
15806
|
const files = [];
|
|
14860
15807
|
for (const name of ["AGENTS.md", "CLAUDE.md", "GEMINI.md"]) {
|
|
14861
|
-
const summary = await summarizeFile(
|
|
15808
|
+
const summary = await summarizeFile(join31(root, name), "instruction-doc");
|
|
14862
15809
|
if (summary) files.push(summary);
|
|
14863
15810
|
}
|
|
14864
|
-
const evolve =
|
|
15811
|
+
const evolve = join31(root, ".evolve");
|
|
14865
15812
|
for (const name of ["skill-runs.jsonl", "governor.jsonl", "experiments.jsonl"]) {
|
|
14866
|
-
const summary = await summarizeFile(
|
|
15813
|
+
const summary = await summarizeFile(join31(evolve, name), "evolve-jsonl");
|
|
14867
15814
|
if (summary) files.push(summary);
|
|
14868
15815
|
}
|
|
14869
15816
|
for (const name of ["scorecard.json"]) {
|
|
14870
|
-
const summary = await summarizeFile(
|
|
15817
|
+
const summary = await summarizeFile(join31(evolve, name), "evolve-json");
|
|
14871
15818
|
if (summary) files.push(summary);
|
|
14872
15819
|
}
|
|
14873
|
-
files.push(...await walkMarkdown(
|
|
14874
|
-
files.push(...await walkMarkdown(
|
|
15820
|
+
files.push(...await walkMarkdown(join31(evolve, "reflections"), "reflection"));
|
|
15821
|
+
files.push(...await walkMarkdown(join31(evolve, "handoffs"), "handoff"));
|
|
14875
15822
|
return { root, files: files.sort((a, b) => a.path.localeCompare(b.path)) };
|
|
14876
15823
|
}
|
|
14877
15824
|
async function collectContextIndex(records) {
|
|
@@ -14917,7 +15864,7 @@ function serializeSessionIndex(index) {
|
|
|
14917
15864
|
`;
|
|
14918
15865
|
}
|
|
14919
15866
|
async function writeSessionIndexFile(index, outPath) {
|
|
14920
|
-
const path = outPath ??
|
|
15867
|
+
const path = outPath ?? join31(await mkdtemp7(join31(tmpdir8(), "traces-index-")), "session-index.json");
|
|
14921
15868
|
await writeFile8(path, serializeSessionIndex(index), "utf8");
|
|
14922
15869
|
return path;
|
|
14923
15870
|
}
|
|
@@ -14925,11 +15872,11 @@ async function writeSessionIndexFile(index, outPath) {
|
|
|
14925
15872
|
// src/upload-state.ts
|
|
14926
15873
|
import { mkdir as mkdir5, readFile as readFile14, writeFile as writeFile9 } from "fs/promises";
|
|
14927
15874
|
import { homedir as homedir12 } from "os";
|
|
14928
|
-
import { dirname as dirname9, join as
|
|
15875
|
+
import { dirname as dirname9, join as join32 } from "path";
|
|
14929
15876
|
import { contentHash } from "@tangle-network/agent-eval";
|
|
14930
15877
|
function stateFile() {
|
|
14931
|
-
const base = process.env.XDG_STATE_HOME ??
|
|
14932
|
-
return
|
|
15878
|
+
const base = process.env.XDG_STATE_HOME ?? join32(homedir12(), ".local", "state");
|
|
15879
|
+
return join32(base, "tangle-traces", "uploaded.json");
|
|
14933
15880
|
}
|
|
14934
15881
|
function uploadKey(harness, sessionId) {
|
|
14935
15882
|
return `${harness}:${sessionId}`;
|
|
@@ -14961,7 +15908,7 @@ function alreadyUploaded(state, harness, sessionId, hash2) {
|
|
|
14961
15908
|
import { readFile as readFile15 } from "fs/promises";
|
|
14962
15909
|
import { createRequire as createRequire2 } from "module";
|
|
14963
15910
|
import { hostname as hostname2 } from "os";
|
|
14964
|
-
import { basename as basename15, join as
|
|
15911
|
+
import { basename as basename15, join as join33 } from "path";
|
|
14965
15912
|
import { hostedClientFromEnv } from "@tangle-network/agent-eval/hosted";
|
|
14966
15913
|
import { REDACTION_VERSION } from "@tangle-network/agent-eval/traces";
|
|
14967
15914
|
var require2 = createRequire2(import.meta.url);
|
|
@@ -14985,7 +15932,7 @@ async function planUpload(opts = {}) {
|
|
|
14985
15932
|
async function gitBranch(cwd) {
|
|
14986
15933
|
if (!cwd) return void 0;
|
|
14987
15934
|
try {
|
|
14988
|
-
const head = await readFile15(
|
|
15935
|
+
const head = await readFile15(join33(cwd, ".git", "HEAD"), "utf8");
|
|
14989
15936
|
const m = head.match(/^ref:\s*refs\/heads\/(.+)$/m);
|
|
14990
15937
|
return m?.[1]?.trim();
|
|
14991
15938
|
} catch {
|
|
@@ -15249,9 +16196,13 @@ export {
|
|
|
15249
16196
|
resolveCaseResources,
|
|
15250
16197
|
enumerateReplayableCases,
|
|
15251
16198
|
zaiChatCaller,
|
|
16199
|
+
clipText,
|
|
15252
16200
|
buildFixPrompt,
|
|
16201
|
+
buildRetryFixPrompt,
|
|
16202
|
+
countScriptCommands,
|
|
15253
16203
|
extractFixCommand,
|
|
15254
16204
|
generateFixCommand,
|
|
16205
|
+
runFixLoop,
|
|
15255
16206
|
derivedImageTag,
|
|
15256
16207
|
dockerImagePreparer,
|
|
15257
16208
|
seededSample,
|
|
@@ -15260,6 +16211,21 @@ export {
|
|
|
15260
16211
|
parseReplayBatchArgs,
|
|
15261
16212
|
replayBatchUsage,
|
|
15262
16213
|
cmdReplayVerifyBatch,
|
|
16214
|
+
parseIncorrectStepsSubject,
|
|
16215
|
+
resolveFindingInvocation,
|
|
16216
|
+
replayVerifyFinding,
|
|
16217
|
+
findingReplayStep,
|
|
16218
|
+
findingTrajectoryId,
|
|
16219
|
+
resolveFindingReplayability,
|
|
16220
|
+
classifyVerdict,
|
|
16221
|
+
DEFAULT_SANDBOX_BASE_URL,
|
|
16222
|
+
assertSandboxReachable,
|
|
16223
|
+
verifyFindings,
|
|
16224
|
+
renderVerifiedFindingsSection,
|
|
16225
|
+
readFindingsFile,
|
|
16226
|
+
verifyFindingsUsage,
|
|
16227
|
+
parseVerifyFindingsArgs,
|
|
16228
|
+
cmdVerifyFindings,
|
|
15263
16229
|
runCommand,
|
|
15264
16230
|
commandAnalyzer,
|
|
15265
16231
|
haloAnalyzer,
|