@tangle-network/traces 0.15.0 → 0.15.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -2
- package/dist/{chunk-7DOBU7BJ.js → chunk-U4EB4HK7.js} +529 -2497
- package/dist/cli.js +49 -9
- package/dist/index.d.ts +76 -879
- package/dist/index.js +6 -91
- package/docs/replay-verify.md +6 -5
- package/package.json +13 -13
|
@@ -1017,9 +1017,9 @@ function serializeSpans(spans) {
|
|
|
1017
1017
|
}
|
|
1018
1018
|
async function writeOtlpFile(spans, outPath) {
|
|
1019
1019
|
const { mkdtemp: mkdtemp8, writeFile: writeFile13 } = await import("fs/promises");
|
|
1020
|
-
const { tmpdir:
|
|
1021
|
-
const { join:
|
|
1022
|
-
const path = outPath ??
|
|
1020
|
+
const { tmpdir: tmpdir8 } = await import("os");
|
|
1021
|
+
const { join: join34 } = await import("path");
|
|
1022
|
+
const path = outPath ?? join34(await mkdtemp8(join34(tmpdir8(), "traces-")), "spans.otlp.jsonl");
|
|
1023
1023
|
await writeFile13(path, serializeSpans(spans), "utf8");
|
|
1024
1024
|
return path;
|
|
1025
1025
|
}
|
|
@@ -1130,6 +1130,10 @@ var KIND = {
|
|
|
1130
1130
|
LLM: "llm",
|
|
1131
1131
|
TOOL: "tool",
|
|
1132
1132
|
CHAIN: "custom",
|
|
1133
|
+
EMBEDDING: "custom",
|
|
1134
|
+
RERANKER: "custom",
|
|
1135
|
+
GUARDRAIL: "custom",
|
|
1136
|
+
PROMPT: "custom",
|
|
1133
1137
|
EVALUATOR: "custom",
|
|
1134
1138
|
RETRIEVER: "custom",
|
|
1135
1139
|
UNKNOWN: "custom"
|
|
@@ -1818,7 +1822,7 @@ import {
|
|
|
1818
1822
|
LLM_INPUT_TOKENS as LLM_INPUT_TOKENS2,
|
|
1819
1823
|
LLM_OUTPUT_TOKENS as LLM_OUTPUT_TOKENS2
|
|
1820
1824
|
} from "@tangle-network/agent-eval/trace-attributes";
|
|
1821
|
-
import { deriveHexId as deriveHexId2 } from "@tangle-network/agent-trace-contract";
|
|
1825
|
+
import { ATTR as ATTR3, deriveHexId as deriveHexId2 } from "@tangle-network/agent-trace-contract";
|
|
1822
1826
|
|
|
1823
1827
|
// src/adapters/claude-subagent-sources.ts
|
|
1824
1828
|
import { readdir as readdir4, realpath } from "fs/promises";
|
|
@@ -1984,7 +1988,10 @@ function selectWorkflowBinding(file, bindings, childStartedAt, explicitToolUseId
|
|
|
1984
1988
|
`Cannot link Claude Workflow subagent ${file}: parent call is ambiguous`
|
|
1985
1989
|
);
|
|
1986
1990
|
}
|
|
1987
|
-
return
|
|
1991
|
+
return {
|
|
1992
|
+
binding: latest[0],
|
|
1993
|
+
confidence: explicitToolUseId || bindings.length === 1 ? "correlated" : "heuristic"
|
|
1994
|
+
};
|
|
1988
1995
|
}
|
|
1989
1996
|
|
|
1990
1997
|
// src/adapters/claude-subagent-sources.ts
|
|
@@ -2092,6 +2099,21 @@ async function collectClaudeSubagentSources(ref, bindings, signal) {
|
|
|
2092
2099
|
|
|
2093
2100
|
// src/adapters/claude.ts
|
|
2094
2101
|
var SERVICE2 = "claude-code";
|
|
2102
|
+
var AGENT_TOOL_RENAMED_IN = [2, 1, 63];
|
|
2103
|
+
function modelFacingToolNames(tools, version) {
|
|
2104
|
+
if (typeof version !== "string" || !atLeast(version, AGENT_TOOL_RENAMED_IN)) return [...tools];
|
|
2105
|
+
return tools.map((name) => name === "Task" ? "Agent" : name);
|
|
2106
|
+
}
|
|
2107
|
+
function atLeast(version, min) {
|
|
2108
|
+
const parts = version.split(".").map((p) => Number.parseInt(p, 10));
|
|
2109
|
+
if (parts.length < min.length || parts.some((p) => !Number.isFinite(p))) return false;
|
|
2110
|
+
for (let i = 0; i < min.length; i++) {
|
|
2111
|
+
if (parts[i] !== min[i]) return parts[i] > min[i];
|
|
2112
|
+
}
|
|
2113
|
+
return true;
|
|
2114
|
+
}
|
|
2115
|
+
var TOOL_DEFINITIONS_ATTR = "gen_ai.tool.definitions";
|
|
2116
|
+
var RUN_STATUS_ATTR = "run.status";
|
|
2095
2117
|
var EPOCH = (/* @__PURE__ */ new Date(0)).toISOString();
|
|
2096
2118
|
var HUMAN_ORIGIN = "human";
|
|
2097
2119
|
var ORIGIN_EVIDENCE_ATTR = "traces.claude.actor_evidence";
|
|
@@ -2788,9 +2810,21 @@ var ClaudeAdapter = class {
|
|
|
2788
2810
|
let selectedTurnId;
|
|
2789
2811
|
let selectedTurnFound = false;
|
|
2790
2812
|
let active = taskScope === "all";
|
|
2813
|
+
let offeredTools;
|
|
2814
|
+
let runResult;
|
|
2791
2815
|
for await (const event of readJsonl(ref.path, sessionJsonlOptions(ref, options))) {
|
|
2792
2816
|
options.signal?.throwIfAborted();
|
|
2793
|
-
|
|
2817
|
+
const eventSessionId = event.sessionId ?? event.session_id;
|
|
2818
|
+
if (!discoveredTraceId && eventSessionId) discoveredTraceId = eventSessionId;
|
|
2819
|
+
if (event.type === "system" && event.subtype === "init" && Array.isArray(event.tools)) {
|
|
2820
|
+
offeredTools = modelFacingToolNames(
|
|
2821
|
+
event.tools.filter((t) => typeof t === "string"),
|
|
2822
|
+
event.claude_code_version
|
|
2823
|
+
);
|
|
2824
|
+
}
|
|
2825
|
+
if (event.type === "result" && typeof event.is_error === "boolean") {
|
|
2826
|
+
runResult = { failed: event.is_error, subtype: event.subtype ?? "unknown" };
|
|
2827
|
+
}
|
|
2794
2828
|
const accepted = distinctClaudeEvent(event, seen, ref.path, `step${state.step}`);
|
|
2795
2829
|
if (!accepted) continue;
|
|
2796
2830
|
indexWorkflowProjection(accepted.projection, workflowIndex);
|
|
@@ -2845,6 +2879,13 @@ var ClaudeAdapter = class {
|
|
|
2845
2879
|
service: SERVICE2,
|
|
2846
2880
|
agent: SERVICE2
|
|
2847
2881
|
});
|
|
2882
|
+
if (offeredTools) {
|
|
2883
|
+
root.attributes[TOOL_DEFINITIONS_ATTR] = JSON.stringify(offeredTools.map((name) => ({ type: "function", name })));
|
|
2884
|
+
}
|
|
2885
|
+
if (runResult) {
|
|
2886
|
+
root.attributes[RUN_STATUS_ATTR] = runResult.failed ? "failed" : "completed";
|
|
2887
|
+
root.status = runResult.failed ? { code: "ERROR", message: `the run ended with ${runResult.subtype}` } : { code: "OK" };
|
|
2888
|
+
}
|
|
2848
2889
|
const spans = [root];
|
|
2849
2890
|
appendAll(spans, main.spans);
|
|
2850
2891
|
await this.foldSubagents(
|
|
@@ -2905,13 +2946,14 @@ var ClaudeAdapter = class {
|
|
|
2905
2946
|
workflowBindings
|
|
2906
2947
|
);
|
|
2907
2948
|
const parsed = await parseClaudeSubagent(ref, traceId, agent, options);
|
|
2908
|
-
const binding = selectWorkflowBinding(
|
|
2949
|
+
const { binding, confidence } = selectWorkflowBinding(
|
|
2909
2950
|
basename5(file),
|
|
2910
2951
|
runBindings,
|
|
2911
2952
|
firstSubagentTimestamp(agent, parsed),
|
|
2912
2953
|
meta.toolUseId
|
|
2913
2954
|
);
|
|
2914
2955
|
agent.parentToolUseId = binding.toolUseId;
|
|
2956
|
+
agent.parentConfidence = confidence;
|
|
2915
2957
|
}
|
|
2916
2958
|
agents.push(agent);
|
|
2917
2959
|
}
|
|
@@ -2968,10 +3010,14 @@ var ClaudeAdapter = class {
|
|
|
2968
3010
|
const parentAgentMissing = Boolean(
|
|
2969
3011
|
agent.meta.parentAgentId && !parsedByAgentId.has(agent.meta.parentAgentId)
|
|
2970
3012
|
);
|
|
3013
|
+
const confidence = parentSpan ? agent.parentConfidence ?? "correlated" : "unknown";
|
|
2971
3014
|
const task = subagentTaskName(agent);
|
|
2972
3015
|
const lifecycle = subagentLifecycleSpan(agent, parsed, traceId, parent, task, subDir);
|
|
2973
3016
|
for (const item of [...parsed.spans, ...lifecycle ? [lifecycle] : []]) {
|
|
2974
|
-
if (item.parent_span_id === `root:${traceId}`
|
|
3017
|
+
if (item.parent_span_id === `root:${traceId}` || item === lifecycle) {
|
|
3018
|
+
item.parent_span_id = parent;
|
|
3019
|
+
item.attributes[ATTR3.parentConfidence] = confidence;
|
|
3020
|
+
}
|
|
2975
3021
|
item.attributes[SUBAGENT_SPAN_ATTR] = true;
|
|
2976
3022
|
if (task) item.attributes[SUBAGENT_TASK_ATTR] = task;
|
|
2977
3023
|
if (agent.parentToolUseId && !parentSpan) {
|
|
@@ -3787,9 +3833,9 @@ function itemCountsJson(counts) {
|
|
|
3787
3833
|
}
|
|
3788
3834
|
function placeInnerSpans(pending, windows) {
|
|
3789
3835
|
return pending.map((entry) => {
|
|
3790
|
-
const { parent, join:
|
|
3836
|
+
const { parent, join: join34 } = joinItem(windows, entry.startMs, entry.endMs);
|
|
3791
3837
|
if (parent) entry.span.parent_span_id = parent.span_id;
|
|
3792
|
-
entry.span.attributes["traces.codex.item_join"] =
|
|
3838
|
+
entry.span.attributes["traces.codex.item_join"] = join34;
|
|
3793
3839
|
return entry.span;
|
|
3794
3840
|
});
|
|
3795
3841
|
}
|
|
@@ -6434,9 +6480,9 @@ function normalizeRemote(url) {
|
|
|
6434
6480
|
}
|
|
6435
6481
|
async function readGit(cwd) {
|
|
6436
6482
|
await stat12(`${cwd}/.git`);
|
|
6437
|
-
const { execFile
|
|
6438
|
-
const { promisify
|
|
6439
|
-
const run =
|
|
6483
|
+
const { execFile } = await import("child_process");
|
|
6484
|
+
const { promisify } = await import("util");
|
|
6485
|
+
const run = promisify(execFile);
|
|
6440
6486
|
const git = async (args) => {
|
|
6441
6487
|
try {
|
|
6442
6488
|
const { stdout } = await run("git", ["-C", cwd, ...args], { timeout: 5e3 });
|
|
@@ -6462,9 +6508,9 @@ async function readGit(cwd) {
|
|
|
6462
6508
|
};
|
|
6463
6509
|
}
|
|
6464
6510
|
async function gitOutput(cwd, args) {
|
|
6465
|
-
const { execFile
|
|
6466
|
-
const { promisify
|
|
6467
|
-
const run =
|
|
6511
|
+
const { execFile } = await import("child_process");
|
|
6512
|
+
const { promisify } = await import("util");
|
|
6513
|
+
const run = promisify(execFile);
|
|
6468
6514
|
try {
|
|
6469
6515
|
const { stdout } = await run("git", ["-C", cwd, ...args], { timeout: 5e3 });
|
|
6470
6516
|
const v = stdout.trim();
|
|
@@ -7530,11 +7576,11 @@ function decodeEvidenceRef(value, label, knownSpanUris) {
|
|
|
7530
7576
|
}
|
|
7531
7577
|
const uri = nonEmptyString3(ref.uri, `${label}.uri`);
|
|
7532
7578
|
if (kind === "span") validateSpanEvidence(uri, knownSpanUris, label);
|
|
7533
|
-
const
|
|
7579
|
+
const excerpt = optionalString(ref.excerpt, `${label}.excerpt`);
|
|
7534
7580
|
return {
|
|
7535
7581
|
kind,
|
|
7536
7582
|
uri,
|
|
7537
|
-
...
|
|
7583
|
+
...excerpt === void 0 ? {} : { excerpt }
|
|
7538
7584
|
};
|
|
7539
7585
|
}
|
|
7540
7586
|
function decodeFinding(value, label, knownSpanUris) {
|
|
@@ -7741,14 +7787,14 @@ function normalizeFinding(row, analystId, index, context) {
|
|
|
7741
7787
|
to: uri
|
|
7742
7788
|
});
|
|
7743
7789
|
}
|
|
7744
|
-
const
|
|
7745
|
-
if (
|
|
7790
|
+
const excerpt = citation.excerpt === void 0 ? void 0 : normalizeExcerpt(citation.excerpt, span2);
|
|
7791
|
+
if (excerpt !== citation.excerpt) {
|
|
7746
7792
|
context.log?.("finding citation normalized: json-escaped excerpt", {
|
|
7747
7793
|
analyst_id: analystId,
|
|
7748
7794
|
uri
|
|
7749
7795
|
});
|
|
7750
7796
|
}
|
|
7751
|
-
return
|
|
7797
|
+
return excerpt === void 0 ? { uri } : { uri, excerpt };
|
|
7752
7798
|
});
|
|
7753
7799
|
const normalized = { ...row, evidence: evidence3 };
|
|
7754
7800
|
const allowed = KIND_EXPECTED_SUBJECTS[analystId];
|
|
@@ -7797,10 +7843,10 @@ function decodeComponent(value) {
|
|
|
7797
7843
|
return value;
|
|
7798
7844
|
}
|
|
7799
7845
|
}
|
|
7800
|
-
function normalizeExcerpt(
|
|
7801
|
-
if (containsText(span2.content,
|
|
7802
|
-
const decoded = decodeJsonEscapes(
|
|
7803
|
-
return decoded !==
|
|
7846
|
+
function normalizeExcerpt(excerpt, span2) {
|
|
7847
|
+
if (containsText(span2.content, excerpt)) return excerpt;
|
|
7848
|
+
const decoded = decodeJsonEscapes(excerpt);
|
|
7849
|
+
return decoded !== excerpt && containsText(span2.content, decoded) ? decoded : excerpt;
|
|
7804
7850
|
}
|
|
7805
7851
|
function decodeJsonEscapes(text2) {
|
|
7806
7852
|
return text2.replace(
|
|
@@ -9118,13 +9164,13 @@ import { OtlpFileTraceStore } from "@tangle-network/agent-eval/traces";
|
|
|
9118
9164
|
import { isUtf8 as isUtf82 } from "buffer";
|
|
9119
9165
|
import { createHash as createHash9 } from "crypto";
|
|
9120
9166
|
import { constants } from "fs";
|
|
9121
|
-
import { lstat, open, realpath as realpath3 } from "fs/promises";
|
|
9167
|
+
import { lstat as lstat2, open, realpath as realpath3 } from "fs/promises";
|
|
9122
9168
|
import { dirname as dirname8, isAbsolute as isAbsolute4, join as join20, relative as relative5, resolve as resolve7, sep as sep2 } from "path";
|
|
9123
9169
|
|
|
9124
9170
|
// src/bundle.ts
|
|
9125
9171
|
import { createHash as createHash8 } from "crypto";
|
|
9126
9172
|
import { readFileSync as readFileSync2 } from "fs";
|
|
9127
|
-
import { copyFile, cp, mkdir as mkdir2, readdir as readdir16, readFile as readFile7, stat as stat14, writeFile as writeFile5 } from "fs/promises";
|
|
9173
|
+
import { copyFile, cp, lstat, mkdir as mkdir2, readdir as readdir16, readFile as readFile7, stat as stat14, writeFile as writeFile5 } from "fs/promises";
|
|
9128
9174
|
import { dirname as dirname7, join as join19, relative as relative4, resolve as resolve6, sep } from "path";
|
|
9129
9175
|
|
|
9130
9176
|
// src/improvement.ts
|
|
@@ -9444,7 +9490,7 @@ function validationExitCode(validation) {
|
|
|
9444
9490
|
|
|
9445
9491
|
// src/loop-analysis.ts
|
|
9446
9492
|
import {
|
|
9447
|
-
ATTR as
|
|
9493
|
+
ATTR as ATTR4,
|
|
9448
9494
|
firstNumberAttr,
|
|
9449
9495
|
firstStringAttr,
|
|
9450
9496
|
ITERATION_ATTR_KEYS,
|
|
@@ -9458,11 +9504,11 @@ function attributes(span2) {
|
|
|
9458
9504
|
return span2.attributes;
|
|
9459
9505
|
}
|
|
9460
9506
|
function outcomeOf(span2) {
|
|
9461
|
-
const raw = firstStringAttr(attributes(span2), [
|
|
9507
|
+
const raw = firstStringAttr(attributes(span2), [ATTR4.outcome]);
|
|
9462
9508
|
return raw !== void 0 && OUTCOMES.includes(raw) ? raw : null;
|
|
9463
9509
|
}
|
|
9464
9510
|
function scoreOf(span2) {
|
|
9465
|
-
return firstNumberAttr(attributes(span2), [
|
|
9511
|
+
return firstNumberAttr(attributes(span2), [ATTR4.score]) ?? null;
|
|
9466
9512
|
}
|
|
9467
9513
|
function iterationOf(span2) {
|
|
9468
9514
|
const value = firstNumberAttr(attributes(span2), ITERATION_ATTR_KEYS);
|
|
@@ -9535,7 +9581,7 @@ function trendOf(iterations) {
|
|
|
9535
9581
|
return {
|
|
9536
9582
|
trend: "unscored",
|
|
9537
9583
|
basis: "none",
|
|
9538
|
-
detail: `${iterations.length} round(s) recorded, ${scored.length + graded.length} graded: the trace carries neither ${
|
|
9584
|
+
detail: `${iterations.length} round(s) recorded, ${scored.length + graded.length} graded: the trace carries neither ${ATTR4.outcome} nor ${ATTR4.score} on enough rounds to compare them`
|
|
9539
9585
|
};
|
|
9540
9586
|
}
|
|
9541
9587
|
function analyzeLoopConvergence(spans) {
|
|
@@ -9546,7 +9592,7 @@ function analyzeLoopConvergence(spans) {
|
|
|
9546
9592
|
const iteration = iterationOf(span2);
|
|
9547
9593
|
if (iteration === null) continue;
|
|
9548
9594
|
iterationSpans += 1;
|
|
9549
|
-
const loopId = firstStringAttr(attributes(span2), [
|
|
9595
|
+
const loopId = firstStringAttr(attributes(span2), [ATTR4.loopId]) ?? `(unnamed loop in ${span2.trace_id})`;
|
|
9550
9596
|
const key = `${span2.trace_id}\0${loopId}`;
|
|
9551
9597
|
const verdict = verdictFor(span2, children);
|
|
9552
9598
|
const bucket = byLoop.get(key) ?? { traceId: span2.trace_id, iterations: [] };
|
|
@@ -10641,8 +10687,8 @@ function renderReport(result, meta) {
|
|
|
10641
10687
|
if (f.recommended_action) lines.push(`- **Fix:** ${f.recommended_action}`);
|
|
10642
10688
|
if (f.validation_plan) lines.push(`- **Check:** ${f.validation_plan}`);
|
|
10643
10689
|
for (const ev of f.evidence_refs.slice(0, 3)) {
|
|
10644
|
-
const
|
|
10645
|
-
lines.push(`- **Evidence:** ${ev.kind} ${ev.uri}${
|
|
10690
|
+
const excerpt = ev.excerpt ? ` \u2014 \`${ev.excerpt.slice(0, 200)}\`` : "";
|
|
10691
|
+
lines.push(`- **Evidence:** ${ev.kind} ${ev.uri}${excerpt}`);
|
|
10646
10692
|
}
|
|
10647
10693
|
lines.push("");
|
|
10648
10694
|
}
|
|
@@ -10966,8 +11012,8 @@ function jsonl(rows) {
|
|
|
10966
11012
|
return rows.length === 0 ? "" : `${rows.map((row) => JSON.stringify(row)).join("\n")}
|
|
10967
11013
|
`;
|
|
10968
11014
|
}
|
|
10969
|
-
function evidence(kind, uri,
|
|
10970
|
-
return
|
|
11015
|
+
function evidence(kind, uri, excerpt) {
|
|
11016
|
+
return excerpt ? { kind, uri, excerpt } : { kind, uri };
|
|
10971
11017
|
}
|
|
10972
11018
|
function totalReactionSignals(reactions) {
|
|
10973
11019
|
return Object.values(reactions.signals).reduce((sum, count2) => sum + count2, 0);
|
|
@@ -11679,8 +11725,8 @@ async function collectContextRoot(root) {
|
|
|
11679
11725
|
if (summary) files.push(summary);
|
|
11680
11726
|
}
|
|
11681
11727
|
}
|
|
11682
|
-
const
|
|
11683
|
-
if (
|
|
11728
|
+
const progress2 = await summarizeFile(join18(evolve, "progress.md"), "other");
|
|
11729
|
+
if (progress2) files.push(progress2);
|
|
11684
11730
|
return { root, files: files.sort((a, b) => a.path.localeCompare(b.path)) };
|
|
11685
11731
|
}
|
|
11686
11732
|
async function listEvolveRootFiles(evolve) {
|
|
@@ -12209,6 +12255,9 @@ var KNOWN_LIMITS = [
|
|
|
12209
12255
|
"progress.md is copied whole; no session-dated section extraction is performed",
|
|
12210
12256
|
"open pull-request state is not captured: the bundler makes no network calls"
|
|
12211
12257
|
];
|
|
12258
|
+
function sha256Hex(bytes) {
|
|
12259
|
+
return createHash8("sha256").update(bytes).digest("hex");
|
|
12260
|
+
}
|
|
12212
12261
|
function bundlePath(...parts) {
|
|
12213
12262
|
return parts.join("/");
|
|
12214
12263
|
}
|
|
@@ -12244,9 +12293,9 @@ function sliceJsonlByWindow(text2, windowStartMs, windowEndMs) {
|
|
|
12244
12293
|
return { kept, totalRows, unparseableTsRows };
|
|
12245
12294
|
}
|
|
12246
12295
|
async function runGitLog(cwd, sinceIso, untilIso) {
|
|
12247
|
-
const { execFile
|
|
12248
|
-
const { promisify
|
|
12249
|
-
const run =
|
|
12296
|
+
const { execFile } = await import("child_process");
|
|
12297
|
+
const { promisify } = await import("util");
|
|
12298
|
+
const run = promisify(execFile);
|
|
12250
12299
|
try {
|
|
12251
12300
|
const { stdout } = await run(
|
|
12252
12301
|
"git",
|
|
@@ -12549,6 +12598,90 @@ async function readSessionBundleManifest(bundleDir) {
|
|
|
12549
12598
|
}
|
|
12550
12599
|
return manifest;
|
|
12551
12600
|
}
|
|
12601
|
+
function unsafeBundlePath(path) {
|
|
12602
|
+
if (path.length === 0) return "empty path";
|
|
12603
|
+
if (path.includes("\\")) return "contains a backslash";
|
|
12604
|
+
if (path.startsWith("/") || /^[A-Za-z]:/.test(path)) return "is absolute";
|
|
12605
|
+
if (path.split("/").some((part) => part === ".." || part === "." || part === "")) {
|
|
12606
|
+
return 'has an empty, "." or ".." segment';
|
|
12607
|
+
}
|
|
12608
|
+
return null;
|
|
12609
|
+
}
|
|
12610
|
+
async function nonRegularEntries(root) {
|
|
12611
|
+
const out = [];
|
|
12612
|
+
const pending = [resolve6(root)];
|
|
12613
|
+
while (pending.length > 0) {
|
|
12614
|
+
const dir = pending.pop();
|
|
12615
|
+
for (const entry of await readdir16(dir, { withFileTypes: true })) {
|
|
12616
|
+
const path = join19(dir, entry.name);
|
|
12617
|
+
if (entry.isDirectory()) pending.push(path);
|
|
12618
|
+
else if (!entry.isFile()) out.push(relative4(resolve6(root), path).split(sep).join("/"));
|
|
12619
|
+
}
|
|
12620
|
+
}
|
|
12621
|
+
return out.sort();
|
|
12622
|
+
}
|
|
12623
|
+
async function verifySessionBundle(bundleDir) {
|
|
12624
|
+
const directory = resolve6(bundleDir);
|
|
12625
|
+
const manifest = await readSessionBundleManifest(directory);
|
|
12626
|
+
const issues = [];
|
|
12627
|
+
const listed = /* @__PURE__ */ new Set();
|
|
12628
|
+
let matched = 0;
|
|
12629
|
+
for (const file of manifest.files) {
|
|
12630
|
+
const unsafe = unsafeBundlePath(file.path);
|
|
12631
|
+
if (unsafe) {
|
|
12632
|
+
issues.push({ kind: "unsafe-path", path: file.path, detail: `manifest path ${unsafe}` });
|
|
12633
|
+
continue;
|
|
12634
|
+
}
|
|
12635
|
+
listed.add(file.path);
|
|
12636
|
+
const info = await lstat(join19(directory, file.path)).catch((error) => {
|
|
12637
|
+
if (isMissingPathError(error)) return null;
|
|
12638
|
+
throw error;
|
|
12639
|
+
});
|
|
12640
|
+
if (!info) {
|
|
12641
|
+
issues.push({ kind: "missing", path: file.path, detail: "listed in the manifest, not on disk" });
|
|
12642
|
+
continue;
|
|
12643
|
+
}
|
|
12644
|
+
if (!info.isFile()) continue;
|
|
12645
|
+
const sha2562 = await fileSha256(join19(directory, file.path));
|
|
12646
|
+
if (info.size !== file.bytes || sha2562 !== file.sha256) {
|
|
12647
|
+
issues.push({
|
|
12648
|
+
kind: "mismatch",
|
|
12649
|
+
path: file.path,
|
|
12650
|
+
detail: `recorded ${file.bytes} bytes sha256 ${file.sha256}, found ${info.size} bytes sha256 ${sha2562}`
|
|
12651
|
+
});
|
|
12652
|
+
continue;
|
|
12653
|
+
}
|
|
12654
|
+
matched += 1;
|
|
12655
|
+
}
|
|
12656
|
+
for (const path of await listSessionBundleFiles(directory)) {
|
|
12657
|
+
if (path === "manifest.json" || listed.has(path)) continue;
|
|
12658
|
+
issues.push({ kind: "unexpected", path, detail: "on disk, not listed in the manifest" });
|
|
12659
|
+
}
|
|
12660
|
+
for (const path of await nonRegularEntries(directory)) {
|
|
12661
|
+
issues.push({ kind: "not-a-file", path, detail: "a symbolic link or special file; a bundle holds regular files only" });
|
|
12662
|
+
}
|
|
12663
|
+
let source;
|
|
12664
|
+
if (manifest.projection) {
|
|
12665
|
+
const sourceManifest = join19(manifest.projection.sourceDirectory, "manifest.json");
|
|
12666
|
+
const bytes = await readFile7(sourceManifest).catch((error) => {
|
|
12667
|
+
if (isMissingPathError(error)) return null;
|
|
12668
|
+
throw error;
|
|
12669
|
+
});
|
|
12670
|
+
source = {
|
|
12671
|
+
directory: manifest.projection.sourceDirectory,
|
|
12672
|
+
status: bytes === null ? "unavailable" : sha256Hex(bytes) === manifest.projection.sourceManifestSha256 ? "match" : "mismatch"
|
|
12673
|
+
};
|
|
12674
|
+
}
|
|
12675
|
+
return {
|
|
12676
|
+
ok: issues.length === 0 && source?.status !== "mismatch",
|
|
12677
|
+
directory,
|
|
12678
|
+
view: manifest.view,
|
|
12679
|
+
listed: manifest.files.length,
|
|
12680
|
+
matched,
|
|
12681
|
+
issues,
|
|
12682
|
+
...source ? { source } : {}
|
|
12683
|
+
};
|
|
12684
|
+
}
|
|
12552
12685
|
var cachedVersion;
|
|
12553
12686
|
function tracesPackageVersion() {
|
|
12554
12687
|
if (cachedVersion) return cachedVersion;
|
|
@@ -12578,7 +12711,7 @@ async function openBundleFile(root, file) {
|
|
|
12578
12711
|
let current = root;
|
|
12579
12712
|
for (const part of relative5(root, path).split(sep2)) {
|
|
12580
12713
|
current = join20(current, part);
|
|
12581
|
-
if ((await
|
|
12714
|
+
if ((await lstat2(current)).isSymbolicLink()) throw new Error("bundle source symlinks are not permitted");
|
|
12582
12715
|
}
|
|
12583
12716
|
if (await realpath3(path) !== path) throw new Error("bundle source resolved outside its declared path");
|
|
12584
12717
|
const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
@@ -12614,9 +12747,9 @@ async function createBundleSourceReader(directory, spans, options = {}) {
|
|
|
12614
12747
|
const maxRecordBytes = options.maxRecordBytes ?? 16 * 1024 * 1024;
|
|
12615
12748
|
if (!Number.isSafeInteger(maxRecordBytes) || maxRecordBytes < 1) throw new Error("maxRecordBytes must be a positive safe integer");
|
|
12616
12749
|
options.signal?.throwIfAborted();
|
|
12617
|
-
if ((await
|
|
12750
|
+
if ((await lstat2(resolve7(directory))).isSymbolicLink()) throw new Error("bundle directory must not be a symlink");
|
|
12618
12751
|
const root = await realpath3(resolve7(directory));
|
|
12619
|
-
if ((await
|
|
12752
|
+
if ((await lstat2(join20(root, "manifest.json"))).isSymbolicLink()) throw new Error("bundle manifest must not be a symlink");
|
|
12620
12753
|
const manifest = await readSessionBundleManifest(root);
|
|
12621
12754
|
if (manifest.view !== "full") throw new Error("original source reads require an explicitly authorized full bundle");
|
|
12622
12755
|
if (!Array.isArray(manifest.files)) throw new Error("bundle manifest has no file receipts");
|
|
@@ -13042,7 +13175,7 @@ import { basename as basename14, dirname as dirname10, join as join23, resolve a
|
|
|
13042
13175
|
import { createHash as createHash11, randomUUID } from "crypto";
|
|
13043
13176
|
import {
|
|
13044
13177
|
link,
|
|
13045
|
-
lstat as
|
|
13178
|
+
lstat as lstat3,
|
|
13046
13179
|
mkdir as mkdir3,
|
|
13047
13180
|
readFile as readFile8,
|
|
13048
13181
|
realpath as realpath4,
|
|
@@ -13211,7 +13344,7 @@ async function optionalContainedFile(root, candidate) {
|
|
|
13211
13344
|
}
|
|
13212
13345
|
async function assertPathMissing(path) {
|
|
13213
13346
|
try {
|
|
13214
|
-
await
|
|
13347
|
+
await lstat3(path);
|
|
13215
13348
|
} catch (error) {
|
|
13216
13349
|
if (isErrno(error, "ENOENT")) return;
|
|
13217
13350
|
throw error;
|
|
@@ -13725,54 +13858,18 @@ function validateRevision(value) {
|
|
|
13725
13858
|
return value;
|
|
13726
13859
|
}
|
|
13727
13860
|
|
|
13728
|
-
// src/replay
|
|
13729
|
-
import {
|
|
13730
|
-
import { join as join24 } from "path";
|
|
13861
|
+
// src/replay.ts
|
|
13862
|
+
import { readFileSync as readFileSync3 } from "fs";
|
|
13731
13863
|
import {
|
|
13732
|
-
|
|
13733
|
-
|
|
13734
|
-
|
|
13735
|
-
|
|
13864
|
+
enumerateReplayableCases,
|
|
13865
|
+
parseCorpusFlag,
|
|
13866
|
+
readFindingsFile,
|
|
13867
|
+
replayVerify,
|
|
13868
|
+
runReplayBatch,
|
|
13869
|
+
verifyFindings
|
|
13870
|
+
} from "@tangle-network/agent-eval/trajectory-replay";
|
|
13736
13871
|
import { Sandbox } from "@tangle-network/sandbox";
|
|
13737
|
-
|
|
13738
|
-
if (!observation) return null;
|
|
13739
|
-
const m = /<returncode>(-?\d+)<\/returncode>/.exec(observation);
|
|
13740
|
-
return m ? Number(m[1]) : null;
|
|
13741
|
-
}
|
|
13742
|
-
function parseObservationOutput(observation) {
|
|
13743
|
-
if (!observation) return "";
|
|
13744
|
-
const m = /<output>\n?([\s\S]*?)\n?<\/output>/.exec(observation);
|
|
13745
|
-
return m ? m[1] : observation;
|
|
13746
|
-
}
|
|
13747
|
-
function deriveFailureSignature(observation) {
|
|
13748
|
-
const line = parseObservationOutput(observation).split("\n").find((l) => /\berror\b/i.test(l));
|
|
13749
|
-
return line ? line.trim().slice(0, 200) : null;
|
|
13750
|
-
}
|
|
13751
|
-
async function ingestCodeTraceBenchSteps(store, steps, caseId) {
|
|
13752
|
-
let tick = 0;
|
|
13753
|
-
const emitter = new TraceEmitter(store, { now: () => ++tick });
|
|
13754
|
-
await emitter.startRun({
|
|
13755
|
-
scenarioId: caseId,
|
|
13756
|
-
tags: { source: "codetracebench-replay", caseId }
|
|
13757
|
-
});
|
|
13758
|
-
for (const step of steps) {
|
|
13759
|
-
if (typeof step.action !== "string" || step.action.length === 0) {
|
|
13760
|
-
throw new Error(`replay-verify: step ${step.step_id} has no action`);
|
|
13761
|
-
}
|
|
13762
|
-
const handle = await emitter.tool({
|
|
13763
|
-
name: `step:${step.step_id}`,
|
|
13764
|
-
toolName: "shell",
|
|
13765
|
-
args: { command: step.action },
|
|
13766
|
-
result: {
|
|
13767
|
-
returncode: parseRecordedReturncode(step.observation),
|
|
13768
|
-
observation: step.observation
|
|
13769
|
-
}
|
|
13770
|
-
});
|
|
13771
|
-
await handle.end();
|
|
13772
|
-
}
|
|
13773
|
-
await emitter.endRun({ pass: true, notes: "recorded trajectory (ingested for replay)" });
|
|
13774
|
-
return { runId: emitter.runId, stepCount: steps.length };
|
|
13775
|
-
}
|
|
13872
|
+
var DEFAULT_SANDBOX_BASE_URL = "http://127.0.0.1:4097";
|
|
13776
13873
|
function sandboxReplayBackend(options) {
|
|
13777
13874
|
const client = new Sandbox({ apiKey: options.apiKey, baseUrl: options.baseUrl });
|
|
13778
13875
|
return {
|
|
@@ -13797,365 +13894,117 @@ function sandboxReplayBackend(options) {
|
|
|
13797
13894
|
}
|
|
13798
13895
|
};
|
|
13799
13896
|
}
|
|
13800
|
-
function
|
|
13801
|
-
|
|
13802
|
-
|
|
13803
|
-
|
|
13804
|
-
|
|
13805
|
-
|
|
13806
|
-
|
|
13807
|
-
|
|
13808
|
-
throw new Error(`replay-verify: span ${span2.name} carries no command`);
|
|
13809
|
-
}
|
|
13810
|
-
return args.command;
|
|
13811
|
-
}
|
|
13812
|
-
function recordedReturncodeOf(span2) {
|
|
13813
|
-
const result = span2.result;
|
|
13814
|
-
return typeof result?.returncode === "number" ? result.returncode : null;
|
|
13815
|
-
}
|
|
13816
|
-
var SandboxCounterfactualRunner = class {
|
|
13817
|
-
constructor(backend, options) {
|
|
13818
|
-
this.backend = backend;
|
|
13819
|
-
this.options = options;
|
|
13820
|
-
}
|
|
13821
|
-
backend;
|
|
13822
|
-
options;
|
|
13823
|
-
lastPrefix = null;
|
|
13824
|
-
lastArm = null;
|
|
13825
|
-
async executeFrom(ctx, emitter) {
|
|
13826
|
-
const { cwd, stepTimeoutMs, prefixLimit, onProgress } = this.options;
|
|
13827
|
-
const session = await this.backend.open();
|
|
13828
|
-
try {
|
|
13829
|
-
const toolSteps = ctx.prefix.filter(
|
|
13830
|
-
(s) => s.span.kind === "tool"
|
|
13831
|
-
);
|
|
13832
|
-
const toExecute = prefixLimit !== void 0 ? toolSteps.slice(0, prefixLimit) : toolSteps;
|
|
13833
|
-
const divergences = [];
|
|
13834
|
-
const prefixStart = Date.now();
|
|
13835
|
-
for (const step of toExecute) {
|
|
13836
|
-
const command = toolCommand(step.span);
|
|
13837
|
-
const expected = recordedReturncodeOf(step.span);
|
|
13838
|
-
const started = Date.now();
|
|
13839
|
-
const result = await session.exec(wrapActionForExec(command, cwd), stepTimeoutMs);
|
|
13840
|
-
const handle = await emitter.sandbox({
|
|
13841
|
-
name: step.span.name,
|
|
13842
|
-
command,
|
|
13843
|
-
exitCode: result.exitCode,
|
|
13844
|
-
wallMs: Date.now() - started
|
|
13845
|
-
});
|
|
13846
|
-
await handle.end();
|
|
13847
|
-
if (expected !== null && expected !== result.exitCode) {
|
|
13848
|
-
divergences.push({
|
|
13849
|
-
step: step.index + 1,
|
|
13850
|
-
expectedReturncode: expected,
|
|
13851
|
-
actualExit: result.exitCode
|
|
13852
|
-
});
|
|
13853
|
-
}
|
|
13854
|
-
onProgress?.(
|
|
13855
|
-
`prefix ${step.span.name}: exit=${result.exitCode}` + (expected !== null && expected !== result.exitCode ? ` (recorded ${expected})` : "")
|
|
13856
|
-
);
|
|
13857
|
-
}
|
|
13858
|
-
this.lastPrefix = {
|
|
13859
|
-
prefixExecuted: toExecute.length,
|
|
13860
|
-
prefixDivergences: divergences,
|
|
13861
|
-
wallMs: Date.now() - prefixStart
|
|
13862
|
-
};
|
|
13863
|
-
if (ctx.mutatedStep.span.kind !== "tool") {
|
|
13864
|
-
throw new Error("replay-verify: mutation target is not a tool span");
|
|
13865
|
-
}
|
|
13866
|
-
const armCommand = toolCommand(ctx.mutatedStep.span);
|
|
13867
|
-
const armStart = Date.now();
|
|
13868
|
-
const armResult = await session.exec(wrapActionForExec(armCommand, cwd), stepTimeoutMs);
|
|
13869
|
-
const armWallMs = Date.now() - armStart;
|
|
13870
|
-
const armHandle = await emitter.sandbox({
|
|
13871
|
-
name: `candidate:${ctx.mutatedStep.span.name}`,
|
|
13872
|
-
command: armCommand,
|
|
13873
|
-
exitCode: armResult.exitCode,
|
|
13874
|
-
wallMs: armWallMs
|
|
13875
|
-
});
|
|
13876
|
-
await armHandle.end();
|
|
13877
|
-
this.lastArm = {
|
|
13878
|
-
command: armCommand,
|
|
13879
|
-
exitCode: armResult.exitCode,
|
|
13880
|
-
stdout: armResult.stdout,
|
|
13881
|
-
stderr: armResult.stderr,
|
|
13882
|
-
wallMs: armWallMs
|
|
13883
|
-
};
|
|
13884
|
-
onProgress?.(`candidate step: exit=${armResult.exitCode} in ${armWallMs}ms`);
|
|
13885
|
-
await emitter.endRun({
|
|
13886
|
-
pass: armResult.exitCode === 0,
|
|
13887
|
-
notes: `prefix ${toExecute.length} steps, ${divergences.length} divergences`
|
|
13888
|
-
});
|
|
13889
|
-
} finally {
|
|
13890
|
-
await session.close();
|
|
13891
|
-
}
|
|
13892
|
-
}
|
|
13893
|
-
};
|
|
13894
|
-
function excerpt(text2, limit = 2400) {
|
|
13895
|
-
if (text2.length <= limit) return text2;
|
|
13896
|
-
const head = text2.slice(0, limit / 2);
|
|
13897
|
-
const tail = text2.slice(-limit / 2);
|
|
13898
|
-
return `${head}
|
|
13899
|
-
\u2026 [${text2.length - limit} chars elided] \u2026
|
|
13900
|
-
${tail}`;
|
|
13901
|
-
}
|
|
13902
|
-
async function replayVerify(options) {
|
|
13903
|
-
const totalStart = Date.now();
|
|
13904
|
-
const steps = JSON.parse(readFileSync3(options.stepsPath, "utf8"));
|
|
13905
|
-
if (!Array.isArray(steps) || steps.length === 0) {
|
|
13906
|
-
throw new Error(`replay-verify: ${options.stepsPath} is not a non-empty steps array`);
|
|
13907
|
-
}
|
|
13908
|
-
const index = options.at - 1;
|
|
13909
|
-
if (index < 0 || index >= steps.length) {
|
|
13910
|
-
throw new Error(`replay-verify: --at ${options.at} out of range [1, ${steps.length}]`);
|
|
13911
|
-
}
|
|
13912
|
-
const target = steps[index];
|
|
13913
|
-
if (target.step_id !== options.at) {
|
|
13897
|
+
function sandboxBackendFactory(options) {
|
|
13898
|
+
return (image) => sandboxReplayBackend({ ...options, image });
|
|
13899
|
+
}
|
|
13900
|
+
async function assertSandboxReachable(baseUrl) {
|
|
13901
|
+
let response;
|
|
13902
|
+
try {
|
|
13903
|
+
response = await fetch(new URL("/health", baseUrl), { signal: AbortSignal.timeout(5e3) });
|
|
13904
|
+
} catch (err) {
|
|
13914
13905
|
throw new Error(
|
|
13915
|
-
`
|
|
13906
|
+
`verify-findings: sandbox API unreachable at ${baseUrl} \u2014 ${err instanceof Error ? err.message : String(err)}. Executing findings requires a running sandbox orchestrator + SDK adapter (see docs/replay-verify.md); start them or pass --base-url.`
|
|
13916
13907
|
);
|
|
13917
13908
|
}
|
|
13918
|
-
|
|
13919
|
-
|
|
13920
|
-
|
|
13921
|
-
const signatureBasis = signature ? "returncode+output-substring" : "returncode-only";
|
|
13922
|
-
const backend = options.backend ?? sandboxReplayBackend({
|
|
13923
|
-
image: options.image,
|
|
13924
|
-
apiKey: requireOption(options.apiKey, "apiKey"),
|
|
13925
|
-
baseUrl: requireOption(options.baseUrl, "baseUrl"),
|
|
13926
|
-
maxLifetimeSeconds: options.maxLifetimeSeconds
|
|
13927
|
-
});
|
|
13928
|
-
const runnerOptions = {
|
|
13929
|
-
cwd: options.cwd,
|
|
13930
|
-
stepTimeoutMs: options.stepTimeoutMs ?? 3e5,
|
|
13931
|
-
prefixLimit: options.prefixLimit,
|
|
13932
|
-
onProgress: options.onProgress
|
|
13933
|
-
};
|
|
13934
|
-
const store = new InMemoryTraceStore2();
|
|
13935
|
-
const { runId } = await ingestCodeTraceBenchSteps(store, steps, caseId);
|
|
13936
|
-
options.onProgress?.(`arm A: replaying ${index} prefix steps then recorded step ${options.at}`);
|
|
13937
|
-
const armARunner = new SandboxCounterfactualRunner(backend, runnerOptions);
|
|
13938
|
-
const armAStart = Date.now();
|
|
13939
|
-
const armAResult = await runCounterfactual(
|
|
13940
|
-
store,
|
|
13941
|
-
runId,
|
|
13942
|
-
{ kind: "custom", at: index, describe: "arm-A identity replay", apply: (s) => s },
|
|
13943
|
-
armARunner
|
|
13944
|
-
);
|
|
13945
|
-
const armAMs = Date.now() - armAStart;
|
|
13946
|
-
const armAExec = requireExec(armARunner, "arm A");
|
|
13947
|
-
const armAOutput = `${armAExec.stdout}
|
|
13948
|
-
${armAExec.stderr}`;
|
|
13949
|
-
const failureSignatureMatch = recordedReturncode !== null && armAExec.exitCode === recordedReturncode && (signature ? armAOutput.includes(signature) : true);
|
|
13950
|
-
let armBRunner = null;
|
|
13951
|
-
let armBExec = null;
|
|
13952
|
-
let armBRunId = null;
|
|
13953
|
-
let armBMs = null;
|
|
13954
|
-
if (options.fixCommand) {
|
|
13955
|
-
const fixCommand = options.fixCommand;
|
|
13956
|
-
options.onProgress?.(`arm B: replaying prefix then corrected step`);
|
|
13957
|
-
armBRunner = new SandboxCounterfactualRunner(backend, runnerOptions);
|
|
13958
|
-
const armBStart = Date.now();
|
|
13959
|
-
const armBResult = await runCounterfactual(
|
|
13960
|
-
store,
|
|
13961
|
-
runId,
|
|
13962
|
-
{
|
|
13963
|
-
kind: "custom",
|
|
13964
|
-
at: index,
|
|
13965
|
-
describe: "arm-B corrected step",
|
|
13966
|
-
apply: (step) => ({
|
|
13967
|
-
...step,
|
|
13968
|
-
span: { ...step.span, args: { command: fixCommand } }
|
|
13969
|
-
})
|
|
13970
|
-
},
|
|
13971
|
-
armBRunner
|
|
13909
|
+
if (!response.ok) {
|
|
13910
|
+
throw new Error(
|
|
13911
|
+
`verify-findings: sandbox API at ${baseUrl} answered /health with HTTP ${response.status} \u2014 refusing to run proofs against degraded infrastructure`
|
|
13972
13912
|
);
|
|
13973
|
-
|
|
13974
|
-
armBRunId = armBResult.counterfactualRunId;
|
|
13975
|
-
armBExec = requireExec(armBRunner, "arm B");
|
|
13976
|
-
}
|
|
13977
|
-
const armBOutput = armBExec ? `${armBExec.stdout}
|
|
13978
|
-
${armBExec.stderr}` : null;
|
|
13979
|
-
const verdict = {
|
|
13980
|
-
case: caseId,
|
|
13981
|
-
image: options.image,
|
|
13982
|
-
driver: options.driverLabel ?? "docker",
|
|
13983
|
-
k: options.at,
|
|
13984
|
-
cwd: options.cwd,
|
|
13985
|
-
recordedReturncode,
|
|
13986
|
-
signature,
|
|
13987
|
-
signatureBasis,
|
|
13988
|
-
prefixExecuted: armARunner.lastPrefix?.prefixExecuted ?? 0,
|
|
13989
|
-
prefixDivergences: armARunner.lastPrefix?.prefixDivergences ?? [],
|
|
13990
|
-
armA: {
|
|
13991
|
-
command: armAExec.command,
|
|
13992
|
-
exitCode: armAExec.exitCode,
|
|
13993
|
-
wallMs: armAExec.wallMs,
|
|
13994
|
-
prefix: armARunner.lastPrefix,
|
|
13995
|
-
failureSignatureMatch
|
|
13996
|
-
},
|
|
13997
|
-
armB: armBExec && armBRunner ? {
|
|
13998
|
-
command: armBExec.command,
|
|
13999
|
-
exitCode: armBExec.exitCode,
|
|
14000
|
-
wallMs: armBExec.wallMs,
|
|
14001
|
-
prefix: armBRunner.lastPrefix,
|
|
14002
|
-
failureVanished: armBExec.exitCode === 0 && (signature && armBOutput ? !armBOutput.includes(signature) : true)
|
|
14003
|
-
} : null,
|
|
14004
|
-
timings: { armAMs, armBMs, totalMs: Date.now() - totalStart },
|
|
14005
|
-
runIds: {
|
|
14006
|
-
original: runId,
|
|
14007
|
-
armA: armAResult.counterfactualRunId,
|
|
14008
|
-
armB: armBRunId
|
|
14009
|
-
}
|
|
14010
|
-
};
|
|
14011
|
-
mkdirSync(options.out, { recursive: true });
|
|
14012
|
-
writeFileSync(join24(options.out, "replay-verdict.json"), `${JSON.stringify(verdict, null, 2)}
|
|
14013
|
-
`);
|
|
14014
|
-
writeFileSync(
|
|
14015
|
-
join24(options.out, "report.md"),
|
|
14016
|
-
renderReport2(verdict, target, armAExec, armBExec)
|
|
14017
|
-
);
|
|
14018
|
-
return verdict;
|
|
14019
|
-
}
|
|
14020
|
-
function requireOption(value, name) {
|
|
14021
|
-
if (value === void 0) throw new Error(`replay-verify: ${name} is required`);
|
|
14022
|
-
return value;
|
|
14023
|
-
}
|
|
14024
|
-
function requireExec(runner, arm) {
|
|
14025
|
-
if (!runner.lastArm) throw new Error(`replay-verify: ${arm} finished without executing step k`);
|
|
14026
|
-
return runner.lastArm;
|
|
13913
|
+
}
|
|
14027
13914
|
}
|
|
14028
|
-
function
|
|
14029
|
-
|
|
14030
|
-
|
|
14031
|
-
|
|
14032
|
-
|
|
14033
|
-
|
|
14034
|
-
|
|
14035
|
-
|
|
14036
|
-
|
|
14037
|
-
|
|
14038
|
-
|
|
14039
|
-
|
|
14040
|
-
|
|
14041
|
-
|
|
14042
|
-
|
|
14043
|
-
|
|
14044
|
-
|
|
14045
|
-
|
|
14046
|
-
|
|
14047
|
-
|
|
14048
|
-
|
|
14049
|
-
|
|
14050
|
-
|
|
13915
|
+
function zaiChatCaller(options) {
|
|
13916
|
+
async function callOnce(system, user, maxTokens) {
|
|
13917
|
+
try {
|
|
13918
|
+
const response = await fetch(`${options.baseUrl.replace(/\/$/, "")}/chat/completions`, {
|
|
13919
|
+
method: "POST",
|
|
13920
|
+
headers: { authorization: `Bearer ${options.apiKey}`, "content-type": "application/json" },
|
|
13921
|
+
body: JSON.stringify({
|
|
13922
|
+
model: options.model,
|
|
13923
|
+
messages: [
|
|
13924
|
+
{ role: "system", content: system },
|
|
13925
|
+
{ role: "user", content: user }
|
|
13926
|
+
],
|
|
13927
|
+
temperature: 0,
|
|
13928
|
+
max_tokens: maxTokens
|
|
13929
|
+
}),
|
|
13930
|
+
signal: AbortSignal.timeout(options.timeoutMs ?? 18e4)
|
|
13931
|
+
});
|
|
13932
|
+
const bodyText = await response.text();
|
|
13933
|
+
if (!response.ok) return { succeeded: false, error: `HTTP ${response.status}: ${bodyText.slice(0, 400)}` };
|
|
13934
|
+
const body = JSON.parse(bodyText);
|
|
13935
|
+
if (body.error) return { succeeded: false, error: String(body.error.message ?? "provider error") };
|
|
13936
|
+
const content = body.choices?.[0]?.message?.content;
|
|
13937
|
+
if (typeof content !== "string" || content.length === 0) {
|
|
13938
|
+
return { succeeded: false, error: `empty completion content: ${bodyText.slice(0, 400)}` };
|
|
13939
|
+
}
|
|
13940
|
+
const promptTokens = body.usage?.prompt_tokens;
|
|
13941
|
+
const completionTokens = body.usage?.completion_tokens;
|
|
13942
|
+
const usage = typeof promptTokens === "number" && typeof completionTokens === "number" ? { promptTokens, completionTokens } : null;
|
|
13943
|
+
return { succeeded: true, value: { content, usage } };
|
|
13944
|
+
} catch (err) {
|
|
13945
|
+
return { succeeded: false, error: err instanceof Error ? err.message : String(err) };
|
|
14051
13946
|
}
|
|
14052
13947
|
}
|
|
14053
|
-
|
|
14054
|
-
|
|
14055
|
-
|
|
14056
|
-
|
|
14057
|
-
|
|
14058
|
-
|
|
14059
|
-
|
|
14060
|
-
|
|
14061
|
-
`Exit ${armA.exitCode} in ${armA.wallMs}ms \u2014 failureSignatureMatch: **${verdict.armA.failureSignatureMatch}**`
|
|
14062
|
-
);
|
|
14063
|
-
lines.push("");
|
|
14064
|
-
lines.push("Recorded observation excerpt:");
|
|
14065
|
-
lines.push("");
|
|
14066
|
-
lines.push("```");
|
|
14067
|
-
lines.push(excerpt(parseObservationOutput(target.observation)));
|
|
14068
|
-
lines.push("```");
|
|
14069
|
-
lines.push("");
|
|
14070
|
-
lines.push("Replayed stdout+stderr excerpt:");
|
|
14071
|
-
lines.push("");
|
|
14072
|
-
lines.push("```");
|
|
14073
|
-
lines.push(excerpt(`${armA.stdout}
|
|
14074
|
-
${armA.stderr}`.trim()));
|
|
14075
|
-
lines.push("```");
|
|
14076
|
-
if (armB && verdict.armB) {
|
|
14077
|
-
lines.push("");
|
|
14078
|
-
lines.push("## Arm B \u2014 corrected step k");
|
|
14079
|
-
lines.push("");
|
|
14080
|
-
lines.push("```sh");
|
|
14081
|
-
lines.push(armB.command);
|
|
14082
|
-
lines.push("```");
|
|
14083
|
-
lines.push("");
|
|
14084
|
-
lines.push(
|
|
14085
|
-
`Exit ${armB.exitCode} in ${armB.wallMs}ms \u2014 failureVanished: **${verdict.armB.failureVanished}** (prefix re-replayed in a fresh sandbox: ${verdict.armB.prefix.prefixExecuted} steps, ${verdict.armB.prefix.prefixDivergences.length} divergences)`
|
|
14086
|
-
);
|
|
14087
|
-
lines.push("");
|
|
14088
|
-
lines.push("Replayed stdout+stderr excerpt:");
|
|
14089
|
-
lines.push("");
|
|
14090
|
-
lines.push("```");
|
|
14091
|
-
lines.push(excerpt(`${armB.stdout}
|
|
14092
|
-
${armB.stderr}`.trim()));
|
|
14093
|
-
lines.push("```");
|
|
14094
|
-
}
|
|
14095
|
-
lines.push("");
|
|
14096
|
-
lines.push("## Timings");
|
|
14097
|
-
lines.push("");
|
|
14098
|
-
lines.push(
|
|
14099
|
-
`arm A ${verdict.timings.armAMs}ms, arm B ${verdict.timings.armBMs ?? "\u2014"}ms, total ${verdict.timings.totalMs}ms.`
|
|
14100
|
-
);
|
|
14101
|
-
lines.push("");
|
|
14102
|
-
return lines.join("\n");
|
|
13948
|
+
return {
|
|
13949
|
+
async complete(system, user) {
|
|
13950
|
+
const baseCap = options.maxTokens ?? 16384;
|
|
13951
|
+
const first = await callOnce(system, user, baseCap);
|
|
13952
|
+
if (first.succeeded || !/finish_reason":"length/.test(first.error)) return first;
|
|
13953
|
+
return callOnce(system, user, baseCap * 2);
|
|
13954
|
+
}
|
|
13955
|
+
};
|
|
14103
13956
|
}
|
|
14104
|
-
function
|
|
13957
|
+
function parseFlags2(command, argv, switches = []) {
|
|
14105
13958
|
const values = /* @__PURE__ */ new Map();
|
|
13959
|
+
const corpora = [];
|
|
13960
|
+
const seen = /* @__PURE__ */ new Set();
|
|
14106
13961
|
for (let i = 0; i < argv.length; i++) {
|
|
14107
13962
|
const arg = argv[i];
|
|
14108
13963
|
if (arg === "--help" || arg === "-h") return "help";
|
|
14109
|
-
if (
|
|
13964
|
+
if (switches.includes(arg)) {
|
|
13965
|
+
seen.add(arg);
|
|
13966
|
+
continue;
|
|
13967
|
+
}
|
|
13968
|
+
if (!arg.startsWith("--")) throw new Error(`${command}: unexpected positional argument: ${arg}`);
|
|
14110
13969
|
const value = argv[++i];
|
|
14111
|
-
if (value === void 0) throw new Error(
|
|
14112
|
-
|
|
13970
|
+
if (value === void 0) throw new Error(`${command}: missing value for ${arg}`);
|
|
13971
|
+
if (arg === "--corpus") corpora.push(parseCorpusFlag(value));
|
|
13972
|
+
else values.set(arg, value);
|
|
14113
13973
|
}
|
|
14114
|
-
|
|
14115
|
-
|
|
14116
|
-
|
|
14117
|
-
|
|
14118
|
-
|
|
14119
|
-
|
|
14120
|
-
|
|
14121
|
-
|
|
14122
|
-
|
|
14123
|
-
|
|
14124
|
-
|
|
13974
|
+
return { values, corpora, switches: seen };
|
|
13975
|
+
}
|
|
13976
|
+
function flagReader(command, values) {
|
|
13977
|
+
return {
|
|
13978
|
+
required(flag) {
|
|
13979
|
+
const v = values.get(flag);
|
|
13980
|
+
if (v === void 0) throw new Error(`${command}: ${flag} is required`);
|
|
13981
|
+
return v;
|
|
13982
|
+
},
|
|
13983
|
+
number(flag) {
|
|
13984
|
+
const v = values.get(flag);
|
|
13985
|
+
if (v === void 0) return void 0;
|
|
13986
|
+
const n = Number(v);
|
|
13987
|
+
if (!Number.isFinite(n)) throw new Error(`${command}: ${flag} must be a number, got ${v}`);
|
|
13988
|
+
return n;
|
|
13989
|
+
}
|
|
14125
13990
|
};
|
|
14126
|
-
|
|
14127
|
-
|
|
14128
|
-
const
|
|
14129
|
-
const
|
|
14130
|
-
|
|
14131
|
-
|
|
14132
|
-
throw new Error(`replay-verify: --at must be a positive step_id, got ${values.get("--at")}`);
|
|
14133
|
-
}
|
|
14134
|
-
let fixCommand = values.get("--fix-command");
|
|
14135
|
-
const fixFile = values.get("--fix-command-file");
|
|
14136
|
-
if (fixCommand !== void 0 && fixFile !== void 0) {
|
|
14137
|
-
throw new Error("replay-verify: pass --fix-command or --fix-command-file, not both");
|
|
13991
|
+
}
|
|
13992
|
+
function sandboxOptions(command, values, requireKey) {
|
|
13993
|
+
const apiKeyEnv = values.get("--api-key-env") ?? "SANDBOX_API_KEY";
|
|
13994
|
+
const apiKey = process.env[apiKeyEnv];
|
|
13995
|
+
if (requireKey && !apiKey) {
|
|
13996
|
+
throw new Error(`${command}: env var ${apiKeyEnv} is empty \u2014 pass --api-key-env or export it`);
|
|
14138
13997
|
}
|
|
14139
|
-
|
|
13998
|
+
const maxLifetime = flagReader(command, values).number("--max-lifetime");
|
|
14140
13999
|
return {
|
|
14141
|
-
|
|
14142
|
-
|
|
14143
|
-
|
|
14144
|
-
fixCommand,
|
|
14145
|
-
cwd,
|
|
14146
|
-
out,
|
|
14147
|
-
caseId: values.get("--case"),
|
|
14148
|
-
signature: values.get("--signature"),
|
|
14149
|
-
stepTimeoutMs: optionalNumber("--step-timeout"),
|
|
14150
|
-
prefixLimit: optionalNumber("--prefix-limit"),
|
|
14151
|
-
driverLabel: values.get("--driver-label"),
|
|
14152
|
-
baseUrl: values.get("--base-url") ?? process.env.SANDBOX_API_URL ?? "http://127.0.0.1:4097",
|
|
14153
|
-
apiKeyEnv: values.get("--api-key-env") ?? "SANDBOX_API_KEY",
|
|
14154
|
-
maxLifetimeSeconds: optionalNumber("--max-lifetime")
|
|
14000
|
+
apiKey: apiKey ?? "",
|
|
14001
|
+
baseUrl: values.get("--base-url") ?? process.env.SANDBOX_API_URL ?? DEFAULT_SANDBOX_BASE_URL,
|
|
14002
|
+
...maxLifetime === void 0 ? {} : { maxLifetimeSeconds: maxLifetime }
|
|
14155
14003
|
};
|
|
14156
14004
|
}
|
|
14157
|
-
|
|
14158
|
-
|
|
14005
|
+
var progress = (message) => process.stderr.write(`${message}
|
|
14006
|
+
`);
|
|
14007
|
+
var REPLAY_VERIFY_USAGE = `traces replay-verify \u2014 replay a trajectory prefix in a sandbox and execute the counterfactual at step k
|
|
14159
14008
|
|
|
14160
14009
|
Usage:
|
|
14161
14010
|
traces replay-verify --steps STEPS.json --image IMG --at K --cwd DIR --out DIR \\
|
|
@@ -14173,1928 +14022,153 @@ Usage:
|
|
|
14173
14022
|
line of the recorded observation containing "error")
|
|
14174
14023
|
--prefix-limit execute at most N prefix steps (weakens fidelity; reported)
|
|
14175
14024
|
--step-timeout per-exec timeout in ms (default 300000)
|
|
14176
|
-
--base-url sandbox API url (default $SANDBOX_API_URL or
|
|
14025
|
+
--base-url sandbox API url (default $SANDBOX_API_URL or ${DEFAULT_SANDBOX_BASE_URL})
|
|
14177
14026
|
--api-key-env env var holding the sandbox API key (default SANDBOX_API_KEY)
|
|
14178
14027
|
|
|
14179
14028
|
Requires a running sandbox orchestrator; only SWE-style trajectories that carry a
|
|
14180
14029
|
docker image are replayable. See docs/replay-verify.md for setup and limits.
|
|
14181
14030
|
`;
|
|
14182
|
-
}
|
|
14183
14031
|
async function cmdReplayVerify(argv) {
|
|
14184
|
-
const parsed =
|
|
14032
|
+
const parsed = parseFlags2("replay-verify", argv);
|
|
14185
14033
|
if (parsed === "help") {
|
|
14186
|
-
process.stdout.write(
|
|
14034
|
+
process.stdout.write(REPLAY_VERIFY_USAGE);
|
|
14187
14035
|
return;
|
|
14188
14036
|
}
|
|
14189
|
-
const
|
|
14190
|
-
|
|
14191
|
-
|
|
14192
|
-
|
|
14193
|
-
);
|
|
14037
|
+
const { values } = parsed;
|
|
14038
|
+
const flags = flagReader("replay-verify", values);
|
|
14039
|
+
const at = Number(flags.required("--at"));
|
|
14040
|
+
if (!Number.isInteger(at) || at < 1) {
|
|
14041
|
+
throw new Error(`replay-verify: --at must be a positive step_id, got ${values.get("--at")}`);
|
|
14042
|
+
}
|
|
14043
|
+
const fixFile = values.get("--fix-command-file");
|
|
14044
|
+
if (values.has("--fix-command") && fixFile !== void 0) {
|
|
14045
|
+
throw new Error("replay-verify: pass --fix-command or --fix-command-file, not both");
|
|
14194
14046
|
}
|
|
14047
|
+
const fixCommand = fixFile === void 0 ? values.get("--fix-command") : readFileSync3(fixFile, "utf8").trimEnd();
|
|
14048
|
+
const image = flags.required("--image");
|
|
14049
|
+
const out = flags.required("--out");
|
|
14195
14050
|
const verdict = await replayVerify({
|
|
14196
|
-
stepsPath:
|
|
14197
|
-
image
|
|
14198
|
-
at
|
|
14199
|
-
fixCommand
|
|
14200
|
-
cwd:
|
|
14201
|
-
out
|
|
14202
|
-
caseId:
|
|
14203
|
-
signature:
|
|
14204
|
-
stepTimeoutMs:
|
|
14205
|
-
prefixLimit:
|
|
14206
|
-
driverLabel:
|
|
14207
|
-
|
|
14208
|
-
|
|
14209
|
-
maxLifetimeSeconds: parsed.maxLifetimeSeconds,
|
|
14210
|
-
onProgress: (message) => process.stderr.write(`${message}
|
|
14211
|
-
`)
|
|
14051
|
+
stepsPath: flags.required("--steps"),
|
|
14052
|
+
image,
|
|
14053
|
+
at,
|
|
14054
|
+
fixCommand,
|
|
14055
|
+
cwd: flags.required("--cwd"),
|
|
14056
|
+
out,
|
|
14057
|
+
caseId: values.get("--case"),
|
|
14058
|
+
signature: values.get("--signature"),
|
|
14059
|
+
stepTimeoutMs: flags.number("--step-timeout"),
|
|
14060
|
+
prefixLimit: flags.number("--prefix-limit"),
|
|
14061
|
+
driverLabel: values.get("--driver-label"),
|
|
14062
|
+
backend: sandboxReplayBackend({ ...sandboxOptions("replay-verify", values, true), image }),
|
|
14063
|
+
onProgress: progress
|
|
14212
14064
|
});
|
|
14213
14065
|
process.stdout.write(`${JSON.stringify(verdict, null, 2)}
|
|
14214
14066
|
`);
|
|
14215
14067
|
const verdictLine = verdict.armB ? `armA.failureSignatureMatch=${verdict.armA.failureSignatureMatch} armB.failureVanished=${verdict.armB.failureVanished}` : `armA.failureSignatureMatch=${verdict.armA.failureSignatureMatch}`;
|
|
14216
|
-
|
|
14217
|
-
`replay-verify: ${verdictLine} (${verdict.prefixDivergences.length}/${verdict.prefixExecuted} prefix divergences) \u2192 ${
|
|
14218
|
-
`
|
|
14068
|
+
progress(
|
|
14069
|
+
`replay-verify: ${verdictLine} (${verdict.prefixDivergences.length}/${verdict.prefixExecuted} prefix divergences) \u2192 ${out}`
|
|
14219
14070
|
);
|
|
14220
14071
|
}
|
|
14072
|
+
var REPLAY_BATCH_USAGE = `traces replay-verify-batch \u2014 measure replayability and fix-flip rates across gold-labeled corpora
|
|
14221
14073
|
|
|
14222
|
-
|
|
14223
|
-
|
|
14224
|
-
|
|
14225
|
-
|
|
14226
|
-
|
|
14227
|
-
|
|
14228
|
-
|
|
14229
|
-
|
|
14230
|
-
|
|
14231
|
-
|
|
14232
|
-
|
|
14074
|
+
Usage:
|
|
14075
|
+
traces replay-verify-batch \\
|
|
14076
|
+
--corpus NAME=<labels.json>::<prepared-dir> [--corpus ...] \\
|
|
14077
|
+
--out DIR [--fix none|generate|loop] [--fix-attempts 3] [--enumerate-only] \\
|
|
14078
|
+
[--fix-model glm-5.2] [--fix-base-url URL] [--fix-api-key-env ZAI_GLM_API_KEY] \\
|
|
14079
|
+
[--max-fix-cases 30] [--seed 17] [--step-timeout MS] [--prefix-limit N] \\
|
|
14080
|
+
[--case-filter SUBSTRING] [--case-limit N] \\
|
|
14081
|
+
[--base-url URL] [--api-key-env SANDBOX_API_KEY] [--max-lifetime SECONDS]
|
|
14082
|
+
|
|
14083
|
+
Replayable = the raw trajectory carries info.docker_config.base_image AND the
|
|
14084
|
+
labels mark at least one gold incorrect step that is a real mid-trajectory
|
|
14085
|
+
action. Excluded cases and image pull failures are report rows, never
|
|
14086
|
+
skipped silently. --enumerate-only prints the enumeration without touching a
|
|
14087
|
+
sandbox.
|
|
14088
|
+
|
|
14089
|
+
--fix generate runs ONE LLM call per arm-A-reproduced case (cap
|
|
14090
|
+
--max-fix-cases, seeded sample beyond it) and executes the corrected command
|
|
14091
|
+
as arm B in a fresh sandbox. --fix loop feeds each failed attempt's REAL
|
|
14092
|
+
executed output into the next prompt, up to --fix-attempts attempts.
|
|
14093
|
+
|
|
14094
|
+
Outputs batch-report.json, batch-report.md, cases.jsonl (incremental), and one
|
|
14095
|
+
directory per case with the full replay-verify artifacts.
|
|
14096
|
+
See docs/replay-verify.md for orchestrator setup and honest limits.
|
|
14097
|
+
`;
|
|
14098
|
+
async function cmdReplayVerifyBatch(argv) {
|
|
14099
|
+
const parsed = parseFlags2("replay-verify-batch", argv, ["--enumerate-only"]);
|
|
14100
|
+
if (parsed === "help") {
|
|
14101
|
+
process.stdout.write(REPLAY_BATCH_USAGE);
|
|
14102
|
+
return;
|
|
14233
14103
|
}
|
|
14234
|
-
|
|
14235
|
-
|
|
14236
|
-
|
|
14237
|
-
|
|
14238
|
-
return action.includes(SUBMIT_ACTION_SIGNATURE);
|
|
14239
|
-
}
|
|
14240
|
-
function readLabelEntries(labelsPath) {
|
|
14241
|
-
const parsed = JSON.parse(readFileSync4(labelsPath, "utf8"));
|
|
14242
|
-
if (!Array.isArray(parsed)) throw new Error(`${labelsPath} is not a JSON array of label entries`);
|
|
14243
|
-
for (const entry of parsed) {
|
|
14244
|
-
if (typeof entry.traj_id !== "string") {
|
|
14245
|
-
throw new Error(`${labelsPath} has an entry without a string traj_id`);
|
|
14246
|
-
}
|
|
14104
|
+
const { values, corpora } = parsed;
|
|
14105
|
+
const flags = flagReader("replay-verify-batch", values);
|
|
14106
|
+
if (corpora.length === 0) {
|
|
14107
|
+
throw new Error("replay-verify-batch: at least one --corpus name=<labels>::<prepared> is required");
|
|
14247
14108
|
}
|
|
14248
|
-
|
|
14249
|
-
|
|
14250
|
-
|
|
14251
|
-
|
|
14252
|
-
|
|
14253
|
-
|
|
14109
|
+
if (parsed.switches.has("--enumerate-only")) {
|
|
14110
|
+
const enumeration = enumerateReplayableCases(corpora);
|
|
14111
|
+
const replayable = enumeration.replayable.map((c) => ({
|
|
14112
|
+
corpus: c.corpus,
|
|
14113
|
+
trajId: c.trajId,
|
|
14114
|
+
image: c.image,
|
|
14115
|
+
cwd: c.cwd,
|
|
14116
|
+
cwdSource: c.cwdSource,
|
|
14117
|
+
k: c.k,
|
|
14118
|
+
submitGoldsSkipped: c.submitGoldsSkipped,
|
|
14119
|
+
recordedReturncodeAtK: c.recordedReturncodeAtK,
|
|
14120
|
+
goldIncorrectSteps: c.goldIncorrectSteps
|
|
14121
|
+
}));
|
|
14122
|
+
const report2 = { labelEntries: enumeration.labelEntryCount, replayable, excluded: enumeration.excluded };
|
|
14123
|
+
process.stdout.write(`${JSON.stringify(report2, null, 2)}
|
|
14124
|
+
`);
|
|
14125
|
+
return;
|
|
14254
14126
|
}
|
|
14255
|
-
|
|
14256
|
-
|
|
14257
|
-
|
|
14258
|
-
|
|
14259
|
-
const entries = readdirSync2(sweRawDir, { recursive: true, encoding: "utf8" });
|
|
14260
|
-
return entries.filter((relative7) => relative7.endsWith(".traj.json")).map((relative7) => join25(sweRawDir, relative7)).sort();
|
|
14261
|
-
}
|
|
14262
|
-
function cwdFromPwdObservation(steps) {
|
|
14263
|
-
for (const step of steps) {
|
|
14264
|
-
if (step.action.trim() !== "pwd") continue;
|
|
14265
|
-
const line = parseObservationOutput(step.observation).split("\n").map((l) => l.trim()).find((l) => l.startsWith("/"));
|
|
14266
|
-
if (line) return line;
|
|
14127
|
+
const out = flags.required("--out");
|
|
14128
|
+
const fix = values.get("--fix") ?? "none";
|
|
14129
|
+
if (fix !== "none" && fix !== "generate" && fix !== "loop") {
|
|
14130
|
+
throw new Error(`replay-verify-batch: --fix must be none, generate, or loop, got ${fix}`);
|
|
14267
14131
|
}
|
|
14268
|
-
|
|
14269
|
-
|
|
14270
|
-
|
|
14271
|
-
const taskPath = join25(preparedDir, "normalized", trajId, "task.md");
|
|
14272
|
-
if (existsSync2(taskPath)) {
|
|
14273
|
-
const text2 = readFileSync4(taskPath, "utf8").trim();
|
|
14274
|
-
if (text2.length > 0) return text2;
|
|
14132
|
+
const fixAttempts = flags.number("--fix-attempts") ?? 3;
|
|
14133
|
+
if (!Number.isInteger(fixAttempts) || fixAttempts < 1) {
|
|
14134
|
+
throw new Error(`replay-verify-batch: --fix-attempts must be a positive integer, got ${fixAttempts}`);
|
|
14275
14135
|
}
|
|
14276
|
-
const
|
|
14277
|
-
|
|
14278
|
-
|
|
14136
|
+
const fixModel = values.get("--fix-model") ?? "glm-5.2";
|
|
14137
|
+
let fixCaller;
|
|
14138
|
+
if (fix !== "none") {
|
|
14139
|
+
const fixApiKeyEnv = values.get("--fix-api-key-env") ?? "ZAI_GLM_API_KEY";
|
|
14140
|
+
const fixApiKey = process.env[fixApiKeyEnv];
|
|
14141
|
+
if (!fixApiKey) {
|
|
14142
|
+
throw new Error(`replay-verify-batch: env var ${fixApiKeyEnv} is empty \u2014 pass --fix-api-key-env or export it`);
|
|
14143
|
+
}
|
|
14144
|
+
fixCaller = zaiChatCaller({
|
|
14145
|
+
baseUrl: values.get("--fix-base-url") ?? "https://api.z.ai/api/coding/paas/v4",
|
|
14146
|
+
apiKey: fixApiKey,
|
|
14147
|
+
model: fixModel
|
|
14148
|
+
});
|
|
14279
14149
|
}
|
|
14280
|
-
|
|
14150
|
+
const report = await runReplayBatch({
|
|
14151
|
+
corpora,
|
|
14152
|
+
out,
|
|
14153
|
+
fix,
|
|
14154
|
+
fixAttempts,
|
|
14155
|
+
fixCaller,
|
|
14156
|
+
fixModelLabel: fixModel,
|
|
14157
|
+
maxFixCases: flags.number("--max-fix-cases") ?? 30,
|
|
14158
|
+
seed: flags.number("--seed") ?? 17,
|
|
14159
|
+
stepTimeoutMs: flags.number("--step-timeout"),
|
|
14160
|
+
prefixLimit: flags.number("--prefix-limit"),
|
|
14161
|
+
caseFilter: values.get("--case-filter"),
|
|
14162
|
+
caseLimit: flags.number("--case-limit"),
|
|
14163
|
+
backendFactory: sandboxBackendFactory(sandboxOptions("replay-verify-batch", values, true)),
|
|
14164
|
+
onProgress: progress
|
|
14165
|
+
});
|
|
14166
|
+
const { replayabilityRate, fixFlipRate } = report.headline;
|
|
14167
|
+
progress(
|
|
14168
|
+
`replay-verify-batch: replayability ${replayabilityRate.numerator}/${replayabilityRate.denominator}` + (fixFlipRate ? `, fix-flip ${fixFlipRate.numerator}/${fixFlipRate.denominator}` : "") + ` \u2192 ${out}`
|
|
14169
|
+
);
|
|
14281
14170
|
}
|
|
14282
|
-
|
|
14283
|
-
const sweRawDir = join25(corpus.preparedDir, "extracted", trajId, "swe_raw");
|
|
14284
|
-
const rawFiles = findRawTrajFiles(sweRawDir);
|
|
14285
|
-
if (rawFiles.length === 0) {
|
|
14286
|
-
return { resolved: false, reason: "no-swe-raw-trajectory", detail: sweRawDir };
|
|
14287
|
-
}
|
|
14288
|
-
if (rawFiles.length > 1) {
|
|
14289
|
-
return { resolved: false, reason: "ambiguous-swe-raw-trajectory", detail: rawFiles.join(", ") };
|
|
14290
|
-
}
|
|
14291
|
-
let raw;
|
|
14292
|
-
try {
|
|
14293
|
-
raw = JSON.parse(readFileSync4(rawFiles[0], "utf8"));
|
|
14294
|
-
} catch (err) {
|
|
14295
|
-
return {
|
|
14296
|
-
resolved: false,
|
|
14297
|
-
reason: "unreadable-raw-trajectory",
|
|
14298
|
-
detail: `${rawFiles[0]}: ${err instanceof Error ? err.message : String(err)}`
|
|
14299
|
-
};
|
|
14300
|
-
}
|
|
14301
|
-
const dockerConfig = raw.info?.docker_config;
|
|
14302
|
-
const image = dockerConfig?.base_image;
|
|
14303
|
-
if (typeof image !== "string" || image.length === 0) {
|
|
14304
|
-
return { resolved: false, reason: "no-docker-image", detail: rawFiles[0] };
|
|
14305
|
-
}
|
|
14306
|
-
const stepsPath = join25(corpus.preparedDir, "normalized", trajId, "steps.json");
|
|
14307
|
-
if (!existsSync2(stepsPath)) {
|
|
14308
|
-
return { resolved: false, reason: "missing-steps-json", detail: stepsPath };
|
|
14309
|
-
}
|
|
14310
|
-
const steps = JSON.parse(readFileSync4(stepsPath, "utf8"));
|
|
14311
|
-
const runConfigCwd = raw.info?.config?.environment?.cwd;
|
|
14312
|
-
const dockerCwd = dockerConfig?.cwd;
|
|
14313
|
-
let cwd;
|
|
14314
|
-
let cwdSource;
|
|
14315
|
-
if (typeof runConfigCwd === "string" && runConfigCwd.length > 0) {
|
|
14316
|
-
cwd = runConfigCwd;
|
|
14317
|
-
cwdSource = "run-config";
|
|
14318
|
-
} else if (typeof dockerCwd === "string" && dockerCwd.length > 0) {
|
|
14319
|
-
cwd = dockerCwd;
|
|
14320
|
-
cwdSource = "docker-config";
|
|
14321
|
-
} else {
|
|
14322
|
-
const observed = cwdFromPwdObservation(steps);
|
|
14323
|
-
if (!observed) return { resolved: false, reason: "cwd-underivable" };
|
|
14324
|
-
cwd = observed;
|
|
14325
|
-
cwdSource = "pwd-observation";
|
|
14326
|
-
}
|
|
14327
|
-
const recordedTimeout = raw.info?.config?.environment?.timeout;
|
|
14328
|
-
return {
|
|
14329
|
-
resolved: true,
|
|
14330
|
-
resources: {
|
|
14331
|
-
corpus: corpus.name,
|
|
14332
|
-
trajId,
|
|
14333
|
-
stepsPath,
|
|
14334
|
-
steps,
|
|
14335
|
-
taskStatement: taskStatementOf(corpus.preparedDir, trajId, raw),
|
|
14336
|
-
image,
|
|
14337
|
-
cwd,
|
|
14338
|
-
cwdSource,
|
|
14339
|
-
recordedStepTimeoutMs: typeof recordedTimeout === "number" && recordedTimeout > 0 ? recordedTimeout * 1e3 : null
|
|
14340
|
-
}
|
|
14341
|
-
};
|
|
14342
|
-
}
|
|
14343
|
-
function enumerateReplayableCases(corpora) {
|
|
14344
|
-
const replayable = [];
|
|
14345
|
-
const excluded = [];
|
|
14346
|
-
let labelEntryCount = 0;
|
|
14347
|
-
for (const corpus of corpora) {
|
|
14348
|
-
for (const entry of readLabelEntries(corpus.labelsPath)) {
|
|
14349
|
-
labelEntryCount += 1;
|
|
14350
|
-
const trajId = entry.traj_id;
|
|
14351
|
-
const resolution = resolveCaseResources(corpus, trajId);
|
|
14352
|
-
if (!resolution.resolved) {
|
|
14353
|
-
excluded.push({ corpus: corpus.name, trajId, reason: resolution.reason, detail: resolution.detail });
|
|
14354
|
-
continue;
|
|
14355
|
-
}
|
|
14356
|
-
const gold = goldIncorrectSteps(entry);
|
|
14357
|
-
if (gold.length === 0) {
|
|
14358
|
-
excluded.push({ corpus: corpus.name, trajId, reason: "no-gold-incorrect-step" });
|
|
14359
|
-
continue;
|
|
14360
|
-
}
|
|
14361
|
-
const resources = resolution.resources;
|
|
14362
|
-
let submitGoldsSkipped = 0;
|
|
14363
|
-
let target = null;
|
|
14364
|
-
let missingGoldId = null;
|
|
14365
|
-
for (const goldId of gold) {
|
|
14366
|
-
const step = resources.steps.find((s) => s.step_id === goldId);
|
|
14367
|
-
if (!step) {
|
|
14368
|
-
missingGoldId = goldId;
|
|
14369
|
-
break;
|
|
14370
|
-
}
|
|
14371
|
-
if (isSubmitAction(step.action)) {
|
|
14372
|
-
submitGoldsSkipped += 1;
|
|
14373
|
-
continue;
|
|
14374
|
-
}
|
|
14375
|
-
target = step;
|
|
14376
|
-
break;
|
|
14377
|
-
}
|
|
14378
|
-
if (missingGoldId !== null) {
|
|
14379
|
-
excluded.push({
|
|
14380
|
-
corpus: corpus.name,
|
|
14381
|
-
trajId,
|
|
14382
|
-
reason: "gold-step-outside-steps",
|
|
14383
|
-
detail: `k=${missingGoldId}, steps=${resources.steps.length}`
|
|
14384
|
-
});
|
|
14385
|
-
continue;
|
|
14386
|
-
}
|
|
14387
|
-
if (!target) {
|
|
14388
|
-
excluded.push({
|
|
14389
|
-
corpus: corpus.name,
|
|
14390
|
-
trajId,
|
|
14391
|
-
reason: "gold-only-submit-step",
|
|
14392
|
-
detail: `${submitGoldsSkipped} gold step(s), all submit commands`
|
|
14393
|
-
});
|
|
14394
|
-
continue;
|
|
14395
|
-
}
|
|
14396
|
-
replayable.push({
|
|
14397
|
-
...resources,
|
|
14398
|
-
goldIncorrectSteps: gold,
|
|
14399
|
-
k: target.step_id,
|
|
14400
|
-
submitGoldsSkipped,
|
|
14401
|
-
recordedReturncodeAtK: parseRecordedReturncode(target.observation)
|
|
14402
|
-
});
|
|
14403
|
-
}
|
|
14404
|
-
}
|
|
14405
|
-
return { replayable, excluded, labelEntryCount };
|
|
14406
|
-
}
|
|
14407
|
-
|
|
14408
|
-
// src/replay-fix.ts
|
|
14409
|
-
function zaiChatCaller(options) {
|
|
14410
|
-
const fetchImpl = options.fetchImpl ?? fetch;
|
|
14411
|
-
return {
|
|
14412
|
-
async complete(system, user) {
|
|
14413
|
-
const baseCap = options.maxTokens ?? 16384;
|
|
14414
|
-
let lastError = "empty completion content";
|
|
14415
|
-
for (const cap of [baseCap, baseCap * 2]) {
|
|
14416
|
-
const outcome = await callOnce(cap);
|
|
14417
|
-
if (outcome.succeeded) return outcome;
|
|
14418
|
-
lastError = outcome.error;
|
|
14419
|
-
if (!/finish_reason":"length/.test(outcome.error)) return outcome;
|
|
14420
|
-
}
|
|
14421
|
-
return { succeeded: false, error: lastError };
|
|
14422
|
-
async function callOnce(maxTokens) {
|
|
14423
|
-
const controller = new AbortController();
|
|
14424
|
-
const timer = setTimeout(() => controller.abort(), options.timeoutMs ?? 18e4);
|
|
14425
|
-
try {
|
|
14426
|
-
const response = await fetchImpl(`${options.baseUrl.replace(/\/$/, "")}/chat/completions`, {
|
|
14427
|
-
method: "POST",
|
|
14428
|
-
headers: {
|
|
14429
|
-
authorization: `Bearer ${options.apiKey}`,
|
|
14430
|
-
"content-type": "application/json"
|
|
14431
|
-
},
|
|
14432
|
-
body: JSON.stringify({
|
|
14433
|
-
model: options.model,
|
|
14434
|
-
messages: [
|
|
14435
|
-
{ role: "system", content: system },
|
|
14436
|
-
{ role: "user", content: user }
|
|
14437
|
-
],
|
|
14438
|
-
temperature: 0,
|
|
14439
|
-
max_tokens: maxTokens
|
|
14440
|
-
}),
|
|
14441
|
-
signal: controller.signal
|
|
14442
|
-
});
|
|
14443
|
-
const bodyText = await response.text();
|
|
14444
|
-
if (!response.ok) {
|
|
14445
|
-
return { succeeded: false, error: `HTTP ${response.status}: ${bodyText.slice(0, 400)}` };
|
|
14446
|
-
}
|
|
14447
|
-
const body = JSON.parse(bodyText);
|
|
14448
|
-
if (body.error) {
|
|
14449
|
-
return { succeeded: false, error: String(body.error.message ?? "provider error") };
|
|
14450
|
-
}
|
|
14451
|
-
const content = body.choices?.[0]?.message?.content;
|
|
14452
|
-
if (typeof content !== "string" || content.length === 0) {
|
|
14453
|
-
return { succeeded: false, error: `empty completion content: ${bodyText.slice(0, 400)}` };
|
|
14454
|
-
}
|
|
14455
|
-
const promptTokens = body.usage?.prompt_tokens;
|
|
14456
|
-
const completionTokens = body.usage?.completion_tokens;
|
|
14457
|
-
const usage = typeof promptTokens === "number" && typeof completionTokens === "number" ? { promptTokens, completionTokens } : null;
|
|
14458
|
-
return { succeeded: true, value: { content, usage } };
|
|
14459
|
-
} catch (err) {
|
|
14460
|
-
return { succeeded: false, error: err instanceof Error ? err.message : String(err) };
|
|
14461
|
-
} finally {
|
|
14462
|
-
clearTimeout(timer);
|
|
14463
|
-
}
|
|
14464
|
-
}
|
|
14465
|
-
}
|
|
14466
|
-
};
|
|
14467
|
-
}
|
|
14468
|
-
function clipText(text2, limit) {
|
|
14469
|
-
if (text2.length <= limit) return text2;
|
|
14470
|
-
const half = Math.floor(limit / 2);
|
|
14471
|
-
return `${text2.slice(0, half)}
|
|
14472
|
-
\u2026 [${text2.length - limit} chars elided] \u2026
|
|
14473
|
-
${text2.slice(-half)}`;
|
|
14474
|
-
}
|
|
14475
|
-
function renderStep(step, marker) {
|
|
14476
|
-
const rc = parseRecordedReturncode(step.observation);
|
|
14477
|
-
const output = clipText(parseObservationOutput(step.observation).trim(), 1600);
|
|
14478
|
-
return [
|
|
14479
|
-
`### step ${step.step_id}${marker}`,
|
|
14480
|
-
"```sh",
|
|
14481
|
-
clipText(step.action, 2e3),
|
|
14482
|
-
"```",
|
|
14483
|
-
`returncode: ${rc ?? "none recorded"}`,
|
|
14484
|
-
output.length > 0 ? `output:
|
|
14485
|
-
\`\`\`
|
|
14486
|
-
${output}
|
|
14487
|
-
\`\`\`` : "output: (empty)"
|
|
14488
|
-
].join("\n");
|
|
14489
|
-
}
|
|
14490
|
-
function promptBody(input) {
|
|
14491
|
-
const radius = input.contextRadius ?? 3;
|
|
14492
|
-
const target = input.steps.find((s) => s.step_id === input.k);
|
|
14493
|
-
if (!target) throw new Error(`replay-fix: no step with step_id ${input.k}`);
|
|
14494
|
-
const context = input.steps.filter(
|
|
14495
|
-
(s) => s.step_id !== input.k && Math.abs(s.step_id - input.k) <= radius
|
|
14496
|
-
);
|
|
14497
|
-
return [
|
|
14498
|
-
"## Task the agent was solving",
|
|
14499
|
-
input.taskStatement ? clipText(input.taskStatement, 4e3) : "(no task statement recorded)",
|
|
14500
|
-
"",
|
|
14501
|
-
"## Surrounding steps",
|
|
14502
|
-
...context.map((s) => renderStep(s, "")),
|
|
14503
|
-
"",
|
|
14504
|
-
"## Failing step to correct",
|
|
14505
|
-
renderStep(target, " (INCORRECT \u2014 correct this one)")
|
|
14506
|
-
];
|
|
14507
|
-
}
|
|
14508
|
-
function buildFixPrompt(input) {
|
|
14509
|
-
const system = [
|
|
14510
|
-
"You repair one failed shell command from a recorded coding-agent trajectory.",
|
|
14511
|
-
"The trajectory replays inside the original docker image; every command runs as a fresh /bin/sh subshell from a fixed working directory.",
|
|
14512
|
-
"You are given the failing step, its recorded output, surrounding steps, and the task statement.",
|
|
14513
|
-
"Reply with exactly ONE corrected shell command (compound commands with && or pipes are fine) inside a single ```sh fenced block.",
|
|
14514
|
-
"The corrected command must accomplish the failing step's intent and exit 0. No prose outside the fenced block."
|
|
14515
|
-
].join("\n");
|
|
14516
|
-
const user = [
|
|
14517
|
-
...promptBody(input),
|
|
14518
|
-
"",
|
|
14519
|
-
"Output the single corrected replacement for the failing step now."
|
|
14520
|
-
].join("\n");
|
|
14521
|
-
return { system, user };
|
|
14522
|
-
}
|
|
14523
|
-
function renderFailedAttempt(prior) {
|
|
14524
|
-
if (prior.command === null) {
|
|
14525
|
-
return [
|
|
14526
|
-
`### attempt ${prior.attempt}`,
|
|
14527
|
-
`model call failed before producing a command: ${prior.llmError ?? "unknown error"}`
|
|
14528
|
-
].join("\n");
|
|
14529
|
-
}
|
|
14530
|
-
const stdout = (prior.stdoutTail ?? "").trim();
|
|
14531
|
-
const stderr = (prior.stderrTail ?? "").trim();
|
|
14532
|
-
return [
|
|
14533
|
-
`### attempt ${prior.attempt}`,
|
|
14534
|
-
"```sh",
|
|
14535
|
-
clipText(prior.command, 2e3),
|
|
14536
|
-
"```",
|
|
14537
|
-
`exit code: ${prior.exitCode ?? "not executed"}`,
|
|
14538
|
-
stdout.length > 0 ? `stdout:
|
|
14539
|
-
\`\`\`
|
|
14540
|
-
${stdout}
|
|
14541
|
-
\`\`\`` : "stdout: (empty)",
|
|
14542
|
-
stderr.length > 0 ? `stderr:
|
|
14543
|
-
\`\`\`
|
|
14544
|
-
${stderr}
|
|
14545
|
-
\`\`\`` : "stderr: (empty)"
|
|
14546
|
-
].join("\n");
|
|
14547
|
-
}
|
|
14548
|
-
function buildRetryFixPrompt(input, priorAttempts, maxScriptCommands = 5) {
|
|
14549
|
-
if (priorAttempts.length === 0) {
|
|
14550
|
-
throw new Error("replay-fix: buildRetryFixPrompt requires at least one prior attempt");
|
|
14551
|
-
}
|
|
14552
|
-
const system = [
|
|
14553
|
-
"You repair one failed shell command from a recorded coding-agent trajectory.",
|
|
14554
|
-
"The trajectory replays inside the original docker image; every command runs as a fresh /bin/sh subshell from a fixed working directory.",
|
|
14555
|
-
"Earlier corrected commands were executed for real and failed; their actual output is included below.",
|
|
14556
|
-
`Reply with a corrected fix inside a single \`\`\`sh fenced block: either one command, or a short script of at most ${maxScriptCommands} commands (one per line).`,
|
|
14557
|
-
"The whole block executes as ONE /bin/sh unit from the fixed working directory and must exit 0.",
|
|
14558
|
-
"Do not repeat a command that already failed. Keep reasoning brief. No prose outside the fenced block."
|
|
14559
|
-
].join("\n");
|
|
14560
|
-
const user = [
|
|
14561
|
-
...promptBody(input),
|
|
14562
|
-
"",
|
|
14563
|
-
"## Previous fix attempts (executed for real \u2014 all failed)",
|
|
14564
|
-
...priorAttempts.map((prior) => renderFailedAttempt(prior)),
|
|
14565
|
-
"",
|
|
14566
|
-
`Output a corrected fix now \u2014 one \`\`\`sh block, at most ${maxScriptCommands} commands.`
|
|
14567
|
-
].join("\n");
|
|
14568
|
-
return { system, user };
|
|
14569
|
-
}
|
|
14570
|
-
function countScriptCommands(script) {
|
|
14571
|
-
return script.split("\n").filter((line) => line.trim().length > 0 && !line.trim().startsWith("#")).length;
|
|
14572
|
-
}
|
|
14573
|
-
function extractFixCommand(content) {
|
|
14574
|
-
const blocks = [...content.matchAll(/```(?:sh|bash|shell)?\n([\s\S]*?)```/g)];
|
|
14575
|
-
const candidate = blocks.length > 0 ? blocks[blocks.length - 1][1] : content;
|
|
14576
|
-
const command = candidate.trim();
|
|
14577
|
-
if (command.length === 0 || command.includes("```")) return null;
|
|
14578
|
-
return command;
|
|
14579
|
-
}
|
|
14580
|
-
async function generateFixCommand(caller, input) {
|
|
14581
|
-
const { system, user } = buildFixPrompt(input);
|
|
14582
|
-
const outcome = await caller.complete(system, user);
|
|
14583
|
-
if (!outcome.succeeded) return outcome;
|
|
14584
|
-
const command = extractFixCommand(outcome.value.content);
|
|
14585
|
-
if (command === null) {
|
|
14586
|
-
return {
|
|
14587
|
-
succeeded: false,
|
|
14588
|
-
error: `completion carried no usable command: ${clipText(outcome.value.content, 300)}`
|
|
14589
|
-
};
|
|
14590
|
-
}
|
|
14591
|
-
return { succeeded: true, value: { command, usage: outcome.value.usage } };
|
|
14592
|
-
}
|
|
14593
|
-
|
|
14594
|
-
// src/replay-fix-loop.ts
|
|
14595
|
-
function toFailedAttempt(record2) {
|
|
14596
|
-
return {
|
|
14597
|
-
attempt: record2.attempt,
|
|
14598
|
-
command: record2.command,
|
|
14599
|
-
exitCode: record2.exitCode,
|
|
14600
|
-
stdoutTail: record2.stdoutTail,
|
|
14601
|
-
stderrTail: record2.stderrTail,
|
|
14602
|
-
llmError: record2.llmError ?? record2.armBError
|
|
14603
|
-
};
|
|
14604
|
-
}
|
|
14605
|
-
async function runFixLoop(caller, input, executor, options) {
|
|
14606
|
-
if (!Number.isInteger(options.maxAttempts) || options.maxAttempts < 1) {
|
|
14607
|
-
throw new Error(
|
|
14608
|
-
`replay-fix-loop: maxAttempts must be a positive integer, got ${options.maxAttempts}`
|
|
14609
|
-
);
|
|
14610
|
-
}
|
|
14611
|
-
const scriptCap = options.maxScriptCommands ?? 5;
|
|
14612
|
-
const tailChars = options.outputTailChars ?? 1600;
|
|
14613
|
-
const attempts = [];
|
|
14614
|
-
let llmCalls = 0;
|
|
14615
|
-
let llmFailures = 0;
|
|
14616
|
-
let promptTokens = 0;
|
|
14617
|
-
let completionTokens = 0;
|
|
14618
|
-
const unexecuted = (attempt, llmError, usage, command, started) => ({
|
|
14619
|
-
attempt,
|
|
14620
|
-
command,
|
|
14621
|
-
llmError,
|
|
14622
|
-
usage,
|
|
14623
|
-
executed: false,
|
|
14624
|
-
exitCode: null,
|
|
14625
|
-
prefixExecuted: null,
|
|
14626
|
-
prefixDivergences: null,
|
|
14627
|
-
failureVanished: null,
|
|
14628
|
-
stdoutTail: null,
|
|
14629
|
-
stderrTail: null,
|
|
14630
|
-
armBError: null,
|
|
14631
|
-
wallMs: Date.now() - started
|
|
14632
|
-
});
|
|
14633
|
-
for (let attempt = 1; attempt <= options.maxAttempts; attempt++) {
|
|
14634
|
-
const started = Date.now();
|
|
14635
|
-
const prompt = attempt === 1 ? buildFixPrompt(input) : buildRetryFixPrompt(input, attempts.map(toFailedAttempt), scriptCap);
|
|
14636
|
-
llmCalls += 1;
|
|
14637
|
-
const outcome = await caller.complete(prompt.system, prompt.user);
|
|
14638
|
-
if (!outcome.succeeded) {
|
|
14639
|
-
llmFailures += 1;
|
|
14640
|
-
attempts.push(unexecuted(attempt, outcome.error, null, null, started));
|
|
14641
|
-
options.onProgress?.(
|
|
14642
|
-
`fix-loop attempt ${attempt}: LLM failed \u2014 ${outcome.error.slice(0, 160)}`
|
|
14643
|
-
);
|
|
14644
|
-
continue;
|
|
14645
|
-
}
|
|
14646
|
-
const usage = outcome.value.usage;
|
|
14647
|
-
promptTokens += usage?.promptTokens ?? 0;
|
|
14648
|
-
completionTokens += usage?.completionTokens ?? 0;
|
|
14649
|
-
const command = extractFixCommand(outcome.value.content);
|
|
14650
|
-
if (command === null) {
|
|
14651
|
-
llmFailures += 1;
|
|
14652
|
-
attempts.push(
|
|
14653
|
-
unexecuted(
|
|
14654
|
-
attempt,
|
|
14655
|
-
`completion carried no usable command: ${clipText(outcome.value.content, 300)}`,
|
|
14656
|
-
usage,
|
|
14657
|
-
null,
|
|
14658
|
-
started
|
|
14659
|
-
)
|
|
14660
|
-
);
|
|
14661
|
-
continue;
|
|
14662
|
-
}
|
|
14663
|
-
if (attempt > 1) {
|
|
14664
|
-
const commandLines = countScriptCommands(command);
|
|
14665
|
-
if (commandLines > scriptCap) {
|
|
14666
|
-
llmFailures += 1;
|
|
14667
|
-
attempts.push(
|
|
14668
|
-
unexecuted(
|
|
14669
|
-
attempt,
|
|
14670
|
-
`script exceeds ${scriptCap} command lines (${commandLines})`,
|
|
14671
|
-
usage,
|
|
14672
|
-
null,
|
|
14673
|
-
started
|
|
14674
|
-
)
|
|
14675
|
-
);
|
|
14676
|
-
options.onProgress?.(
|
|
14677
|
-
`fix-loop attempt ${attempt}: rejected script with ${commandLines} command lines`
|
|
14678
|
-
);
|
|
14679
|
-
continue;
|
|
14680
|
-
}
|
|
14681
|
-
}
|
|
14682
|
-
let execution;
|
|
14683
|
-
try {
|
|
14684
|
-
execution = await executor(command, attempt);
|
|
14685
|
-
} catch (err) {
|
|
14686
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
14687
|
-
attempts.push({
|
|
14688
|
-
...unexecuted(attempt, "", usage, command, started),
|
|
14689
|
-
llmError: null,
|
|
14690
|
-
armBError: message.slice(0, 500)
|
|
14691
|
-
});
|
|
14692
|
-
options.onProgress?.(
|
|
14693
|
-
`fix-loop attempt ${attempt}: sandbox error \u2014 ${message.slice(0, 160)}`
|
|
14694
|
-
);
|
|
14695
|
-
return {
|
|
14696
|
-
flipped: false,
|
|
14697
|
-
flippedAtAttempt: null,
|
|
14698
|
-
attempts,
|
|
14699
|
-
aborted: true,
|
|
14700
|
-
llmCalls,
|
|
14701
|
-
llmFailures,
|
|
14702
|
-
promptTokens,
|
|
14703
|
-
completionTokens
|
|
14704
|
-
};
|
|
14705
|
-
}
|
|
14706
|
-
attempts.push({
|
|
14707
|
-
attempt,
|
|
14708
|
-
command,
|
|
14709
|
-
llmError: null,
|
|
14710
|
-
usage,
|
|
14711
|
-
executed: true,
|
|
14712
|
-
exitCode: execution.exitCode,
|
|
14713
|
-
prefixExecuted: execution.prefixExecuted,
|
|
14714
|
-
prefixDivergences: execution.prefixDivergences,
|
|
14715
|
-
failureVanished: execution.failureVanished,
|
|
14716
|
-
stdoutTail: clipText(execution.stdout, tailChars),
|
|
14717
|
-
stderrTail: clipText(execution.stderr, tailChars),
|
|
14718
|
-
armBError: null,
|
|
14719
|
-
wallMs: Date.now() - started
|
|
14720
|
-
});
|
|
14721
|
-
options.onProgress?.(
|
|
14722
|
-
`fix-loop attempt ${attempt}: exit=${execution.exitCode} failureVanished=${execution.failureVanished}`
|
|
14723
|
-
);
|
|
14724
|
-
if (execution.failureVanished) {
|
|
14725
|
-
return {
|
|
14726
|
-
flipped: true,
|
|
14727
|
-
flippedAtAttempt: attempt,
|
|
14728
|
-
attempts,
|
|
14729
|
-
aborted: false,
|
|
14730
|
-
llmCalls,
|
|
14731
|
-
llmFailures,
|
|
14732
|
-
promptTokens,
|
|
14733
|
-
completionTokens
|
|
14734
|
-
};
|
|
14735
|
-
}
|
|
14736
|
-
}
|
|
14737
|
-
return {
|
|
14738
|
-
flipped: false,
|
|
14739
|
-
flippedAtAttempt: null,
|
|
14740
|
-
attempts,
|
|
14741
|
-
aborted: false,
|
|
14742
|
-
llmCalls,
|
|
14743
|
-
llmFailures,
|
|
14744
|
-
promptTokens,
|
|
14745
|
-
completionTokens
|
|
14746
|
-
};
|
|
14747
|
-
}
|
|
14748
|
-
|
|
14749
|
-
// src/replay-batch.ts
|
|
14750
|
-
import { execFile } from "child_process";
|
|
14751
|
-
import { createHash as createHash12 } from "crypto";
|
|
14752
|
-
import { appendFileSync, mkdirSync as mkdirSync2, mkdtempSync, writeFileSync as writeFileSync2 } from "fs";
|
|
14753
|
-
import { tmpdir as tmpdir5 } from "os";
|
|
14754
|
-
import { join as join26 } from "path";
|
|
14755
|
-
import { promisify } from "util";
|
|
14756
|
-
import {
|
|
14757
|
-
InMemoryTraceStore as InMemoryTraceStore3,
|
|
14758
|
-
runCounterfactual as runCounterfactual2
|
|
14759
|
-
} from "@tangle-network/agent-eval";
|
|
14760
|
-
var execFileAsync = promisify(execFile);
|
|
14761
|
-
function derivedImageTag(image, cwd) {
|
|
14762
|
-
const digest = createHash12("sha256").update(`${image}
|
|
14763
|
-
${cwd}`).digest("hex").slice(0, 12);
|
|
14764
|
-
return `ctb-replay:${digest}-uid1000`;
|
|
14765
|
-
}
|
|
14766
|
-
function dockerImagePreparer(options = {}) {
|
|
14767
|
-
const pullTimeoutMs = options.pullTimeoutMs ?? 12e5;
|
|
14768
|
-
const buildTimeoutMs = options.buildTimeoutMs ?? 9e5;
|
|
14769
|
-
const imageExists = async (tag) => {
|
|
14770
|
-
try {
|
|
14771
|
-
await execFileAsync("docker", ["image", "inspect", tag], { maxBuffer: 8 * 1024 * 1024 });
|
|
14772
|
-
return true;
|
|
14773
|
-
} catch {
|
|
14774
|
-
return false;
|
|
14775
|
-
}
|
|
14776
|
-
};
|
|
14777
|
-
const errorTail = (err) => {
|
|
14778
|
-
const raw = err && typeof err === "object" && "stderr" in err && typeof err.stderr === "string" && err.stderr.length > 0 ? err.stderr : err instanceof Error ? err.message : String(err);
|
|
14779
|
-
return raw.trim().split("\n").slice(-3).join(" | ").slice(0, 400);
|
|
14780
|
-
};
|
|
14781
|
-
return {
|
|
14782
|
-
async ensure(image, cwd) {
|
|
14783
|
-
if (cwd === "/") {
|
|
14784
|
-
if (await imageExists(image)) {
|
|
14785
|
-
return { succeeded: true, value: { derivedImage: image, pulled: false, built: false } };
|
|
14786
|
-
}
|
|
14787
|
-
try {
|
|
14788
|
-
await execFileAsync("docker", ["pull", image], {
|
|
14789
|
-
timeout: pullTimeoutMs,
|
|
14790
|
-
maxBuffer: 32 * 1024 * 1024
|
|
14791
|
-
});
|
|
14792
|
-
} catch (err) {
|
|
14793
|
-
return { succeeded: false, error: `pull ${image}: ${errorTail(err)}` };
|
|
14794
|
-
}
|
|
14795
|
-
return { succeeded: true, value: { derivedImage: image, pulled: true, built: false } };
|
|
14796
|
-
}
|
|
14797
|
-
const derived = derivedImageTag(image, cwd);
|
|
14798
|
-
if (await imageExists(derived)) {
|
|
14799
|
-
return { succeeded: true, value: { derivedImage: derived, pulled: false, built: false } };
|
|
14800
|
-
}
|
|
14801
|
-
let pulled = false;
|
|
14802
|
-
if (!await imageExists(image)) {
|
|
14803
|
-
try {
|
|
14804
|
-
await execFileAsync("docker", ["pull", image], {
|
|
14805
|
-
timeout: pullTimeoutMs,
|
|
14806
|
-
maxBuffer: 32 * 1024 * 1024
|
|
14807
|
-
});
|
|
14808
|
-
pulled = true;
|
|
14809
|
-
} catch (err) {
|
|
14810
|
-
return { succeeded: false, error: `pull ${image}: ${errorTail(err)}` };
|
|
14811
|
-
}
|
|
14812
|
-
}
|
|
14813
|
-
const contextDir = mkdtempSync(join26(tmpdir5(), "ctb-replay-image-"));
|
|
14814
|
-
const quotedCwd = `'${cwd.replaceAll("'", `'\\''`)}'`;
|
|
14815
|
-
writeFileSync2(
|
|
14816
|
-
join26(contextDir, "Dockerfile"),
|
|
14817
|
-
`FROM ${image}
|
|
14818
|
-
RUN chown -R 1000:1000 ${quotedCwd}
|
|
14819
|
-
`
|
|
14820
|
-
);
|
|
14821
|
-
try {
|
|
14822
|
-
await execFileAsync("docker", ["build", "-t", derived, contextDir], {
|
|
14823
|
-
timeout: buildTimeoutMs,
|
|
14824
|
-
maxBuffer: 32 * 1024 * 1024
|
|
14825
|
-
});
|
|
14826
|
-
} catch (err) {
|
|
14827
|
-
return { succeeded: false, error: `build ${derived} from ${image}: ${errorTail(err)}` };
|
|
14828
|
-
}
|
|
14829
|
-
return { succeeded: true, value: { derivedImage: derived, pulled, built: true } };
|
|
14830
|
-
}
|
|
14831
|
-
};
|
|
14832
|
-
}
|
|
14833
|
-
function mulberry32(seed) {
|
|
14834
|
-
let state = seed >>> 0;
|
|
14835
|
-
return () => {
|
|
14836
|
-
state = state + 1831565813 >>> 0;
|
|
14837
|
-
let t = state;
|
|
14838
|
-
t = Math.imul(t ^ t >>> 15, t | 1);
|
|
14839
|
-
t ^= t + Math.imul(t ^ t >>> 7, t | 61);
|
|
14840
|
-
return ((t ^ t >>> 14) >>> 0) / 4294967296;
|
|
14841
|
-
};
|
|
14842
|
-
}
|
|
14843
|
-
function seededSample(items, size, seed) {
|
|
14844
|
-
const pool = [...items];
|
|
14845
|
-
const random = mulberry32(seed);
|
|
14846
|
-
for (let i = pool.length - 1; i > 0; i--) {
|
|
14847
|
-
const j = Math.floor(random() * (i + 1));
|
|
14848
|
-
[pool[i], pool[j]] = [pool[j], pool[i]];
|
|
14849
|
-
}
|
|
14850
|
-
return new Set(pool.slice(0, size));
|
|
14851
|
-
}
|
|
14852
|
-
function caseOutDirName(row) {
|
|
14853
|
-
return `${row.corpus}--${row.trajId}`.replaceAll(/[^A-Za-z0-9._-]/g, "_").slice(0, 180);
|
|
14854
|
-
}
|
|
14855
|
-
async function executeArmB(replayCase, fixCommand, backend, signature, stepTimeoutMs, prefixLimit, onProgress) {
|
|
14856
|
-
const store = new InMemoryTraceStore3();
|
|
14857
|
-
const { runId } = await ingestCodeTraceBenchSteps(store, replayCase.steps, replayCase.trajId);
|
|
14858
|
-
const runner = new SandboxCounterfactualRunner(backend, {
|
|
14859
|
-
cwd: replayCase.cwd,
|
|
14860
|
-
stepTimeoutMs,
|
|
14861
|
-
prefixLimit,
|
|
14862
|
-
onProgress
|
|
14863
|
-
});
|
|
14864
|
-
await runCounterfactual2(
|
|
14865
|
-
store,
|
|
14866
|
-
runId,
|
|
14867
|
-
{
|
|
14868
|
-
kind: "custom",
|
|
14869
|
-
at: replayCase.k - 1,
|
|
14870
|
-
describe: "arm-B corrected step",
|
|
14871
|
-
apply: (step) => ({
|
|
14872
|
-
...step,
|
|
14873
|
-
span: { ...step.span, args: { command: fixCommand } }
|
|
14874
|
-
})
|
|
14875
|
-
},
|
|
14876
|
-
runner
|
|
14877
|
-
);
|
|
14878
|
-
const exec = runner.lastArm;
|
|
14879
|
-
if (!exec) throw new Error("arm B finished without executing the corrected step");
|
|
14880
|
-
const output = `${exec.stdout}
|
|
14881
|
-
${exec.stderr}`;
|
|
14882
|
-
return {
|
|
14883
|
-
exitCode: exec.exitCode,
|
|
14884
|
-
prefixExecuted: runner.lastPrefix?.prefixExecuted ?? 0,
|
|
14885
|
-
prefixDivergences: runner.lastPrefix?.prefixDivergences.length ?? 0,
|
|
14886
|
-
failureVanished: exec.exitCode === 0 && (signature ? !output.includes(signature) : true),
|
|
14887
|
-
stdout: exec.stdout,
|
|
14888
|
-
stderr: exec.stderr
|
|
14889
|
-
};
|
|
14890
|
-
}
|
|
14891
|
-
function rate(numerator, denominator) {
|
|
14892
|
-
return denominator > 0 ? numerator / denominator : null;
|
|
14893
|
-
}
|
|
14894
|
-
async function runReplayBatch(options) {
|
|
14895
|
-
const onProgress = options.onProgress ?? (() => {
|
|
14896
|
-
});
|
|
14897
|
-
const enumeration = enumerateReplayableCases(options.corpora);
|
|
14898
|
-
let selected = enumeration.replayable;
|
|
14899
|
-
if (options.caseFilter) {
|
|
14900
|
-
selected = selected.filter((c) => c.trajId.includes(options.caseFilter));
|
|
14901
|
-
}
|
|
14902
|
-
if (options.caseLimit !== void 0) selected = selected.slice(0, options.caseLimit);
|
|
14903
|
-
onProgress(
|
|
14904
|
-
`enumerated ${enumeration.labelEntryCount} label entries \u2192 ${enumeration.replayable.length} replayable, ${enumeration.excluded.length} excluded; executing ${selected.length}`
|
|
14905
|
-
);
|
|
14906
|
-
const preparer = options.preparer ?? dockerImagePreparer();
|
|
14907
|
-
const backendFactory = options.backendFactory ?? ((derivedImage) => sandboxReplayBackend({
|
|
14908
|
-
image: derivedImage,
|
|
14909
|
-
apiKey: requireOption2(options.apiKey, "apiKey"),
|
|
14910
|
-
baseUrl: requireOption2(options.baseUrl, "baseUrl"),
|
|
14911
|
-
maxLifetimeSeconds: options.maxLifetimeSeconds ?? 3600
|
|
14912
|
-
}));
|
|
14913
|
-
mkdirSync2(options.out, { recursive: true });
|
|
14914
|
-
const progressPath = join26(options.out, "cases.jsonl");
|
|
14915
|
-
const rows = [];
|
|
14916
|
-
const pullFailures = [];
|
|
14917
|
-
const verdictByTraj = /* @__PURE__ */ new Map();
|
|
14918
|
-
for (const [index, replayCase] of selected.entries()) {
|
|
14919
|
-
const caseStart = Date.now();
|
|
14920
|
-
const label = `[${index + 1}/${selected.length}] ${replayCase.corpus}/${replayCase.trajId}`;
|
|
14921
|
-
onProgress(`${label}: preparing image ${replayCase.image}`);
|
|
14922
|
-
const preparation = await preparer.ensure(replayCase.image, replayCase.cwd);
|
|
14923
|
-
const base = {
|
|
14924
|
-
corpus: replayCase.corpus,
|
|
14925
|
-
trajId: replayCase.trajId,
|
|
14926
|
-
image: replayCase.image,
|
|
14927
|
-
cwd: replayCase.cwd,
|
|
14928
|
-
cwdSource: replayCase.cwdSource,
|
|
14929
|
-
k: replayCase.k,
|
|
14930
|
-
stepCount: replayCase.steps.length,
|
|
14931
|
-
goldIncorrectSteps: replayCase.goldIncorrectSteps,
|
|
14932
|
-
submitGoldsSkipped: replayCase.submitGoldsSkipped,
|
|
14933
|
-
recordedReturncodeAtK: replayCase.recordedReturncodeAtK
|
|
14934
|
-
};
|
|
14935
|
-
if (!preparation.succeeded) {
|
|
14936
|
-
pullFailures.push({
|
|
14937
|
-
corpus: replayCase.corpus,
|
|
14938
|
-
trajId: replayCase.trajId,
|
|
14939
|
-
image: replayCase.image,
|
|
14940
|
-
error: preparation.error
|
|
14941
|
-
});
|
|
14942
|
-
const row = {
|
|
14943
|
-
...base,
|
|
14944
|
-
derivedImage: null,
|
|
14945
|
-
signature: null,
|
|
14946
|
-
status: "image-unavailable",
|
|
14947
|
-
error: preparation.error,
|
|
14948
|
-
imagePulled: false,
|
|
14949
|
-
imageBuilt: false,
|
|
14950
|
-
prefixExecuted: null,
|
|
14951
|
-
prefixDivergences: null,
|
|
14952
|
-
prefixDivergencePct: null,
|
|
14953
|
-
armAExit: null,
|
|
14954
|
-
armAReturncodeMatch: false,
|
|
14955
|
-
armASignatureMatch: false,
|
|
14956
|
-
replayed: false,
|
|
14957
|
-
fix: null,
|
|
14958
|
-
wallMs: Date.now() - caseStart
|
|
14959
|
-
};
|
|
14960
|
-
rows.push(row);
|
|
14961
|
-
appendFileSync(progressPath, `${JSON.stringify(row)}
|
|
14962
|
-
`);
|
|
14963
|
-
onProgress(`${label}: image unavailable \u2014 ${preparation.error}`);
|
|
14964
|
-
continue;
|
|
14965
|
-
}
|
|
14966
|
-
const { derivedImage, pulled, built } = preparation.value;
|
|
14967
|
-
const stepTimeoutMs = options.stepTimeoutMs ?? replayCase.recordedStepTimeoutMs ?? 12e4;
|
|
14968
|
-
const caseOut = join26(options.out, caseOutDirName(replayCase));
|
|
14969
|
-
onProgress(`${label}: arm A on ${derivedImage} (k=${replayCase.k}, timeout ${stepTimeoutMs}ms)`);
|
|
14970
|
-
try {
|
|
14971
|
-
const verdict = await replayVerify({
|
|
14972
|
-
stepsPath: replayCase.stepsPath,
|
|
14973
|
-
image: derivedImage,
|
|
14974
|
-
at: replayCase.k,
|
|
14975
|
-
cwd: replayCase.cwd,
|
|
14976
|
-
out: caseOut,
|
|
14977
|
-
caseId: replayCase.trajId,
|
|
14978
|
-
stepTimeoutMs,
|
|
14979
|
-
prefixLimit: options.prefixLimit,
|
|
14980
|
-
backend: backendFactory(derivedImage),
|
|
14981
|
-
onProgress: (message) => onProgress(`${label}: ${message}`)
|
|
14982
|
-
});
|
|
14983
|
-
verdictByTraj.set(replayCase.trajId, verdict);
|
|
14984
|
-
const divergencePct = verdict.prefixExecuted > 0 ? verdict.prefixDivergences.length / verdict.prefixExecuted * 100 : 0;
|
|
14985
|
-
const returncodeMatch = verdict.recordedReturncode !== null && verdict.armA.exitCode === verdict.recordedReturncode;
|
|
14986
|
-
const row = {
|
|
14987
|
-
...base,
|
|
14988
|
-
derivedImage,
|
|
14989
|
-
signature: verdict.signature,
|
|
14990
|
-
status: "ok",
|
|
14991
|
-
error: null,
|
|
14992
|
-
imagePulled: pulled,
|
|
14993
|
-
imageBuilt: built,
|
|
14994
|
-
prefixExecuted: verdict.prefixExecuted,
|
|
14995
|
-
prefixDivergences: verdict.prefixDivergences.length,
|
|
14996
|
-
prefixDivergencePct: Number(divergencePct.toFixed(1)),
|
|
14997
|
-
armAExit: verdict.armA.exitCode,
|
|
14998
|
-
armAReturncodeMatch: returncodeMatch,
|
|
14999
|
-
armASignatureMatch: verdict.armA.failureSignatureMatch,
|
|
15000
|
-
replayed: divergencePct <= 10 && returncodeMatch,
|
|
15001
|
-
fix: null,
|
|
15002
|
-
wallMs: Date.now() - caseStart
|
|
15003
|
-
};
|
|
15004
|
-
rows.push(row);
|
|
15005
|
-
appendFileSync(progressPath, `${JSON.stringify(row)}
|
|
15006
|
-
`);
|
|
15007
|
-
onProgress(
|
|
15008
|
-
`${label}: armA exit=${verdict.armA.exitCode} rcMatch=${returncodeMatch} divergences=${verdict.prefixDivergences.length}/${verdict.prefixExecuted}`
|
|
15009
|
-
);
|
|
15010
|
-
} catch (err) {
|
|
15011
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
15012
|
-
const row = {
|
|
15013
|
-
...base,
|
|
15014
|
-
derivedImage,
|
|
15015
|
-
signature: null,
|
|
15016
|
-
status: "replay-error",
|
|
15017
|
-
error: message.slice(0, 500),
|
|
15018
|
-
imagePulled: pulled,
|
|
15019
|
-
imageBuilt: built,
|
|
15020
|
-
prefixExecuted: null,
|
|
15021
|
-
prefixDivergences: null,
|
|
15022
|
-
prefixDivergencePct: null,
|
|
15023
|
-
armAExit: null,
|
|
15024
|
-
armAReturncodeMatch: false,
|
|
15025
|
-
armASignatureMatch: false,
|
|
15026
|
-
replayed: false,
|
|
15027
|
-
fix: null,
|
|
15028
|
-
wallMs: Date.now() - caseStart
|
|
15029
|
-
};
|
|
15030
|
-
rows.push(row);
|
|
15031
|
-
appendFileSync(progressPath, `${JSON.stringify(row)}
|
|
15032
|
-
`);
|
|
15033
|
-
onProgress(`${label}: replay error \u2014 ${message.slice(0, 200)}`);
|
|
15034
|
-
}
|
|
15035
|
-
}
|
|
15036
|
-
let llm = null;
|
|
15037
|
-
if (options.fix !== "none") {
|
|
15038
|
-
const caller = options.fixCaller;
|
|
15039
|
-
if (!caller) throw new Error(`replay-verify-batch: fix=${options.fix} requires a fixCaller`);
|
|
15040
|
-
const fixAttempts = options.fixAttempts ?? 3;
|
|
15041
|
-
const maxFixCases = options.maxFixCases ?? 30;
|
|
15042
|
-
const seed = options.seed ?? 17;
|
|
15043
|
-
const eligible = rows.filter((r) => r.status === "ok" && r.replayed);
|
|
15044
|
-
const sampled = eligible.length > maxFixCases ? seededSample(eligible, maxFixCases, seed) : new Set(eligible);
|
|
15045
|
-
if (eligible.length > maxFixCases) {
|
|
15046
|
-
onProgress(
|
|
15047
|
-
`fix phase: ${eligible.length} eligible > cap ${maxFixCases}; seeded sample (seed=${seed})`
|
|
15048
|
-
);
|
|
15049
|
-
}
|
|
15050
|
-
let calls = 0;
|
|
15051
|
-
let failures = 0;
|
|
15052
|
-
let promptTokens = 0;
|
|
15053
|
-
let completionTokens = 0;
|
|
15054
|
-
for (const row of eligible) {
|
|
15055
|
-
const index = rows.indexOf(row);
|
|
15056
|
-
if (!sampled.has(row)) {
|
|
15057
|
-
rows[index] = {
|
|
15058
|
-
...row,
|
|
15059
|
-
fix: {
|
|
15060
|
-
attempted: false,
|
|
15061
|
-
sampledOut: true,
|
|
15062
|
-
command: null,
|
|
15063
|
-
llmError: null,
|
|
15064
|
-
usage: null,
|
|
15065
|
-
armBExit: null,
|
|
15066
|
-
armBPrefixExecuted: null,
|
|
15067
|
-
armBPrefixDivergences: null,
|
|
15068
|
-
failureVanished: null,
|
|
15069
|
-
armBError: null,
|
|
15070
|
-
attempts: null,
|
|
15071
|
-
flippedAtAttempt: null
|
|
15072
|
-
}
|
|
15073
|
-
};
|
|
15074
|
-
continue;
|
|
15075
|
-
}
|
|
15076
|
-
const replayCase = selected.find((c) => c.trajId === row.trajId && c.corpus === row.corpus);
|
|
15077
|
-
const label = `fix ${row.corpus}/${row.trajId}`;
|
|
15078
|
-
const signature = verdictByTraj.get(row.trajId)?.signature ?? null;
|
|
15079
|
-
const stepTimeoutMs = options.stepTimeoutMs ?? replayCase.recordedStepTimeoutMs ?? 12e4;
|
|
15080
|
-
const promptInput = {
|
|
15081
|
-
taskStatement: replayCase.taskStatement,
|
|
15082
|
-
steps: replayCase.steps,
|
|
15083
|
-
k: replayCase.k
|
|
15084
|
-
};
|
|
15085
|
-
const caseOut = join26(options.out, caseOutDirName(row));
|
|
15086
|
-
if (options.fix === "loop") {
|
|
15087
|
-
onProgress(`${label}: fix loop (budget ${fixAttempts} attempts)`);
|
|
15088
|
-
const result = await runFixLoop(
|
|
15089
|
-
caller,
|
|
15090
|
-
promptInput,
|
|
15091
|
-
async (command, attempt) => {
|
|
15092
|
-
onProgress(
|
|
15093
|
-
`${label}: attempt ${attempt} arm B \u2014 ${command.split("\n")[0].slice(0, 120)}`
|
|
15094
|
-
);
|
|
15095
|
-
const armB = await executeArmB(
|
|
15096
|
-
replayCase,
|
|
15097
|
-
command,
|
|
15098
|
-
backendFactory(row.derivedImage),
|
|
15099
|
-
signature,
|
|
15100
|
-
stepTimeoutMs,
|
|
15101
|
-
options.prefixLimit,
|
|
15102
|
-
(message) => onProgress(`${label}: attempt ${attempt}: ${message}`)
|
|
15103
|
-
);
|
|
15104
|
-
mkdirSync2(caseOut, { recursive: true });
|
|
15105
|
-
writeFileSync2(
|
|
15106
|
-
join26(caseOut, `armB-attempt${attempt}-result.json`),
|
|
15107
|
-
`${JSON.stringify({ command, ...armB }, null, 2)}
|
|
15108
|
-
`
|
|
15109
|
-
);
|
|
15110
|
-
writeFileSync2(
|
|
15111
|
-
join26(caseOut, "armB-result.json"),
|
|
15112
|
-
`${JSON.stringify({ command, attempt, ...armB }, null, 2)}
|
|
15113
|
-
`
|
|
15114
|
-
);
|
|
15115
|
-
return armB;
|
|
15116
|
-
},
|
|
15117
|
-
{ maxAttempts: fixAttempts, onProgress: (message) => onProgress(`${label}: ${message}`) }
|
|
15118
|
-
);
|
|
15119
|
-
calls += result.llmCalls;
|
|
15120
|
-
failures += result.llmFailures;
|
|
15121
|
-
promptTokens += result.promptTokens;
|
|
15122
|
-
completionTokens += result.completionTokens;
|
|
15123
|
-
const usage = result.promptTokens + result.completionTokens > 0 ? { promptTokens: result.promptTokens, completionTokens: result.completionTokens } : null;
|
|
15124
|
-
const flippedRecord = result.flippedAtAttempt !== null ? result.attempts.find((a) => a.attempt === result.flippedAtAttempt) : null;
|
|
15125
|
-
const summary = flippedRecord ?? [...result.attempts].reverse().find((a) => a.executed) ?? null;
|
|
15126
|
-
const lastRecord = result.attempts.at(-1) ?? null;
|
|
15127
|
-
rows[index] = {
|
|
15128
|
-
...row,
|
|
15129
|
-
fix: summary ? {
|
|
15130
|
-
attempted: true,
|
|
15131
|
-
sampledOut: false,
|
|
15132
|
-
command: summary.command,
|
|
15133
|
-
llmError: null,
|
|
15134
|
-
usage,
|
|
15135
|
-
armBExit: summary.exitCode,
|
|
15136
|
-
armBPrefixExecuted: summary.prefixExecuted,
|
|
15137
|
-
armBPrefixDivergences: summary.prefixDivergences,
|
|
15138
|
-
failureVanished: summary.failureVanished,
|
|
15139
|
-
armBError: null,
|
|
15140
|
-
attempts: result.attempts,
|
|
15141
|
-
flippedAtAttempt: result.flippedAtAttempt
|
|
15142
|
-
} : result.aborted ? {
|
|
15143
|
-
attempted: true,
|
|
15144
|
-
sampledOut: false,
|
|
15145
|
-
command: lastRecord?.command ?? null,
|
|
15146
|
-
llmError: null,
|
|
15147
|
-
usage,
|
|
15148
|
-
armBExit: null,
|
|
15149
|
-
armBPrefixExecuted: null,
|
|
15150
|
-
armBPrefixDivergences: null,
|
|
15151
|
-
failureVanished: null,
|
|
15152
|
-
armBError: lastRecord?.armBError ?? "sandbox error",
|
|
15153
|
-
attempts: result.attempts,
|
|
15154
|
-
flippedAtAttempt: null
|
|
15155
|
-
} : {
|
|
15156
|
-
attempted: true,
|
|
15157
|
-
sampledOut: false,
|
|
15158
|
-
command: null,
|
|
15159
|
-
llmError: lastRecord?.llmError ?? "no attempt produced a runnable fix",
|
|
15160
|
-
usage,
|
|
15161
|
-
armBExit: null,
|
|
15162
|
-
armBPrefixExecuted: null,
|
|
15163
|
-
armBPrefixDivergences: null,
|
|
15164
|
-
failureVanished: null,
|
|
15165
|
-
armBError: null,
|
|
15166
|
-
attempts: result.attempts,
|
|
15167
|
-
flippedAtAttempt: null
|
|
15168
|
-
}
|
|
15169
|
-
};
|
|
15170
|
-
onProgress(
|
|
15171
|
-
`${label}: loop done \u2014 flipped=${result.flipped}` + (result.flippedAtAttempt !== null ? ` at attempt ${result.flippedAtAttempt}` : "") + ` (${result.llmCalls} calls, ${result.attempts.filter((a) => a.executed).length} arms)`
|
|
15172
|
-
);
|
|
15173
|
-
continue;
|
|
15174
|
-
}
|
|
15175
|
-
onProgress(`${label}: generating corrected command`);
|
|
15176
|
-
calls += 1;
|
|
15177
|
-
const generated = await generateFixCommand(caller, promptInput);
|
|
15178
|
-
if (!generated.succeeded) {
|
|
15179
|
-
failures += 1;
|
|
15180
|
-
rows[index] = {
|
|
15181
|
-
...row,
|
|
15182
|
-
fix: {
|
|
15183
|
-
attempted: true,
|
|
15184
|
-
sampledOut: false,
|
|
15185
|
-
command: null,
|
|
15186
|
-
llmError: generated.error,
|
|
15187
|
-
usage: null,
|
|
15188
|
-
armBExit: null,
|
|
15189
|
-
armBPrefixExecuted: null,
|
|
15190
|
-
armBPrefixDivergences: null,
|
|
15191
|
-
failureVanished: null,
|
|
15192
|
-
armBError: null,
|
|
15193
|
-
attempts: null,
|
|
15194
|
-
flippedAtAttempt: null
|
|
15195
|
-
}
|
|
15196
|
-
};
|
|
15197
|
-
onProgress(`${label}: LLM failed \u2014 ${generated.error.slice(0, 200)}`);
|
|
15198
|
-
continue;
|
|
15199
|
-
}
|
|
15200
|
-
promptTokens += generated.value.usage?.promptTokens ?? 0;
|
|
15201
|
-
completionTokens += generated.value.usage?.completionTokens ?? 0;
|
|
15202
|
-
onProgress(`${label}: arm B \u2014 ${generated.value.command.split("\n")[0].slice(0, 120)}`);
|
|
15203
|
-
try {
|
|
15204
|
-
const armB = await executeArmB(
|
|
15205
|
-
replayCase,
|
|
15206
|
-
generated.value.command,
|
|
15207
|
-
backendFactory(row.derivedImage),
|
|
15208
|
-
signature,
|
|
15209
|
-
stepTimeoutMs,
|
|
15210
|
-
options.prefixLimit,
|
|
15211
|
-
(message) => onProgress(`${label}: ${message}`)
|
|
15212
|
-
);
|
|
15213
|
-
rows[index] = {
|
|
15214
|
-
...row,
|
|
15215
|
-
fix: {
|
|
15216
|
-
attempted: true,
|
|
15217
|
-
sampledOut: false,
|
|
15218
|
-
command: generated.value.command,
|
|
15219
|
-
llmError: null,
|
|
15220
|
-
usage: generated.value.usage,
|
|
15221
|
-
armBExit: armB.exitCode,
|
|
15222
|
-
armBPrefixExecuted: armB.prefixExecuted,
|
|
15223
|
-
armBPrefixDivergences: armB.prefixDivergences,
|
|
15224
|
-
failureVanished: armB.failureVanished,
|
|
15225
|
-
armBError: null,
|
|
15226
|
-
attempts: null,
|
|
15227
|
-
flippedAtAttempt: armB.failureVanished ? 1 : null
|
|
15228
|
-
}
|
|
15229
|
-
};
|
|
15230
|
-
mkdirSync2(caseOut, { recursive: true });
|
|
15231
|
-
writeFileSync2(
|
|
15232
|
-
join26(caseOut, "armB-result.json"),
|
|
15233
|
-
`${JSON.stringify({ command: generated.value.command, ...armB }, null, 2)}
|
|
15234
|
-
`
|
|
15235
|
-
);
|
|
15236
|
-
onProgress(`${label}: armB exit=${armB.exitCode} failureVanished=${armB.failureVanished}`);
|
|
15237
|
-
} catch (err) {
|
|
15238
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
15239
|
-
rows[index] = {
|
|
15240
|
-
...row,
|
|
15241
|
-
fix: {
|
|
15242
|
-
attempted: true,
|
|
15243
|
-
sampledOut: false,
|
|
15244
|
-
command: generated.value.command,
|
|
15245
|
-
llmError: null,
|
|
15246
|
-
usage: generated.value.usage,
|
|
15247
|
-
armBExit: null,
|
|
15248
|
-
armBPrefixExecuted: null,
|
|
15249
|
-
armBPrefixDivergences: null,
|
|
15250
|
-
failureVanished: null,
|
|
15251
|
-
armBError: message.slice(0, 500),
|
|
15252
|
-
attempts: null,
|
|
15253
|
-
flippedAtAttempt: null
|
|
15254
|
-
}
|
|
15255
|
-
};
|
|
15256
|
-
onProgress(`${label}: arm B error \u2014 ${message.slice(0, 200)}`);
|
|
15257
|
-
}
|
|
15258
|
-
}
|
|
15259
|
-
llm = {
|
|
15260
|
-
model: options.fixModelLabel ?? "unknown",
|
|
15261
|
-
calls,
|
|
15262
|
-
failures,
|
|
15263
|
-
promptTokens,
|
|
15264
|
-
completionTokens
|
|
15265
|
-
};
|
|
15266
|
-
}
|
|
15267
|
-
const executedRows = rows.filter((r) => r.status === "ok");
|
|
15268
|
-
const replayed = executedRows.filter((r) => r.replayed);
|
|
15269
|
-
const signatureStrict = executedRows.filter(
|
|
15270
|
-
(r) => r.replayed && r.armASignatureMatch
|
|
15271
|
-
);
|
|
15272
|
-
const armBExecuted = rows.filter((r) => r.fix?.attempted && r.fix.failureVanished !== null);
|
|
15273
|
-
const flipped = armBExecuted.filter((r) => r.fix.failureVanished === true);
|
|
15274
|
-
const armBNonzeroRc = armBExecuted.filter(
|
|
15275
|
-
(r) => r.recordedReturncodeAtK !== null && r.recordedReturncodeAtK !== 0
|
|
15276
|
-
);
|
|
15277
|
-
const flippedNonzeroRc = armBNonzeroRc.filter((r) => r.fix.failureVanished === true);
|
|
15278
|
-
const attempt1Executed = rows.filter(
|
|
15279
|
-
(r) => r.fix?.attempts?.find((a) => a.attempt === 1)?.executed === true
|
|
15280
|
-
);
|
|
15281
|
-
const flippedAt1 = rows.filter((r) => r.fix?.flippedAtAttempt === 1);
|
|
15282
|
-
const flipsByAttempt = {};
|
|
15283
|
-
for (const row of rows) {
|
|
15284
|
-
const at = row.fix?.flippedAtAttempt;
|
|
15285
|
-
if (typeof at === "number") flipsByAttempt[String(at)] = (flipsByAttempt[String(at)] ?? 0) + 1;
|
|
15286
|
-
}
|
|
15287
|
-
const excludedByReason = {};
|
|
15288
|
-
for (const excluded of enumeration.excluded) {
|
|
15289
|
-
excludedByReason[excluded.reason] = (excludedByReason[excluded.reason] ?? 0) + 1;
|
|
15290
|
-
}
|
|
15291
|
-
const submitGoldsByCorpus = {};
|
|
15292
|
-
const submitEntry = (corpus) => submitGoldsByCorpus[corpus] ??= { submitOnlyCases: 0, goldsSkippedWithinReplayable: 0 };
|
|
15293
|
-
for (const excluded of enumeration.excluded) {
|
|
15294
|
-
if (excluded.reason === "gold-only-submit-step") submitEntry(excluded.corpus).submitOnlyCases += 1;
|
|
15295
|
-
}
|
|
15296
|
-
for (const replayCase of enumeration.replayable) {
|
|
15297
|
-
if (replayCase.submitGoldsSkipped > 0) {
|
|
15298
|
-
submitEntry(replayCase.corpus).goldsSkippedWithinReplayable += replayCase.submitGoldsSkipped;
|
|
15299
|
-
}
|
|
15300
|
-
}
|
|
15301
|
-
const report = {
|
|
15302
|
-
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
15303
|
-
corpora: options.corpora.map((c) => ({
|
|
15304
|
-
name: c.name,
|
|
15305
|
-
labelsPath: c.labelsPath,
|
|
15306
|
-
preparedDir: c.preparedDir
|
|
15307
|
-
})),
|
|
15308
|
-
totals: {
|
|
15309
|
-
labelEntries: enumeration.labelEntryCount,
|
|
15310
|
-
replayable: enumeration.replayable.length,
|
|
15311
|
-
executed: selected.length,
|
|
15312
|
-
excludedByReason,
|
|
15313
|
-
submitGoldsByCorpus
|
|
15314
|
-
},
|
|
15315
|
-
headline: {
|
|
15316
|
-
replayabilityRate: {
|
|
15317
|
-
numerator: replayed.length,
|
|
15318
|
-
denominator: selected.length,
|
|
15319
|
-
value: rate(replayed.length, selected.length)
|
|
15320
|
-
},
|
|
15321
|
-
signatureStrictRate: {
|
|
15322
|
-
numerator: signatureStrict.length,
|
|
15323
|
-
denominator: selected.length,
|
|
15324
|
-
value: rate(signatureStrict.length, selected.length)
|
|
15325
|
-
},
|
|
15326
|
-
fixFlipRate: options.fix !== "none" ? {
|
|
15327
|
-
numerator: flipped.length,
|
|
15328
|
-
denominator: armBExecuted.length,
|
|
15329
|
-
value: rate(flipped.length, armBExecuted.length)
|
|
15330
|
-
} : null,
|
|
15331
|
-
fixFlipRateNonzeroRc: options.fix !== "none" ? {
|
|
15332
|
-
numerator: flippedNonzeroRc.length,
|
|
15333
|
-
denominator: armBNonzeroRc.length,
|
|
15334
|
-
value: rate(flippedNonzeroRc.length, armBNonzeroRc.length)
|
|
15335
|
-
} : null,
|
|
15336
|
-
fixFlipAttempt1: options.fix === "loop" ? {
|
|
15337
|
-
numerator: flippedAt1.length,
|
|
15338
|
-
denominator: attempt1Executed.length,
|
|
15339
|
-
value: rate(flippedAt1.length, attempt1Executed.length)
|
|
15340
|
-
} : null,
|
|
15341
|
-
flipsByAttempt: options.fix === "loop" ? flipsByAttempt : null
|
|
15342
|
-
},
|
|
15343
|
-
llm,
|
|
15344
|
-
excluded: enumeration.excluded,
|
|
15345
|
-
pullFailures,
|
|
15346
|
-
cases: rows
|
|
15347
|
-
};
|
|
15348
|
-
writeFileSync2(join26(options.out, "batch-report.json"), `${JSON.stringify(report, null, 2)}
|
|
15349
|
-
`);
|
|
15350
|
-
writeFileSync2(join26(options.out, "batch-report.md"), renderBatchReport(report));
|
|
15351
|
-
return report;
|
|
15352
|
-
}
|
|
15353
|
-
function requireOption2(value, name) {
|
|
15354
|
-
if (value === void 0) throw new Error(`replay-verify-batch: ${name} is required`);
|
|
15355
|
-
return value;
|
|
15356
|
-
}
|
|
15357
|
-
function pct(value) {
|
|
15358
|
-
return value === null ? "\u2014" : `${(value * 100).toFixed(1)}%`;
|
|
15359
|
-
}
|
|
15360
|
-
function renderBatchReport(report) {
|
|
15361
|
-
const lines = [];
|
|
15362
|
-
const { headline, totals } = report;
|
|
15363
|
-
lines.push("# Replay-verify batch report");
|
|
15364
|
-
lines.push("");
|
|
15365
|
-
lines.push(`Generated ${report.generatedAt}.`);
|
|
15366
|
-
lines.push("");
|
|
15367
|
-
lines.push("## Headline");
|
|
15368
|
-
lines.push("");
|
|
15369
|
-
lines.push(
|
|
15370
|
-
`- **Replayability rate: ${pct(headline.replayabilityRate.value)}** (${headline.replayabilityRate.numerator}/${headline.replayabilityRate.denominator} replayable cases where the prefix replayed with \u226410% returncode divergence AND arm A reproduced the recorded returncode at the gold step k).`
|
|
15371
|
-
);
|
|
15372
|
-
lines.push(
|
|
15373
|
-
`- Signature-strict rate: ${pct(headline.signatureStrictRate.value)} (${headline.signatureStrictRate.numerator}/${headline.signatureStrictRate.denominator}; additionally requires the recorded error substring in arm A output).`
|
|
15374
|
-
);
|
|
15375
|
-
if (headline.fixFlipRate) {
|
|
15376
|
-
lines.push(
|
|
15377
|
-
`- **Fix-flip rate: ${pct(headline.fixFlipRate.value)}** (${headline.fixFlipRate.numerator}/${headline.fixFlipRate.denominator} arm-B-executed cases where the generated fix made the failure vanish).`
|
|
15378
|
-
);
|
|
15379
|
-
}
|
|
15380
|
-
if (headline.fixFlipRateNonzeroRc) {
|
|
15381
|
-
lines.push(
|
|
15382
|
-
`- Fix-flip rate on recorded-rc\u22600 cases: ${pct(headline.fixFlipRateNonzeroRc.value)} (${headline.fixFlipRateNonzeroRc.numerator}/${headline.fixFlipRateNonzeroRc.denominator}; real recorded failures \u2014 a gold step recorded with rc 0 flips vacuously).`
|
|
15383
|
-
);
|
|
15384
|
-
}
|
|
15385
|
-
if (headline.fixFlipAttempt1) {
|
|
15386
|
-
lines.push(
|
|
15387
|
-
`- Fix-flip@1: ${pct(headline.fixFlipAttempt1.value)} (${headline.fixFlipAttempt1.numerator}/${headline.fixFlipAttempt1.denominator} cases whose attempt 1 executed \u2014 the one-shot-comparable number).`
|
|
15388
|
-
);
|
|
15389
|
-
}
|
|
15390
|
-
if (headline.flipsByAttempt && Object.keys(headline.flipsByAttempt).length > 0) {
|
|
15391
|
-
const parts = Object.entries(headline.flipsByAttempt).sort((a, b) => Number(a[0]) - Number(b[0])).map(([attempt, count2]) => `attempt ${attempt}: ${count2}`);
|
|
15392
|
-
lines.push(`- Flips by attempt: ${parts.join(", ")}.`);
|
|
15393
|
-
}
|
|
15394
|
-
lines.push("");
|
|
15395
|
-
lines.push("## Enumeration");
|
|
15396
|
-
lines.push("");
|
|
15397
|
-
lines.push(
|
|
15398
|
-
`${totals.labelEntries} label entries across ${report.corpora.length} corpora \u2192 ${totals.replayable} replayable (SWE-style docker image + \u22651 gold incorrect step), ${totals.executed} executed.`
|
|
15399
|
-
);
|
|
15400
|
-
lines.push("");
|
|
15401
|
-
lines.push("| exclusion reason | count |");
|
|
15402
|
-
lines.push("| --- | --- |");
|
|
15403
|
-
for (const [reason, count2] of Object.entries(totals.excludedByReason).sort((a, b) => b[1] - a[1])) {
|
|
15404
|
-
lines.push(`| ${reason} | ${count2} |`);
|
|
15405
|
-
}
|
|
15406
|
-
const submitEntries = Object.entries(totals.submitGoldsByCorpus);
|
|
15407
|
-
if (submitEntries.length > 0) {
|
|
15408
|
-
lines.push("");
|
|
15409
|
-
lines.push(
|
|
15410
|
-
"Submit-command golds are never counterfactual targets (a gold on the submit step marks a bad submit decision, not a failed command):"
|
|
15411
|
-
);
|
|
15412
|
-
lines.push("");
|
|
15413
|
-
lines.push("| corpus | cases excluded (all golds = submit) | golds skipped within replayable cases |");
|
|
15414
|
-
lines.push("| --- | --- | --- |");
|
|
15415
|
-
for (const [corpus, stats] of submitEntries.sort((a, b) => a[0].localeCompare(b[0]))) {
|
|
15416
|
-
lines.push(`| ${corpus} | ${stats.submitOnlyCases} | ${stats.goldsSkippedWithinReplayable} |`);
|
|
15417
|
-
}
|
|
15418
|
-
}
|
|
15419
|
-
if (report.pullFailures.length > 0) {
|
|
15420
|
-
lines.push("");
|
|
15421
|
-
lines.push("## Image pull/build failures");
|
|
15422
|
-
lines.push("");
|
|
15423
|
-
lines.push("| corpus | trajectory | image | error |");
|
|
15424
|
-
lines.push("| --- | --- | --- | --- |");
|
|
15425
|
-
for (const failure2 of report.pullFailures) {
|
|
15426
|
-
lines.push(
|
|
15427
|
-
`| ${failure2.corpus} | ${failure2.trajId} | \`${failure2.image}\` | ${failure2.error.replaceAll("|", "\\|")} |`
|
|
15428
|
-
);
|
|
15429
|
-
}
|
|
15430
|
-
}
|
|
15431
|
-
lines.push("");
|
|
15432
|
-
lines.push("## Per-case results");
|
|
15433
|
-
lines.push("");
|
|
15434
|
-
lines.push(
|
|
15435
|
-
"| corpus | trajectory | k | rc@k | prefix | div | div% | armA exit | rc match | sig match | replayed | fix | armB exit | vanished | wall s |"
|
|
15436
|
-
);
|
|
15437
|
-
lines.push(
|
|
15438
|
-
"| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |"
|
|
15439
|
-
);
|
|
15440
|
-
for (const row of report.cases) {
|
|
15441
|
-
const fix = row.fix;
|
|
15442
|
-
const fixCell = !fix ? "\u2014" : fix.sampledOut ? "sampled-out" : fix.llmError ? "llm-failed" : fix.armBError ? "armB-error" : fix.attempts ? fix.flippedAtAttempt !== null ? `flip@${fix.flippedAtAttempt}` : `exhausted(${fix.attempts.length})` : "generated";
|
|
15443
|
-
lines.push(
|
|
15444
|
-
[
|
|
15445
|
-
row.corpus,
|
|
15446
|
-
row.trajId.length > 48 ? `${row.trajId.slice(0, 45)}\u2026` : row.trajId,
|
|
15447
|
-
row.k,
|
|
15448
|
-
row.recordedReturncodeAtK ?? "null",
|
|
15449
|
-
row.prefixExecuted ?? "\u2014",
|
|
15450
|
-
row.prefixDivergences ?? "\u2014",
|
|
15451
|
-
row.prefixDivergencePct ?? "\u2014",
|
|
15452
|
-
row.status === "ok" ? row.armAExit : row.status,
|
|
15453
|
-
row.armAReturncodeMatch ? "yes" : "no",
|
|
15454
|
-
row.armASignatureMatch ? "yes" : "no",
|
|
15455
|
-
row.replayed ? "**yes**" : "no",
|
|
15456
|
-
fixCell,
|
|
15457
|
-
fix?.armBExit ?? "\u2014",
|
|
15458
|
-
fix?.failureVanished === null || fix === null ? "\u2014" : fix.failureVanished ? "**yes**" : "no",
|
|
15459
|
-
(row.wallMs / 1e3).toFixed(1)
|
|
15460
|
-
].join(" | ")
|
|
15461
|
-
);
|
|
15462
|
-
}
|
|
15463
|
-
if (report.llm) {
|
|
15464
|
-
lines.push("");
|
|
15465
|
-
lines.push("## LLM fix generation");
|
|
15466
|
-
lines.push("");
|
|
15467
|
-
lines.push(
|
|
15468
|
-
`Model ${report.llm.model}: ${report.llm.calls} calls (${report.llm.failures} failed), ${report.llm.promptTokens} prompt + ${report.llm.completionTokens} completion tokens.`
|
|
15469
|
-
);
|
|
15470
|
-
}
|
|
15471
|
-
lines.push("");
|
|
15472
|
-
return lines.join("\n");
|
|
15473
|
-
}
|
|
15474
|
-
function parseReplayBatchArgs(argv) {
|
|
15475
|
-
const corpora = [];
|
|
15476
|
-
const values = /* @__PURE__ */ new Map();
|
|
15477
|
-
let enumerateOnly = false;
|
|
15478
|
-
for (let i = 0; i < argv.length; i++) {
|
|
15479
|
-
const arg = argv[i];
|
|
15480
|
-
if (arg === "--help" || arg === "-h") return "help";
|
|
15481
|
-
if (arg === "--enumerate-only") {
|
|
15482
|
-
enumerateOnly = true;
|
|
15483
|
-
continue;
|
|
15484
|
-
}
|
|
15485
|
-
if (!arg.startsWith("--")) throw new Error(`unexpected positional argument: ${arg}`);
|
|
15486
|
-
const value = argv[++i];
|
|
15487
|
-
if (value === void 0) throw new Error(`missing value for ${arg}`);
|
|
15488
|
-
if (arg === "--corpus") corpora.push(parseCorpusFlag(value));
|
|
15489
|
-
else values.set(arg, value);
|
|
15490
|
-
}
|
|
15491
|
-
if (corpora.length === 0) {
|
|
15492
|
-
throw new Error("replay-verify-batch: at least one --corpus name=<labels>::<prepared> is required");
|
|
15493
|
-
}
|
|
15494
|
-
const out = values.get("--out");
|
|
15495
|
-
if (!out && !enumerateOnly) throw new Error("replay-verify-batch: --out is required");
|
|
15496
|
-
const optionalNumber = (flag) => {
|
|
15497
|
-
const v = values.get(flag);
|
|
15498
|
-
if (v === void 0) return void 0;
|
|
15499
|
-
const n = Number(v);
|
|
15500
|
-
if (!Number.isFinite(n)) throw new Error(`replay-verify-batch: ${flag} must be a number, got ${v}`);
|
|
15501
|
-
return n;
|
|
15502
|
-
};
|
|
15503
|
-
const fix = values.get("--fix") ?? "none";
|
|
15504
|
-
if (fix !== "none" && fix !== "generate" && fix !== "loop") {
|
|
15505
|
-
throw new Error(`replay-verify-batch: --fix must be none, generate, or loop, got ${fix}`);
|
|
15506
|
-
}
|
|
15507
|
-
const fixAttempts = optionalNumber("--fix-attempts") ?? 3;
|
|
15508
|
-
if (!Number.isInteger(fixAttempts) || fixAttempts < 1) {
|
|
15509
|
-
throw new Error(
|
|
15510
|
-
`replay-verify-batch: --fix-attempts must be a positive integer, got ${values.get("--fix-attempts")}`
|
|
15511
|
-
);
|
|
15512
|
-
}
|
|
15513
|
-
return {
|
|
15514
|
-
corpora,
|
|
15515
|
-
out: out ?? ".",
|
|
15516
|
-
fix,
|
|
15517
|
-
fixAttempts,
|
|
15518
|
-
fixModel: values.get("--fix-model") ?? "glm-5.2",
|
|
15519
|
-
fixBaseUrl: values.get("--fix-base-url") ?? "https://api.z.ai/api/coding/paas/v4",
|
|
15520
|
-
fixApiKeyEnv: values.get("--fix-api-key-env") ?? "ZAI_GLM_API_KEY",
|
|
15521
|
-
maxFixCases: optionalNumber("--max-fix-cases") ?? 30,
|
|
15522
|
-
seed: optionalNumber("--seed") ?? 17,
|
|
15523
|
-
stepTimeoutMs: optionalNumber("--step-timeout"),
|
|
15524
|
-
prefixLimit: optionalNumber("--prefix-limit"),
|
|
15525
|
-
caseFilter: values.get("--case-filter"),
|
|
15526
|
-
caseLimit: optionalNumber("--case-limit"),
|
|
15527
|
-
baseUrl: values.get("--base-url") ?? process.env.SANDBOX_API_URL ?? "http://127.0.0.1:4097",
|
|
15528
|
-
apiKeyEnv: values.get("--api-key-env") ?? "SANDBOX_API_KEY",
|
|
15529
|
-
maxLifetimeSeconds: optionalNumber("--max-lifetime"),
|
|
15530
|
-
enumerateOnly
|
|
15531
|
-
};
|
|
15532
|
-
}
|
|
15533
|
-
function replayBatchUsage() {
|
|
15534
|
-
return `traces replay-verify-batch \u2014 measure replayability and fix-flip rates across gold-labeled corpora
|
|
15535
|
-
|
|
15536
|
-
Usage:
|
|
15537
|
-
traces replay-verify-batch \\
|
|
15538
|
-
--corpus NAME=<labels.json>::<prepared-dir> [--corpus ...] \\
|
|
15539
|
-
--out DIR [--fix none|generate|loop] [--fix-attempts 3] [--enumerate-only] \\
|
|
15540
|
-
[--fix-model glm-5.2] [--fix-base-url URL] [--fix-api-key-env ZAI_GLM_API_KEY] \\
|
|
15541
|
-
[--max-fix-cases 30] [--seed 17] [--step-timeout MS] [--prefix-limit N] \\
|
|
15542
|
-
[--case-filter SUBSTRING] [--case-limit N] \\
|
|
15543
|
-
[--base-url URL] [--api-key-env SANDBOX_API_KEY] [--max-lifetime SECONDS]
|
|
15544
|
-
|
|
15545
|
-
Replayable = the raw trajectory carries info.docker_config.base_image AND the
|
|
15546
|
-
labels mark at least one gold incorrect step that is a real mid-trajectory
|
|
15547
|
-
action. k = the first such gold step; golds whose action is the agent's
|
|
15548
|
-
submit command (COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT) are skipped and
|
|
15549
|
-
counted, and cases whose golds are all submit steps are excluded as
|
|
15550
|
-
gold-only-submit-step. Excluded cases and image pull failures are report
|
|
15551
|
-
rows, never skipped silently. --enumerate-only prints the enumeration
|
|
15552
|
-
without touching a sandbox.
|
|
15553
|
-
|
|
15554
|
-
--fix generate runs ONE LLM call per arm-A-reproduced case (cap
|
|
15555
|
-
--max-fix-cases, seeded sample beyond it) and executes the corrected command
|
|
15556
|
-
as arm B in a fresh sandbox.
|
|
15557
|
-
|
|
15558
|
-
--fix loop iterates: attempt 1 is the one-shot prompt; when an arm fails
|
|
15559
|
-
(nonzero exit or the failure signature persists) or the model call fails,
|
|
15560
|
-
the next prompt carries every prior command with its REAL executed
|
|
15561
|
-
stdout/stderr, up to --fix-attempts attempts (default 3). Retries may
|
|
15562
|
-
answer with a short script (<=5 commands) executed as one /bin/sh unit.
|
|
15563
|
-
Every attempt runs in its own fresh sandbox with the same replayed prefix.
|
|
15564
|
-
|
|
15565
|
-
Outputs batch-report.json, batch-report.md, cases.jsonl (incremental), and one
|
|
15566
|
-
directory per case with the full replay-verify artifacts.
|
|
15567
|
-
See docs/replay-verify.md for orchestrator setup and honest limits.
|
|
15568
|
-
`;
|
|
15569
|
-
}
|
|
15570
|
-
async function cmdReplayVerifyBatch(argv) {
|
|
15571
|
-
const parsed = parseReplayBatchArgs(argv);
|
|
15572
|
-
if (parsed === "help") {
|
|
15573
|
-
process.stdout.write(replayBatchUsage());
|
|
15574
|
-
return;
|
|
15575
|
-
}
|
|
15576
|
-
if (parsed.enumerateOnly) {
|
|
15577
|
-
const enumeration = enumerateReplayableCases(parsed.corpora);
|
|
15578
|
-
process.stdout.write(
|
|
15579
|
-
`${JSON.stringify(
|
|
15580
|
-
{
|
|
15581
|
-
labelEntries: enumeration.labelEntryCount,
|
|
15582
|
-
replayable: enumeration.replayable.map((c) => ({
|
|
15583
|
-
corpus: c.corpus,
|
|
15584
|
-
trajId: c.trajId,
|
|
15585
|
-
image: c.image,
|
|
15586
|
-
cwd: c.cwd,
|
|
15587
|
-
cwdSource: c.cwdSource,
|
|
15588
|
-
k: c.k,
|
|
15589
|
-
submitGoldsSkipped: c.submitGoldsSkipped,
|
|
15590
|
-
recordedReturncodeAtK: c.recordedReturncodeAtK,
|
|
15591
|
-
goldIncorrectSteps: c.goldIncorrectSteps
|
|
15592
|
-
})),
|
|
15593
|
-
excluded: enumeration.excluded
|
|
15594
|
-
},
|
|
15595
|
-
null,
|
|
15596
|
-
2
|
|
15597
|
-
)}
|
|
15598
|
-
`
|
|
15599
|
-
);
|
|
15600
|
-
return;
|
|
15601
|
-
}
|
|
15602
|
-
const apiKey = process.env[parsed.apiKeyEnv];
|
|
15603
|
-
if (!apiKey) {
|
|
15604
|
-
throw new Error(
|
|
15605
|
-
`replay-verify-batch: env var ${parsed.apiKeyEnv} is empty \u2014 pass --api-key-env or export it`
|
|
15606
|
-
);
|
|
15607
|
-
}
|
|
15608
|
-
let fixCaller;
|
|
15609
|
-
if (parsed.fix !== "none") {
|
|
15610
|
-
const fixApiKey = process.env[parsed.fixApiKeyEnv];
|
|
15611
|
-
if (!fixApiKey) {
|
|
15612
|
-
throw new Error(
|
|
15613
|
-
`replay-verify-batch: env var ${parsed.fixApiKeyEnv} is empty \u2014 pass --fix-api-key-env or export it`
|
|
15614
|
-
);
|
|
15615
|
-
}
|
|
15616
|
-
fixCaller = zaiChatCaller({
|
|
15617
|
-
baseUrl: parsed.fixBaseUrl,
|
|
15618
|
-
apiKey: fixApiKey,
|
|
15619
|
-
model: parsed.fixModel
|
|
15620
|
-
});
|
|
15621
|
-
}
|
|
15622
|
-
const report = await runReplayBatch({
|
|
15623
|
-
corpora: parsed.corpora,
|
|
15624
|
-
out: parsed.out,
|
|
15625
|
-
baseUrl: parsed.baseUrl,
|
|
15626
|
-
apiKey,
|
|
15627
|
-
fix: parsed.fix,
|
|
15628
|
-
fixAttempts: parsed.fixAttempts,
|
|
15629
|
-
fixCaller,
|
|
15630
|
-
fixModelLabel: parsed.fixModel,
|
|
15631
|
-
maxFixCases: parsed.maxFixCases,
|
|
15632
|
-
seed: parsed.seed,
|
|
15633
|
-
stepTimeoutMs: parsed.stepTimeoutMs,
|
|
15634
|
-
prefixLimit: parsed.prefixLimit,
|
|
15635
|
-
caseFilter: parsed.caseFilter,
|
|
15636
|
-
caseLimit: parsed.caseLimit,
|
|
15637
|
-
maxLifetimeSeconds: parsed.maxLifetimeSeconds,
|
|
15638
|
-
onProgress: (message) => process.stderr.write(`${message}
|
|
15639
|
-
`)
|
|
15640
|
-
});
|
|
15641
|
-
const { replayabilityRate, fixFlipRate } = report.headline;
|
|
15642
|
-
process.stderr.write(
|
|
15643
|
-
`replay-verify-batch: replayability ${replayabilityRate.numerator}/${replayabilityRate.denominator}` + (fixFlipRate ? `, fix-flip ${fixFlipRate.numerator}/${fixFlipRate.denominator}` : "") + ` \u2192 ${parsed.out}
|
|
15644
|
-
`
|
|
15645
|
-
);
|
|
15646
|
-
}
|
|
15647
|
-
|
|
15648
|
-
// src/replay-wire.ts
|
|
15649
|
-
var SUBJECT_PATTERN = /^incorrect-steps-(\d+)-(\d+)-(escaped|unescaped)-consequence-(\d+)$/;
|
|
15650
|
-
function parseIncorrectStepsSubject(subject) {
|
|
15651
|
-
const match = SUBJECT_PATTERN.exec(subject);
|
|
15652
|
-
if (!match) return null;
|
|
15653
|
-
return {
|
|
15654
|
-
firstStep: Number(match[1]),
|
|
15655
|
-
lastStep: Number(match[2]),
|
|
15656
|
-
escapeStatus: match[3],
|
|
15657
|
-
consequenceStep: Number(match[4])
|
|
15658
|
-
};
|
|
15659
|
-
}
|
|
15660
|
-
function resolveFindingInvocation(finding2, corpora) {
|
|
15661
|
-
const subject = parseIncorrectStepsSubject(finding2.subject);
|
|
15662
|
-
if (!subject) {
|
|
15663
|
-
throw new Error(
|
|
15664
|
-
`replay-wire: subject '${finding2.subject}' is not incorrect-steps-<first>-<last>-<escaped|unescaped>-consequence-<step>`
|
|
15665
|
-
);
|
|
15666
|
-
}
|
|
15667
|
-
const failures = [];
|
|
15668
|
-
for (const corpus of corpora) {
|
|
15669
|
-
const resolution = resolveCaseResources(corpus, finding2.trajId);
|
|
15670
|
-
if (resolution.resolved) {
|
|
15671
|
-
const at = subject.firstStep;
|
|
15672
|
-
const step = resolution.resources.steps.find((s) => s.step_id === at);
|
|
15673
|
-
if (!step) {
|
|
15674
|
-
throw new Error(
|
|
15675
|
-
`replay-wire: finding step ${at} is outside ${finding2.trajId} (${resolution.resources.steps.length} steps)`
|
|
15676
|
-
);
|
|
15677
|
-
}
|
|
15678
|
-
return { resources: resolution.resources, subject, at };
|
|
15679
|
-
}
|
|
15680
|
-
failures.push(`${corpus.name}: ${resolution.reason}`);
|
|
15681
|
-
}
|
|
15682
|
-
throw new Error(
|
|
15683
|
-
`replay-wire: trajectory ${finding2.trajId} is not replayable in any corpus \u2014 ${failures.join("; ")}`
|
|
15684
|
-
);
|
|
15685
|
-
}
|
|
15686
|
-
async function replayVerifyFinding(finding2, options) {
|
|
15687
|
-
if (options.fixCaller && options.fixCommand !== void 0) {
|
|
15688
|
-
throw new Error("replay-wire: pass fixCaller or fixCommand, not both");
|
|
15689
|
-
}
|
|
15690
|
-
const invocation = resolveFindingInvocation(finding2, options.corpora);
|
|
15691
|
-
let fixCommand = options.fixCommand ?? null;
|
|
15692
|
-
if (options.fixCaller) {
|
|
15693
|
-
const generated = await generateFixCommand(options.fixCaller, {
|
|
15694
|
-
taskStatement: invocation.resources.taskStatement,
|
|
15695
|
-
steps: invocation.resources.steps,
|
|
15696
|
-
k: invocation.at
|
|
15697
|
-
});
|
|
15698
|
-
if (!generated.succeeded) {
|
|
15699
|
-
throw new Error(`replay-wire: fix generation failed \u2014 ${generated.error}`);
|
|
15700
|
-
}
|
|
15701
|
-
fixCommand = generated.value.command;
|
|
15702
|
-
}
|
|
15703
|
-
const verdict = await replayVerify({
|
|
15704
|
-
stepsPath: invocation.resources.stepsPath,
|
|
15705
|
-
image: invocation.resources.image,
|
|
15706
|
-
at: invocation.at,
|
|
15707
|
-
fixCommand: fixCommand ?? void 0,
|
|
15708
|
-
cwd: invocation.resources.cwd,
|
|
15709
|
-
out: options.out,
|
|
15710
|
-
caseId: invocation.resources.trajId,
|
|
15711
|
-
stepTimeoutMs: options.stepTimeoutMs ?? invocation.resources.recordedStepTimeoutMs ?? void 0,
|
|
15712
|
-
prefixLimit: options.prefixLimit,
|
|
15713
|
-
backend: options.backend,
|
|
15714
|
-
apiKey: options.apiKey,
|
|
15715
|
-
baseUrl: options.baseUrl,
|
|
15716
|
-
onProgress: options.onProgress
|
|
15717
|
-
});
|
|
15718
|
-
return { invocation, fixCommand, verdict };
|
|
15719
|
-
}
|
|
15720
|
-
|
|
15721
|
-
// src/analyze-verify.ts
|
|
15722
|
-
import { mkdirSync as mkdirSync3, readFileSync as readFileSync5, writeFileSync as writeFileSync3 } from "fs";
|
|
15723
|
-
import { join as join27 } from "path";
|
|
15724
|
-
var SINGLE_STEP_SUBJECT = /^incorrect-step-(\d+)$/;
|
|
15725
|
-
function findingReplayStep(finding2) {
|
|
15726
|
-
const fromMetadata = finding2.metadata?.block_first_step;
|
|
15727
|
-
if (typeof fromMetadata === "number" && Number.isInteger(fromMetadata) && fromMetadata >= 1) {
|
|
15728
|
-
return fromMetadata;
|
|
15729
|
-
}
|
|
15730
|
-
const subject = finding2.subject ?? "";
|
|
15731
|
-
const wire = parseIncorrectStepsSubject(subject);
|
|
15732
|
-
if (wire) return wire.firstStep;
|
|
15733
|
-
const single = SINGLE_STEP_SUBJECT.exec(subject);
|
|
15734
|
-
if (single) return Number(single[1]);
|
|
15735
|
-
return null;
|
|
15736
|
-
}
|
|
15737
|
-
var TRACE_EVIDENCE_URI = /^trace:\/\/([^/]+)\//;
|
|
15738
|
-
function findingTrajectoryId(finding2) {
|
|
15739
|
-
for (const ref of finding2.evidence_refs ?? []) {
|
|
15740
|
-
if (typeof ref.uri !== "string") continue;
|
|
15741
|
-
const match = TRACE_EVIDENCE_URI.exec(ref.uri);
|
|
15742
|
-
if (match) return match[1];
|
|
15743
|
-
}
|
|
15744
|
-
return null;
|
|
15745
|
-
}
|
|
15746
|
-
function checkStep(steps, at) {
|
|
15747
|
-
const step = steps.find((s) => s.step_id === at);
|
|
15748
|
-
if (!step) {
|
|
15749
|
-
return { ok: false, reason: `step ${at} is outside the trajectory (${steps.length} steps)` };
|
|
15750
|
-
}
|
|
15751
|
-
if (isSubmitAction(step.action)) {
|
|
15752
|
-
return {
|
|
15753
|
-
ok: false,
|
|
15754
|
-
reason: `step ${at} is the submit action \u2014 a submit decision has no executable failure to replay`
|
|
15755
|
-
};
|
|
15756
|
-
}
|
|
15757
|
-
const recordedReturncode = parseRecordedReturncode(step.observation);
|
|
15758
|
-
if (recordedReturncode === null) {
|
|
15759
|
-
return {
|
|
15760
|
-
ok: false,
|
|
15761
|
-
reason: `step ${at} recorded no returncode \u2014 there is no executable failure signature to reproduce`
|
|
15762
|
-
};
|
|
15763
|
-
}
|
|
15764
|
-
return { ok: true, recordedReturncode };
|
|
15765
|
-
}
|
|
15766
|
-
function resolveFindingReplayability(finding2, source) {
|
|
15767
|
-
const at = findingReplayStep(finding2);
|
|
15768
|
-
if (at === null) {
|
|
15769
|
-
return {
|
|
15770
|
-
replayable: false,
|
|
15771
|
-
reason: `subject '${finding2.subject ?? "(none)"}' names no trajectory step (expected incorrect-step-<n>, incorrect-steps-<f>-<l>-\u2026, or metadata.block_first_step)`
|
|
15772
|
-
};
|
|
15773
|
-
}
|
|
15774
|
-
if (source.kind === "direct") {
|
|
15775
|
-
const trajId2 = findingTrajectoryId(finding2);
|
|
15776
|
-
if (trajId2 && source.caseId && trajId2 !== source.caseId) {
|
|
15777
|
-
return {
|
|
15778
|
-
replayable: false,
|
|
15779
|
-
reason: `finding cites trajectory '${trajId2}' but the supplied steps are case '${source.caseId}'`
|
|
15780
|
-
};
|
|
15781
|
-
}
|
|
15782
|
-
let steps;
|
|
15783
|
-
try {
|
|
15784
|
-
steps = JSON.parse(readFileSync5(source.stepsPath, "utf8"));
|
|
15785
|
-
} catch (err) {
|
|
15786
|
-
throw new Error(
|
|
15787
|
-
`verify-findings: cannot read steps file ${source.stepsPath} \u2014 ${err instanceof Error ? err.message : String(err)}`
|
|
15788
|
-
);
|
|
15789
|
-
}
|
|
15790
|
-
if (!Array.isArray(steps) || steps.length === 0) {
|
|
15791
|
-
throw new Error(`verify-findings: ${source.stepsPath} is not a non-empty steps array`);
|
|
15792
|
-
}
|
|
15793
|
-
const step = checkStep(steps, at);
|
|
15794
|
-
if (!step.ok) return { replayable: false, reason: step.reason };
|
|
15795
|
-
return {
|
|
15796
|
-
replayable: true,
|
|
15797
|
-
resolved: {
|
|
15798
|
-
caseId: source.caseId ?? trajId2 ?? source.stepsPath,
|
|
15799
|
-
stepsPath: source.stepsPath,
|
|
15800
|
-
image: source.image,
|
|
15801
|
-
cwd: source.cwd,
|
|
15802
|
-
at,
|
|
15803
|
-
recordedReturncode: step.recordedReturncode,
|
|
15804
|
-
recordedStepTimeoutMs: null
|
|
15805
|
-
}
|
|
15806
|
-
};
|
|
15807
|
-
}
|
|
15808
|
-
const trajId = findingTrajectoryId(finding2);
|
|
15809
|
-
if (!trajId) {
|
|
15810
|
-
return {
|
|
15811
|
-
replayable: false,
|
|
15812
|
-
reason: "finding carries no trace://<trajectory>/ evidence ref naming its trajectory"
|
|
15813
|
-
};
|
|
15814
|
-
}
|
|
15815
|
-
const failures = [];
|
|
15816
|
-
for (const corpus of source.corpora) {
|
|
15817
|
-
const resolution = resolveCaseResources(corpus, trajId);
|
|
15818
|
-
if (!resolution.resolved) {
|
|
15819
|
-
failures.push(`${corpus.name}: ${resolution.reason}${resolution.detail ? ` (${resolution.detail})` : ""}`);
|
|
15820
|
-
continue;
|
|
15821
|
-
}
|
|
15822
|
-
const step = checkStep(resolution.resources.steps, at);
|
|
15823
|
-
if (!step.ok) return { replayable: false, reason: step.reason };
|
|
15824
|
-
return {
|
|
15825
|
-
replayable: true,
|
|
15826
|
-
resolved: {
|
|
15827
|
-
caseId: trajId,
|
|
15828
|
-
stepsPath: resolution.resources.stepsPath,
|
|
15829
|
-
image: resolution.resources.image,
|
|
15830
|
-
cwd: resolution.resources.cwd,
|
|
15831
|
-
at,
|
|
15832
|
-
recordedReturncode: step.recordedReturncode,
|
|
15833
|
-
recordedStepTimeoutMs: resolution.resources.recordedStepTimeoutMs
|
|
15834
|
-
}
|
|
15835
|
-
};
|
|
15836
|
-
}
|
|
15837
|
-
return {
|
|
15838
|
-
replayable: false,
|
|
15839
|
-
reason: `trajectory ${trajId} is not replayable in any corpus \u2014 ${failures.join("; ")}`
|
|
15840
|
-
};
|
|
15841
|
-
}
|
|
15842
|
-
function classifyVerdict(verdict) {
|
|
15843
|
-
if (!verdict.armA.failureSignatureMatch) return "divergent";
|
|
15844
|
-
if (verdict.armB?.failureVanished) return "fix-flipped";
|
|
15845
|
-
return "reproduced";
|
|
15846
|
-
}
|
|
15847
|
-
var DEFAULT_SANDBOX_BASE_URL = "http://127.0.0.1:4097";
|
|
15848
|
-
async function assertSandboxReachable(baseUrl) {
|
|
15849
|
-
let response;
|
|
15850
|
-
try {
|
|
15851
|
-
response = await fetch(new URL("/health", baseUrl), { signal: AbortSignal.timeout(5e3) });
|
|
15852
|
-
} catch (err) {
|
|
15853
|
-
throw new Error(
|
|
15854
|
-
`verify-findings: sandbox API unreachable at ${baseUrl} \u2014 ${err instanceof Error ? err.message : String(err)}. Executing findings requires a running sandbox orchestrator + SDK adapter (see docs/replay-verify.md); start them or pass --base-url.`
|
|
15855
|
-
);
|
|
15856
|
-
}
|
|
15857
|
-
if (!response.ok) {
|
|
15858
|
-
throw new Error(
|
|
15859
|
-
`verify-findings: sandbox API at ${baseUrl} answered /health with HTTP ${response.status} \u2014 refusing to run proofs against degraded infrastructure`
|
|
15860
|
-
);
|
|
15861
|
-
}
|
|
15862
|
-
}
|
|
15863
|
-
function receiptExecution(verdict) {
|
|
15864
|
-
return {
|
|
15865
|
-
image: verdict.image,
|
|
15866
|
-
cwd: verdict.cwd,
|
|
15867
|
-
recordedReturncode: verdict.recordedReturncode,
|
|
15868
|
-
signature: verdict.signature,
|
|
15869
|
-
signatureBasis: verdict.signatureBasis,
|
|
15870
|
-
armA: {
|
|
15871
|
-
command: verdict.armA.command,
|
|
15872
|
-
exitCode: verdict.armA.exitCode,
|
|
15873
|
-
wallMs: verdict.armA.wallMs,
|
|
15874
|
-
failureSignatureMatch: verdict.armA.failureSignatureMatch
|
|
15875
|
-
},
|
|
15876
|
-
armB: verdict.armB ? {
|
|
15877
|
-
command: verdict.armB.command,
|
|
15878
|
-
exitCode: verdict.armB.exitCode,
|
|
15879
|
-
wallMs: verdict.armB.wallMs,
|
|
15880
|
-
failureVanished: verdict.armB.failureVanished
|
|
15881
|
-
} : null,
|
|
15882
|
-
prefixExecuted: verdict.prefixExecuted,
|
|
15883
|
-
prefixDivergences: verdict.prefixDivergences.length,
|
|
15884
|
-
totalMs: verdict.timings.totalMs
|
|
15885
|
-
};
|
|
15886
|
-
}
|
|
15887
|
-
function writeReceipt(receiptDir, finding2, verification, execution) {
|
|
15888
|
-
const receipt = {
|
|
15889
|
-
schema_version: "1.0.0",
|
|
15890
|
-
produced_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
15891
|
-
finding_id: verification.finding_id,
|
|
15892
|
-
analyst_id: finding2.analyst_id ?? null,
|
|
15893
|
-
subject: verification.subject,
|
|
15894
|
-
claim: typeof finding2.claim === "string" ? finding2.claim.slice(0, 600) : null,
|
|
15895
|
-
trajectory_id: verification.trajectory_id,
|
|
15896
|
-
step: verification.step,
|
|
15897
|
-
verified: verification.verified,
|
|
15898
|
-
reason: verification.reason,
|
|
15899
|
-
execution,
|
|
15900
|
-
verdict_path: verification.verdict_path,
|
|
15901
|
-
deduplicated_with: verification.deduplicated_with
|
|
15902
|
-
};
|
|
15903
|
-
writeFileSync3(join27(receiptDir, "receipt.json"), `${JSON.stringify(receipt, null, 2)}
|
|
15904
|
-
`);
|
|
15905
|
-
}
|
|
15906
|
-
function receiptDirName(index, finding2) {
|
|
15907
|
-
const id = typeof finding2.finding_id === "string" && finding2.finding_id.length > 0 ? finding2.finding_id.replace(/[^A-Za-z0-9_-]/g, "_") : "finding";
|
|
15908
|
-
return `${String(index + 1).padStart(3, "0")}-${id}`;
|
|
15909
|
-
}
|
|
15910
|
-
async function verifyFindings(findings, options) {
|
|
15911
|
-
if (findings.length === 0) throw new Error("verify-findings: no findings to verify");
|
|
15912
|
-
mkdirSync3(options.out, { recursive: true });
|
|
15913
|
-
const resolutions = findings.map((finding2) => resolveFindingReplayability(finding2, options.source));
|
|
15914
|
-
const baseUrl = options.baseUrl ?? DEFAULT_SANDBOX_BASE_URL;
|
|
15915
|
-
if (resolutions.some((resolution) => resolution.replayable) && !options.backend) {
|
|
15916
|
-
if (!options.apiKey) {
|
|
15917
|
-
throw new Error(
|
|
15918
|
-
"verify-findings: replayable findings need a sandbox API key \u2014 export SANDBOX_API_KEY or pass --api-key-env"
|
|
15919
|
-
);
|
|
15920
|
-
}
|
|
15921
|
-
await assertSandboxReachable(baseUrl);
|
|
15922
|
-
}
|
|
15923
|
-
const preparer = options.source.kind === "corpus" && !options.backend ? options.source.preparer ?? dockerImagePreparer() : null;
|
|
15924
|
-
const preparedImages = /* @__PURE__ */ new Map();
|
|
15925
|
-
const executedByKey = /* @__PURE__ */ new Map();
|
|
15926
|
-
const verifications = [];
|
|
15927
|
-
let executions = 0;
|
|
15928
|
-
for (let index = 0; index < findings.length; index++) {
|
|
15929
|
-
const finding2 = findings[index];
|
|
15930
|
-
const resolution = resolutions[index];
|
|
15931
|
-
const receiptDir = join27(options.out, receiptDirName(index, finding2));
|
|
15932
|
-
mkdirSync3(receiptDir, { recursive: true });
|
|
15933
|
-
const identity2 = {
|
|
15934
|
-
finding_id: finding2.finding_id ?? null,
|
|
15935
|
-
subject: finding2.subject ?? null,
|
|
15936
|
-
trajectory_id: findingTrajectoryId(finding2)
|
|
15937
|
-
};
|
|
15938
|
-
if (!resolution.replayable) {
|
|
15939
|
-
const verification2 = {
|
|
15940
|
-
...identity2,
|
|
15941
|
-
step: findingReplayStep(finding2),
|
|
15942
|
-
verified: "not-replayable",
|
|
15943
|
-
reason: resolution.reason,
|
|
15944
|
-
receipt: receiptDir,
|
|
15945
|
-
verdict_path: null,
|
|
15946
|
-
deduplicated_with: null
|
|
15947
|
-
};
|
|
15948
|
-
writeReceipt(receiptDir, finding2, verification2, null);
|
|
15949
|
-
verifications.push(verification2);
|
|
15950
|
-
options.onProgress?.(`${identity2.finding_id ?? `finding ${index + 1}`}: not-replayable \u2014 ${resolution.reason}`);
|
|
15951
|
-
continue;
|
|
15952
|
-
}
|
|
15953
|
-
const resolved = resolution.resolved;
|
|
15954
|
-
const dedupeKey = `${resolved.caseId}::${resolved.at}::${options.fixCommand ?? ""}`;
|
|
15955
|
-
const prior = executedByKey.get(dedupeKey);
|
|
15956
|
-
if (prior) {
|
|
15957
|
-
const verification2 = {
|
|
15958
|
-
...identity2,
|
|
15959
|
-
step: resolved.at,
|
|
15960
|
-
verified: prior.verified,
|
|
15961
|
-
reason: prior.reason,
|
|
15962
|
-
receipt: receiptDir,
|
|
15963
|
-
verdict_path: prior.verdict_path,
|
|
15964
|
-
deduplicated_with: prior.receipt
|
|
15965
|
-
};
|
|
15966
|
-
const priorVerdict = prior.verdict_path ? JSON.parse(readFileSync5(prior.verdict_path, "utf8")) : null;
|
|
15967
|
-
writeReceipt(receiptDir, finding2, verification2, priorVerdict ? receiptExecution(priorVerdict) : null);
|
|
15968
|
-
verifications.push(verification2);
|
|
15969
|
-
options.onProgress?.(
|
|
15970
|
-
`${identity2.finding_id ?? `finding ${index + 1}`}: ${prior.verified} (shares proof with ${prior.finding_id ?? prior.receipt})`
|
|
15971
|
-
);
|
|
15972
|
-
continue;
|
|
15973
|
-
}
|
|
15974
|
-
let image = resolved.image;
|
|
15975
|
-
if (preparer) {
|
|
15976
|
-
const preparationKey = `${resolved.image}::${resolved.cwd}`;
|
|
15977
|
-
let preparation = preparedImages.get(preparationKey);
|
|
15978
|
-
if (!preparation) {
|
|
15979
|
-
const ensured = await preparer.ensure(resolved.image, resolved.cwd);
|
|
15980
|
-
preparation = ensured.succeeded ? { succeeded: true, image: ensured.value.derivedImage } : { succeeded: false, error: ensured.error };
|
|
15981
|
-
preparedImages.set(preparationKey, preparation);
|
|
15982
|
-
}
|
|
15983
|
-
if (!preparation.succeeded) {
|
|
15984
|
-
const verification2 = {
|
|
15985
|
-
...identity2,
|
|
15986
|
-
step: resolved.at,
|
|
15987
|
-
verified: "not-replayable",
|
|
15988
|
-
reason: `replay image could not be prepared \u2014 ${preparation.error}`,
|
|
15989
|
-
receipt: receiptDir,
|
|
15990
|
-
verdict_path: null,
|
|
15991
|
-
deduplicated_with: null
|
|
15992
|
-
};
|
|
15993
|
-
writeReceipt(receiptDir, finding2, verification2, null);
|
|
15994
|
-
verifications.push(verification2);
|
|
15995
|
-
options.onProgress?.(`${identity2.finding_id ?? `finding ${index + 1}`}: not-replayable \u2014 image preparation failed`);
|
|
15996
|
-
continue;
|
|
15997
|
-
}
|
|
15998
|
-
image = preparation.image;
|
|
15999
|
-
}
|
|
16000
|
-
options.onProgress?.(
|
|
16001
|
-
`${identity2.finding_id ?? `finding ${index + 1}`}: executing arm A at step ${resolved.at} of ${resolved.caseId} on ${image}`
|
|
16002
|
-
);
|
|
16003
|
-
const verdict = await replayVerify({
|
|
16004
|
-
stepsPath: resolved.stepsPath,
|
|
16005
|
-
image,
|
|
16006
|
-
at: resolved.at,
|
|
16007
|
-
fixCommand: options.fixCommand,
|
|
16008
|
-
cwd: resolved.cwd,
|
|
16009
|
-
out: receiptDir,
|
|
16010
|
-
caseId: resolved.caseId,
|
|
16011
|
-
stepTimeoutMs: options.stepTimeoutMs ?? resolved.recordedStepTimeoutMs ?? void 0,
|
|
16012
|
-
prefixLimit: options.prefixLimit,
|
|
16013
|
-
backend: options.backend,
|
|
16014
|
-
apiKey: options.apiKey,
|
|
16015
|
-
baseUrl,
|
|
16016
|
-
maxLifetimeSeconds: options.maxLifetimeSeconds,
|
|
16017
|
-
onProgress: options.onProgress
|
|
16018
|
-
});
|
|
16019
|
-
executions += 1;
|
|
16020
|
-
const verification = {
|
|
16021
|
-
...identity2,
|
|
16022
|
-
step: resolved.at,
|
|
16023
|
-
verified: classifyVerdict(verdict),
|
|
16024
|
-
reason: null,
|
|
16025
|
-
receipt: receiptDir,
|
|
16026
|
-
verdict_path: join27(receiptDir, "replay-verdict.json"),
|
|
16027
|
-
deduplicated_with: null
|
|
16028
|
-
};
|
|
16029
|
-
executedByKey.set(dedupeKey, verification);
|
|
16030
|
-
writeReceipt(receiptDir, finding2, verification, receiptExecution(verdict));
|
|
16031
|
-
verifications.push(verification);
|
|
16032
|
-
options.onProgress?.(`${identity2.finding_id ?? `finding ${index + 1}`}: ${verification.verified}`);
|
|
16033
|
-
}
|
|
16034
|
-
const counts = {
|
|
16035
|
-
reproduced: 0,
|
|
16036
|
-
"fix-flipped": 0,
|
|
16037
|
-
divergent: 0,
|
|
16038
|
-
"not-replayable": 0
|
|
16039
|
-
};
|
|
16040
|
-
for (const verification of verifications) counts[verification.verified] += 1;
|
|
16041
|
-
const run = { out: options.out, verifications, counts, executions };
|
|
16042
|
-
writeFileSync3(
|
|
16043
|
-
join27(options.out, "verifications.json"),
|
|
16044
|
-
`${JSON.stringify({ schema_version: "1.0.0", ...run }, null, 2)}
|
|
16045
|
-
`
|
|
16046
|
-
);
|
|
16047
|
-
return run;
|
|
16048
|
-
}
|
|
16049
|
-
function verdictCell(verification) {
|
|
16050
|
-
switch (verification.verified) {
|
|
16051
|
-
case "reproduced":
|
|
16052
|
-
return "**VERIFIED** \u2014 reproduced";
|
|
16053
|
-
case "fix-flipped":
|
|
16054
|
-
return "**VERIFIED** \u2014 fix-flipped";
|
|
16055
|
-
case "divergent":
|
|
16056
|
-
return "DIVERGENT \u2014 recorded failure did not reproduce";
|
|
16057
|
-
case "not-replayable":
|
|
16058
|
-
return `UNVERIFIABLE \u2014 ${verification.reason ?? "no reason recorded"}`;
|
|
16059
|
-
}
|
|
16060
|
-
}
|
|
16061
|
-
function renderVerifiedFindingsSection(run) {
|
|
16062
|
-
const lines = ["## Verified findings (executed replay)", ""];
|
|
16063
|
-
const total = run.verifications.length;
|
|
16064
|
-
lines.push(
|
|
16065
|
-
`${total} finding(s) \u2192 ${run.counts.reproduced} reproduced, ${run.counts["fix-flipped"]} fix-flipped, ${run.counts.divergent} divergent, ${run.counts["not-replayable"]} not replayable (${run.executions} sandbox execution(s); findings accusing the same step share one proof).`
|
|
16066
|
-
);
|
|
16067
|
-
lines.push("");
|
|
16068
|
-
lines.push("| Finding | Subject | Step | Verdict | Receipt |");
|
|
16069
|
-
lines.push("|---|---|---:|---|---|");
|
|
16070
|
-
for (const verification of run.verifications) {
|
|
16071
|
-
const shared = verification.deduplicated_with ? " (shared proof)" : "";
|
|
16072
|
-
lines.push(
|
|
16073
|
-
`| \`${verification.finding_id ?? "\u2014"}\` | \`${verification.subject ?? "\u2014"}\` | ${verification.step ?? "\u2014"} | ${verdictCell(verification)} | \`${verification.receipt}\`${shared} |`
|
|
16074
|
-
);
|
|
16075
|
-
}
|
|
16076
|
-
lines.push("");
|
|
16077
|
-
lines.push(
|
|
16078
|
-
"VERIFIED = the accused step was re-executed in a sandbox after replaying the trajectory prefix, and the recorded failure signature reproduced (fix-flipped: a corrected command additionally made it vanish). Each receipt directory carries receipt.json and, when executed, replay-verdict.json + report.md with real stdout/stderr."
|
|
16079
|
-
);
|
|
16080
|
-
lines.push("");
|
|
16081
|
-
return lines.join("\n");
|
|
16082
|
-
}
|
|
16083
|
-
function readFindingsFile(path) {
|
|
16084
|
-
const parsed = JSON.parse(readFileSync5(path, "utf8"));
|
|
16085
|
-
const array = Array.isArray(parsed) ? parsed : parsed && typeof parsed === "object" && Array.isArray(parsed.findings) ? parsed.findings : null;
|
|
16086
|
-
if (!array) {
|
|
16087
|
-
throw new Error(`${path} is neither a findings array nor an object with a findings array`);
|
|
16088
|
-
}
|
|
16089
|
-
for (const entry of array) {
|
|
16090
|
-
if (!entry || typeof entry !== "object") {
|
|
16091
|
-
throw new Error(`${path}: every finding must be an object, got ${JSON.stringify(entry)}`);
|
|
16092
|
-
}
|
|
16093
|
-
}
|
|
16094
|
-
return array;
|
|
16095
|
-
}
|
|
16096
|
-
function verifyFindingsUsage() {
|
|
16097
|
-
return `traces verify-findings \u2014 execute analyst findings as sandbox replay proofs
|
|
14171
|
+
var VERIFY_FINDINGS_USAGE = `traces verify-findings \u2014 execute analyst findings as sandbox replay proofs
|
|
16098
14172
|
|
|
16099
14173
|
Usage:
|
|
16100
14174
|
traces verify-findings --findings FINDINGS.json --out DIR \\
|
|
@@ -16111,7 +14185,7 @@ Usage:
|
|
|
16111
14185
|
verify against one trajectory; the image must be replay-ready
|
|
16112
14186
|
(uid-1000 derived), exactly like traces replay-verify
|
|
16113
14187
|
--case trajectory id of --steps; findings citing other trajectories
|
|
16114
|
-
are
|
|
14188
|
+
are marked not-replayable
|
|
16115
14189
|
--corpus resolve each finding's trajectory in CodeTraceBench corpora;
|
|
16116
14190
|
images are derived via the batch uid-1000 preparer (docker)
|
|
16117
14191
|
--fix-command corrected step for arm B; a reproduced finding whose fix
|
|
@@ -16124,110 +14198,83 @@ with a receipt directory carrying the executed evidence or the precise reason.
|
|
|
16124
14198
|
Requires a running sandbox orchestrator when any finding is replayable \u2014 infra
|
|
16125
14199
|
absence is an error, never a silent skip. See docs/replay-verify.md for setup.
|
|
16126
14200
|
`;
|
|
14201
|
+
function verifyFindingsInSandbox(findings, options) {
|
|
14202
|
+
if (findings.length === 0) throw new Error("verify-findings: no findings to verify");
|
|
14203
|
+
const { sandbox, ...rest } = options;
|
|
14204
|
+
return verifyFindings(findings, {
|
|
14205
|
+
...rest,
|
|
14206
|
+
backendFactory: sandboxBackendFactory(sandbox),
|
|
14207
|
+
preflight: async () => {
|
|
14208
|
+
if (!sandbox.apiKey) {
|
|
14209
|
+
throw new Error(
|
|
14210
|
+
"verify-findings: replayable findings need a sandbox API key \u2014 export SANDBOX_API_KEY or pass --api-key-env"
|
|
14211
|
+
);
|
|
14212
|
+
}
|
|
14213
|
+
await assertSandboxReachable(sandbox.baseUrl);
|
|
14214
|
+
},
|
|
14215
|
+
onProgress: progress
|
|
14216
|
+
});
|
|
16127
14217
|
}
|
|
16128
|
-
function
|
|
16129
|
-
const
|
|
16130
|
-
|
|
16131
|
-
|
|
16132
|
-
|
|
16133
|
-
if (arg === "--help" || arg === "-h") return "help";
|
|
16134
|
-
if (!arg.startsWith("--")) throw new Error(`unexpected positional argument: ${arg}`);
|
|
16135
|
-
const value = argv[++i];
|
|
16136
|
-
if (value === void 0) throw new Error(`missing value for ${arg}`);
|
|
16137
|
-
if (arg === "--corpus") {
|
|
16138
|
-
corpora.push(parseCorpusFlag(value));
|
|
16139
|
-
continue;
|
|
16140
|
-
}
|
|
16141
|
-
values.set(arg, value);
|
|
14218
|
+
async function cmdVerifyFindings(argv) {
|
|
14219
|
+
const parsed = parseFlags2("verify-findings", argv);
|
|
14220
|
+
if (parsed === "help") {
|
|
14221
|
+
process.stdout.write(VERIFY_FINDINGS_USAGE);
|
|
14222
|
+
return;
|
|
16142
14223
|
}
|
|
16143
|
-
const
|
|
16144
|
-
|
|
16145
|
-
|
|
16146
|
-
|
|
16147
|
-
};
|
|
16148
|
-
const optionalNumber = (flag) => {
|
|
16149
|
-
const v = values.get(flag);
|
|
16150
|
-
if (v === void 0) return void 0;
|
|
16151
|
-
const n = Number(v);
|
|
16152
|
-
if (!Number.isFinite(n)) throw new Error(`verify-findings: ${flag} must be a number, got ${v}`);
|
|
16153
|
-
return n;
|
|
16154
|
-
};
|
|
16155
|
-
const findingsPath = required("--findings");
|
|
16156
|
-
const out = required("--out");
|
|
14224
|
+
const { values, corpora } = parsed;
|
|
14225
|
+
const flags = flagReader("verify-findings", values);
|
|
14226
|
+
const findingsPath = flags.required("--findings");
|
|
14227
|
+
const out = flags.required("--out");
|
|
16157
14228
|
const direct = values.has("--steps") || values.has("--image") || values.has("--cwd");
|
|
16158
14229
|
if (direct && corpora.length > 0) {
|
|
16159
14230
|
throw new Error("verify-findings: pass --steps/--image/--cwd or --corpus, not both");
|
|
16160
14231
|
}
|
|
16161
14232
|
let source;
|
|
16162
14233
|
if (direct) {
|
|
14234
|
+
const caseId = values.get("--case");
|
|
16163
14235
|
source = {
|
|
16164
14236
|
kind: "direct",
|
|
16165
|
-
stepsPath: required("--steps"),
|
|
16166
|
-
image: required("--image"),
|
|
16167
|
-
cwd: required("--cwd"),
|
|
16168
|
-
...
|
|
14237
|
+
stepsPath: flags.required("--steps"),
|
|
14238
|
+
image: flags.required("--image"),
|
|
14239
|
+
cwd: flags.required("--cwd"),
|
|
14240
|
+
...caseId === void 0 ? {} : { caseId }
|
|
16169
14241
|
};
|
|
16170
14242
|
} else if (corpora.length > 0) {
|
|
16171
14243
|
source = { kind: "corpus", corpora };
|
|
16172
14244
|
} else {
|
|
16173
14245
|
throw new Error("verify-findings: a replay source is required \u2014 --steps/--image/--cwd or --corpus");
|
|
16174
14246
|
}
|
|
16175
|
-
|
|
16176
|
-
findingsPath,
|
|
14247
|
+
const run = await verifyFindingsInSandbox(readFindingsFile(findingsPath), {
|
|
16177
14248
|
source,
|
|
16178
14249
|
out,
|
|
14250
|
+
sandbox: sandboxOptions("verify-findings", values, false),
|
|
16179
14251
|
fixCommand: values.get("--fix-command"),
|
|
16180
|
-
stepTimeoutMs:
|
|
16181
|
-
prefixLimit:
|
|
16182
|
-
baseUrl: values.get("--base-url") ?? process.env.SANDBOX_API_URL ?? DEFAULT_SANDBOX_BASE_URL,
|
|
16183
|
-
apiKeyEnv: values.get("--api-key-env") ?? "SANDBOX_API_KEY",
|
|
16184
|
-
maxLifetimeSeconds: optionalNumber("--max-lifetime")
|
|
16185
|
-
};
|
|
16186
|
-
}
|
|
16187
|
-
async function cmdVerifyFindings(argv) {
|
|
16188
|
-
const parsed = parseVerifyFindingsArgs(argv);
|
|
16189
|
-
if (parsed === "help") {
|
|
16190
|
-
process.stdout.write(verifyFindingsUsage());
|
|
16191
|
-
return;
|
|
16192
|
-
}
|
|
16193
|
-
const findings = readFindingsFile(parsed.findingsPath);
|
|
16194
|
-
const apiKey = process.env[parsed.apiKeyEnv];
|
|
16195
|
-
const run = await verifyFindings(findings, {
|
|
16196
|
-
source: parsed.source,
|
|
16197
|
-
out: parsed.out,
|
|
16198
|
-
fixCommand: parsed.fixCommand,
|
|
16199
|
-
stepTimeoutMs: parsed.stepTimeoutMs,
|
|
16200
|
-
prefixLimit: parsed.prefixLimit,
|
|
16201
|
-
apiKey,
|
|
16202
|
-
baseUrl: parsed.baseUrl,
|
|
16203
|
-
maxLifetimeSeconds: parsed.maxLifetimeSeconds,
|
|
16204
|
-
onProgress: (message) => process.stderr.write(`${message}
|
|
16205
|
-
`)
|
|
14252
|
+
stepTimeoutMs: flags.number("--step-timeout"),
|
|
14253
|
+
prefixLimit: flags.number("--prefix-limit")
|
|
16206
14254
|
});
|
|
16207
14255
|
process.stdout.write(`${JSON.stringify(run, null, 2)}
|
|
16208
14256
|
`);
|
|
16209
|
-
|
|
16210
|
-
`verify-findings: ${run.verifications.length} finding(s) \u2192 ${run.counts.reproduced} reproduced, ${run.counts["fix-flipped"]} fix-flipped, ${run.counts.divergent} divergent, ${run.counts["not-replayable"]} not-replayable (${run.executions} execution(s)) \u2192 ${run.out}
|
|
16211
|
-
`
|
|
14257
|
+
progress(
|
|
14258
|
+
`verify-findings: ${run.verifications.length} finding(s) \u2192 ${run.counts.reproduced} reproduced, ${run.counts["fix-flipped"]} fix-flipped, ${run.counts.divergent} divergent, ${run.counts["not-replayable"]} not-replayable (${run.executions} execution(s)) \u2192 ${run.out}`
|
|
16212
14259
|
);
|
|
16213
14260
|
}
|
|
16214
14261
|
|
|
16215
14262
|
// src/hodoscope.ts
|
|
16216
|
-
import { createHash as
|
|
14263
|
+
import { createHash as createHash12 } from "crypto";
|
|
16217
14264
|
import { mkdir as mkdir5, mkdtemp as mkdtemp5, readFile as readFile11, readdir as readdir17, writeFile as writeFile7 } from "fs/promises";
|
|
16218
|
-
import { endianness, tmpdir as
|
|
16219
|
-
import { join as
|
|
14265
|
+
import { endianness, tmpdir as tmpdir5 } from "os";
|
|
14266
|
+
import { join as join24, resolve as resolve10 } from "path";
|
|
16220
14267
|
var HODOSCOPE_VERSION = "0.2.4";
|
|
16221
14268
|
async function writeHodoscopeInput(otlpPath, options = {}) {
|
|
16222
14269
|
const explicitDirectory = options.directory ? resolve10(options.directory) : void 0;
|
|
16223
|
-
const directory = explicitDirectory ?? await mkdtemp5(
|
|
14270
|
+
const directory = explicitDirectory ?? await mkdtemp5(join24(tmpdir5(), "traces-hodoscope-input-"));
|
|
16224
14271
|
if (explicitDirectory) {
|
|
16225
14272
|
await mkdir5(directory, { recursive: true });
|
|
16226
14273
|
if ((await readdir17(directory)).length > 0) {
|
|
16227
14274
|
throw new Error(`Hodoscope explicit directory must be empty: ${directory}`);
|
|
16228
14275
|
}
|
|
16229
14276
|
}
|
|
16230
|
-
const samples =
|
|
14277
|
+
const samples = join24(directory, "samples");
|
|
16231
14278
|
await mkdir5(samples);
|
|
16232
14279
|
const rows = [];
|
|
16233
14280
|
for await (const value of readJsonl(otlpPath)) rows.push(parseOpenInferenceRow(value));
|
|
@@ -16252,8 +14299,8 @@ async function writeHodoscopeInput(otlpPath, options = {}) {
|
|
|
16252
14299
|
throw new Error(`Hodoscope trace '${traceId}' contains no assistant actions`);
|
|
16253
14300
|
}
|
|
16254
14301
|
actionCount += actions;
|
|
16255
|
-
const file = `${
|
|
16256
|
-
await writeFile7(
|
|
14302
|
+
const file = `${createHash12("sha256").update(traceId).digest("hex").slice(0, 24)}.json`;
|
|
14303
|
+
await writeFile7(join24(samples, file), `${JSON.stringify(trajectory, null, 2)}
|
|
16257
14304
|
`, "utf8");
|
|
16258
14305
|
}
|
|
16259
14306
|
return {
|
|
@@ -16275,12 +14322,12 @@ function hodoscopeAnalyzer(options = {}) {
|
|
|
16275
14322
|
return {
|
|
16276
14323
|
name: "hodoscope",
|
|
16277
14324
|
async analyze(otlpPath, runOptions2 = {}) {
|
|
16278
|
-
const outputRoot = options.outputDir ? resolve10(options.outputDir) :
|
|
14325
|
+
const outputRoot = options.outputDir ? resolve10(options.outputDir) : tmpdir5();
|
|
16279
14326
|
await mkdir5(outputRoot, { recursive: true });
|
|
16280
|
-
const directory = await mkdtemp5(
|
|
16281
|
-
const input =
|
|
16282
|
-
const analysisFile =
|
|
16283
|
-
const samplesFile =
|
|
14327
|
+
const directory = await mkdtemp5(join24(outputRoot, "traces-hodoscope-"));
|
|
14328
|
+
const input = join24(directory, "input");
|
|
14329
|
+
const analysisFile = join24(directory, "analysis.hodoscope.json");
|
|
14330
|
+
const samplesFile = join24(directory, "samples.json");
|
|
16284
14331
|
try {
|
|
16285
14332
|
const prepared = await writeHodoscopeInput(otlpPath, {
|
|
16286
14333
|
directory: input,
|
|
@@ -16900,7 +14947,7 @@ function assertPositiveInteger(name, value) {
|
|
|
16900
14947
|
}
|
|
16901
14948
|
|
|
16902
14949
|
// src/analyst-engine-prime.ts
|
|
16903
|
-
import { createHash as
|
|
14950
|
+
import { createHash as createHash13 } from "crypto";
|
|
16904
14951
|
import { request as httpRequest } from "http";
|
|
16905
14952
|
import { request as httpsRequest } from "https";
|
|
16906
14953
|
var DEFAULT_BRIDGE_URL = "http://localhost:4181";
|
|
@@ -17124,7 +15171,7 @@ function toAnalystFinding(row, index, spanTraces, model, producedAt) {
|
|
|
17124
15171
|
const traces = spanTraces.get(spanId);
|
|
17125
15172
|
return spanEvidenceUri([...traces][0], spanId);
|
|
17126
15173
|
});
|
|
17127
|
-
const findingId = `prime-${
|
|
15174
|
+
const findingId = `prime-${createHash13("sha256").update(`${index}
|
|
17128
15175
|
${row.claim}
|
|
17129
15176
|
${uris.join("\n")}`).digest("hex").slice(0, 12)}`;
|
|
17130
15177
|
return {
|
|
@@ -17408,10 +15455,10 @@ async function applyRedactor(spans, redactor) {
|
|
|
17408
15455
|
}
|
|
17409
15456
|
|
|
17410
15457
|
// src/file-export.ts
|
|
17411
|
-
import { createHash as
|
|
15458
|
+
import { createHash as createHash14 } from "crypto";
|
|
17412
15459
|
import { mkdtemp as mkdtemp6, readFile as readFile12, writeFile as writeFile8 } from "fs/promises";
|
|
17413
|
-
import { tmpdir as
|
|
17414
|
-
import { join as
|
|
15460
|
+
import { tmpdir as tmpdir6 } from "os";
|
|
15461
|
+
import { join as join25 } from "path";
|
|
17415
15462
|
import {
|
|
17416
15463
|
firstNumberAttr as firstNumberAttr2,
|
|
17417
15464
|
LLM_CACHED_TOKEN_ATTR_KEYS,
|
|
@@ -17446,7 +15493,7 @@ function stableJson(value) {
|
|
|
17446
15493
|
});
|
|
17447
15494
|
}
|
|
17448
15495
|
function hashId(value, chars = 16) {
|
|
17449
|
-
return
|
|
15496
|
+
return createHash14("sha256").update(stableJson(value)).digest("hex").slice(0, chars);
|
|
17450
15497
|
}
|
|
17451
15498
|
function isoTime2(value) {
|
|
17452
15499
|
if (typeof value === "string") {
|
|
@@ -17757,6 +15804,10 @@ function llmEventUsage(row) {
|
|
|
17757
15804
|
...costUsd !== void 0 ? { costUsd } : {}
|
|
17758
15805
|
};
|
|
17759
15806
|
}
|
|
15807
|
+
function toolPartId(row) {
|
|
15808
|
+
const part = objectValue2(objectValue2(row.data)?.part) ?? objectValue2(row.part);
|
|
15809
|
+
return part && part.type === "tool" ? stringValue3(part.id) : void 0;
|
|
15810
|
+
}
|
|
17760
15811
|
function sandboxEventsToSpans(rows, wrapper) {
|
|
17761
15812
|
const context = wrapper ?? {};
|
|
17762
15813
|
const sessionId = findStringKey(context, ["session_id", "sessionId"], 3) ?? findStringKey(rows, ["session_id", "sessionId"], 4);
|
|
@@ -17788,12 +15839,32 @@ function sandboxEventsToSpans(rows, wrapper) {
|
|
|
17788
15839
|
}
|
|
17789
15840
|
})
|
|
17790
15841
|
];
|
|
15842
|
+
const lastEventOfPart = /* @__PURE__ */ new Map();
|
|
15843
|
+
const firstTimeOfPart = /* @__PURE__ */ new Map();
|
|
15844
|
+
const errorOfPart = /* @__PURE__ */ new Map();
|
|
15845
|
+
rows.forEach((row, index) => {
|
|
15846
|
+
const part = toolPartId(row);
|
|
15847
|
+
if (part === void 0) return;
|
|
15848
|
+
lastEventOfPart.set(part, index);
|
|
15849
|
+
const time = eventTimestamp(row);
|
|
15850
|
+
const first = firstTimeOfPart.get(part);
|
|
15851
|
+
if (time && (first === void 0 || time < first)) firstTimeOfPart.set(part, time);
|
|
15852
|
+
const status = eventStatus(row, eventType(row) ?? "event");
|
|
15853
|
+
if (status.status === "ERROR") errorOfPart.set(part, status);
|
|
15854
|
+
});
|
|
15855
|
+
const mergedToolEvents = rows.filter((row, index) => {
|
|
15856
|
+
const part = toolPartId(row);
|
|
15857
|
+
return part !== void 0 && lastEventOfPart.get(part) !== index;
|
|
15858
|
+
}).length;
|
|
15859
|
+
if (mergedToolEvents > 0) spans[0].attributes["traces.event.tool_part_updates_merged"] = mergedToolEvents;
|
|
17791
15860
|
let previousTime = rootStart;
|
|
17792
15861
|
rows.forEach((row, index) => {
|
|
15862
|
+
const part = toolPartId(row);
|
|
15863
|
+
if (part !== void 0 && lastEventOfPart.get(part) !== index) return;
|
|
17793
15864
|
const type = eventType(row) ?? "event";
|
|
17794
|
-
const time = eventTimestamp(row) ?? previousTime;
|
|
15865
|
+
const time = (part !== void 0 ? firstTimeOfPart.get(part) : void 0) ?? eventTimestamp(row) ?? previousTime;
|
|
17795
15866
|
previousTime = time;
|
|
17796
|
-
const { status, message } = eventStatus(row, type);
|
|
15867
|
+
const { status, message } = (part !== void 0 ? errorOfPart.get(part) : void 0) ?? eventStatus(row, type);
|
|
17797
15868
|
const { kind, tool } = eventKind(row, type);
|
|
17798
15869
|
const name = kind === "TOOL" && tool ? `tool.${tool}` : `event.${type}`;
|
|
17799
15870
|
const toolInput2 = kind === "TOOL" ? eventValue(row, ["toolInput", "tool_input", "input", "args", "arguments", "full_command"]) : void 0;
|
|
@@ -17808,6 +15879,7 @@ function sandboxEventsToSpans(rows, wrapper) {
|
|
|
17808
15879
|
}) : {}
|
|
17809
15880
|
};
|
|
17810
15881
|
copyDefined(extra, {
|
|
15882
|
+
"traces.event.part_id": part,
|
|
17811
15883
|
"traces.event.id": stringValue3(row.id),
|
|
17812
15884
|
"traces.event.raw_type": stringValue3(row.type),
|
|
17813
15885
|
"traces.event.raw_event": stringValue3(row.event)
|
|
@@ -18057,15 +16129,15 @@ async function exportTraceEvidenceFile(inputPath, opts = {}) {
|
|
|
18057
16129
|
}
|
|
18058
16130
|
async function writeTraceEvidenceExportFile(inputPath, outPath, opts = {}) {
|
|
18059
16131
|
const result = await exportTraceEvidenceFile(inputPath, opts);
|
|
18060
|
-
const path = outPath ??
|
|
16132
|
+
const path = outPath ?? join25(await mkdtemp6(join25(tmpdir6(), "traces-export-")), "spans.openinference.jsonl");
|
|
18061
16133
|
await writeFile8(path, serializeSpans(result.spans), "utf8");
|
|
18062
16134
|
return { ...result, path };
|
|
18063
16135
|
}
|
|
18064
16136
|
|
|
18065
16137
|
// src/inspect.ts
|
|
18066
16138
|
import { mkdtemp as mkdtemp7, readFile as readFile13, writeFile as writeFile9 } from "fs/promises";
|
|
18067
|
-
import { tmpdir as
|
|
18068
|
-
import { basename as basename15, join as
|
|
16139
|
+
import { tmpdir as tmpdir7 } from "os";
|
|
16140
|
+
import { basename as basename15, join as join26 } from "path";
|
|
18069
16141
|
var severityRank2 = {
|
|
18070
16142
|
high: 0,
|
|
18071
16143
|
medium: 1,
|
|
@@ -18081,7 +16153,7 @@ function sessionLabel(row) {
|
|
|
18081
16153
|
const id = row.session.sessionId.length > 12 ? row.session.sessionId.slice(0, 12) : row.session.sessionId;
|
|
18082
16154
|
return `${row.session.harness}:${id} (${repoLabel(row)})`;
|
|
18083
16155
|
}
|
|
18084
|
-
function
|
|
16156
|
+
function pct(value) {
|
|
18085
16157
|
return `${(value * 100).toFixed(value >= 0.1 ? 0 : 1)}%`;
|
|
18086
16158
|
}
|
|
18087
16159
|
function topToolErrors(row) {
|
|
@@ -18160,7 +16232,7 @@ function addSessionFindings(index, findings) {
|
|
|
18160
16232
|
area: "session",
|
|
18161
16233
|
title: `High tool-error count/rate in ${errorSessions.length}/${index.totals.sessions} session(s)`,
|
|
18162
16234
|
impact: errors,
|
|
18163
|
-
evidence: errorSessions.slice(0, 5).map((row) => `${sessionLabel(row)}: ${row.metrics.erroredToolCallCount}/${row.metrics.toolCallCount} tool errors (${
|
|
16235
|
+
evidence: errorSessions.slice(0, 5).map((row) => `${sessionLabel(row)}: ${row.metrics.erroredToolCallCount}/${row.metrics.toolCallCount} tool errors (${pct(row.metrics.toolErrorRate)}); ${topToolErrors(row)}`),
|
|
18164
16236
|
next: "Fix the most common failing command/tool first, then rerun `traces index` to confirm the error count drops.",
|
|
18165
16237
|
refs: errorSessions.slice(0, 5).map((row) => row.session.path)
|
|
18166
16238
|
});
|
|
@@ -18354,13 +16426,13 @@ async function readSessionIndexFile(path) {
|
|
|
18354
16426
|
return parsed;
|
|
18355
16427
|
}
|
|
18356
16428
|
async function writeInspectionReportFile(report, outPath) {
|
|
18357
|
-
const path = outPath ??
|
|
16429
|
+
const path = outPath ?? join26(await mkdtemp7(join26(tmpdir7(), "traces-inspect-")), "inspection-report.md");
|
|
18358
16430
|
await writeFile9(path, renderInspectionReport(report), "utf8");
|
|
18359
16431
|
return path;
|
|
18360
16432
|
}
|
|
18361
16433
|
|
|
18362
16434
|
// src/live.ts
|
|
18363
|
-
import { createHash as
|
|
16435
|
+
import { createHash as createHash15 } from "crypto";
|
|
18364
16436
|
var VERIFY_RE = /\b(test|tests|vitest|jest|pytest|go test|cargo test|tsc|typecheck|lint|biome|eslint|build|check:invariants|preflight)\b/i;
|
|
18365
16437
|
var CHANGE_RE = /\b(apply_patch|edit|write|create|delete|rename|replace|update|git apply|sed -i|perl -pi|tee\s|cat\s+>)/i;
|
|
18366
16438
|
var CLAIM_DONE_RE = /\b(done|fixed|complete|completed|implemented|merged|verified|all green|tests pass|ready|works now)\b/i;
|
|
@@ -18380,7 +16452,7 @@ var sleep = (ms2, signal) => new Promise((resolve13) => {
|
|
|
18380
16452
|
signal?.addEventListener("abort", onAbort, { once: true });
|
|
18381
16453
|
});
|
|
18382
16454
|
function hash(parts) {
|
|
18383
|
-
return
|
|
16455
|
+
return createHash15("sha256").update(parts.join("\0")).digest("hex").slice(0, 16);
|
|
18384
16456
|
}
|
|
18385
16457
|
function asString(value) {
|
|
18386
16458
|
if (typeof value === "string" && value.trim()) return value;
|
|
@@ -18788,7 +16860,7 @@ async function streamSessions(opts) {
|
|
|
18788
16860
|
|
|
18789
16861
|
// src/supervisor-run-context.ts
|
|
18790
16862
|
import { readFile as readFile14, stat as stat16 } from "fs/promises";
|
|
18791
|
-
import { basename as basename16, join as
|
|
16863
|
+
import { basename as basename16, join as join27 } from "path";
|
|
18792
16864
|
import { materializeTreeView } from "@tangle-network/agent-runtime/kernel";
|
|
18793
16865
|
import {
|
|
18794
16866
|
readRuntimeSupervisorRun
|
|
@@ -18950,14 +17022,14 @@ function parseInstant(v) {
|
|
|
18950
17022
|
}
|
|
18951
17023
|
async function readFileRunContext(runDir, opts = {}) {
|
|
18952
17024
|
const sources = await readRuntimeSupervisorRun(runDir);
|
|
18953
|
-
const coordination = readCoordinationLog(await readMaybe(
|
|
17025
|
+
const coordination = readCoordinationLog(await readMaybe(join27(runDir, COORDINATION_LOG_FILE)));
|
|
18954
17026
|
return {
|
|
18955
17027
|
...sources,
|
|
18956
17028
|
instanceId: opts.instanceId !== void 0 ? opts.instanceId : sources.instanceId,
|
|
18957
17029
|
arm: opts.arm !== void 0 ? opts.arm : basename16(runDir),
|
|
18958
17030
|
workers: sources.workers !== null && coordination.present ? workerSources(sources.workers, coordination) : sources.workers,
|
|
18959
17031
|
...sources.journal !== null ? {
|
|
18960
|
-
rootTranscriptRef:
|
|
17032
|
+
rootTranscriptRef: join27(runDir, SPAWN_JOURNAL_FILE),
|
|
18961
17033
|
traceCommand: `traces watch ${runDir}`
|
|
18962
17034
|
} : {}
|
|
18963
17035
|
};
|
|
@@ -18966,7 +17038,7 @@ function fileRunContextSupervisorRunReader(runDir, opts = {}) {
|
|
|
18966
17038
|
return { runRef: runDir, read: () => readFileRunContext(runDir, opts) };
|
|
18967
17039
|
}
|
|
18968
17040
|
async function isFileRunContextDir(runDir) {
|
|
18969
|
-
return stat16(
|
|
17041
|
+
return stat16(join27(runDir, SPAWN_JOURNAL_FILE)).then((s) => s.isFile()).catch(() => false);
|
|
18970
17042
|
}
|
|
18971
17043
|
function parseJsonObject(text2) {
|
|
18972
17044
|
if (text2 === null) return null;
|
|
@@ -19032,9 +17104,9 @@ function mergeRolls(a, b) {
|
|
|
19032
17104
|
};
|
|
19033
17105
|
}
|
|
19034
17106
|
async function readRunContextSnapshot(runDir) {
|
|
19035
|
-
const journal = readRunContextJournal(await readMaybe(
|
|
19036
|
-
const coordination = readCoordinationLog(await readMaybe(
|
|
19037
|
-
const resultText = await readMaybe(
|
|
17107
|
+
const journal = readRunContextJournal(await readMaybe(join27(runDir, SPAWN_JOURNAL_FILE)));
|
|
17108
|
+
const coordination = readCoordinationLog(await readMaybe(join27(runDir, COORDINATION_LOG_FILE)));
|
|
17109
|
+
const resultText = await readMaybe(join27(runDir, RESULT_FILE));
|
|
19038
17110
|
const result = parseResult(resultText);
|
|
19039
17111
|
const view = safeTreeView(journal.events);
|
|
19040
17112
|
const nodes = view === null ? [] : orderNodes(view, journal);
|
|
@@ -19217,7 +17289,7 @@ function wrap(text2, indent) {
|
|
|
19217
17289
|
|
|
19218
17290
|
// src/run-span-tree.ts
|
|
19219
17291
|
import { readdir as readdir18 } from "fs/promises";
|
|
19220
|
-
import { join as
|
|
17292
|
+
import { join as join28 } from "path";
|
|
19221
17293
|
import {
|
|
19222
17294
|
firstNumberAttr as firstNumberAttr3,
|
|
19223
17295
|
LLM_COST_ATTR_KEYS as LLM_COST_ATTR_KEYS2,
|
|
@@ -19492,7 +17564,7 @@ async function findSpanSourceFiles(dir) {
|
|
|
19492
17564
|
const found = [];
|
|
19493
17565
|
const entries = await readdir18(dir, { withFileTypes: true }).catch(() => []);
|
|
19494
17566
|
for (const entry of entries) {
|
|
19495
|
-
const child =
|
|
17567
|
+
const child = join28(dir, entry.name);
|
|
19496
17568
|
if (entry.isFile()) {
|
|
19497
17569
|
if (SPAN_FILE_PATTERN.test(entry.name)) found.push(child);
|
|
19498
17570
|
continue;
|
|
@@ -19500,7 +17572,7 @@ async function findSpanSourceFiles(dir) {
|
|
|
19500
17572
|
if (!entry.isDirectory()) continue;
|
|
19501
17573
|
const nested = await readdir18(child, { withFileTypes: true }).catch(() => []);
|
|
19502
17574
|
for (const inner of nested) {
|
|
19503
|
-
if (inner.isFile() && SPAN_FILE_PATTERN.test(inner.name)) found.push(
|
|
17575
|
+
if (inner.isFile() && SPAN_FILE_PATTERN.test(inner.name)) found.push(join28(child, inner.name));
|
|
19504
17576
|
}
|
|
19505
17577
|
}
|
|
19506
17578
|
return found.sort();
|
|
@@ -19508,7 +17580,7 @@ async function findSpanSourceFiles(dir) {
|
|
|
19508
17580
|
|
|
19509
17581
|
// src/supervisor-run-watch.ts
|
|
19510
17582
|
import { readdir as readdir19 } from "fs/promises";
|
|
19511
|
-
import { join as
|
|
17583
|
+
import { join as join29 } from "path";
|
|
19512
17584
|
function budget(node) {
|
|
19513
17585
|
const parts = [`${int(node.budget.maxIterations)}it`, `${int(node.budget.maxTokens)}tok`];
|
|
19514
17586
|
if (node.budget.maxUsd !== void 0) parts.push(`$${node.budget.maxUsd.toFixed(2)}`);
|
|
@@ -19652,7 +17724,7 @@ async function resolveRunContextDir(dir) {
|
|
|
19652
17724
|
const candidates = [];
|
|
19653
17725
|
for (const entry of entries) {
|
|
19654
17726
|
if (!entry.isDirectory()) continue;
|
|
19655
|
-
const child =
|
|
17727
|
+
const child = join29(dir, entry.name);
|
|
19656
17728
|
if (await isFileRunContextDir(child)) candidates.push(child);
|
|
19657
17729
|
}
|
|
19658
17730
|
if (candidates.length === 1) return candidates[0];
|
|
@@ -19988,7 +18060,7 @@ function parseJsonAnswer(text2) {
|
|
|
19988
18060
|
|
|
19989
18061
|
// src/ask.ts
|
|
19990
18062
|
import { mkdir as mkdir6, readFile as readFile15, writeFile as writeFile10 } from "fs/promises";
|
|
19991
|
-
import { join as
|
|
18063
|
+
import { join as join30, resolve as resolve11 } from "path";
|
|
19992
18064
|
import { performance } from "perf_hooks";
|
|
19993
18065
|
import { CostCeilingReachedError, CostLedger } from "@tangle-network/agent-eval";
|
|
19994
18066
|
import {
|
|
@@ -20543,8 +18615,8 @@ async function writeTraceQuestionsArtifacts(result, outDir) {
|
|
|
20543
18615
|
await mkdir6(directory, { recursive: true });
|
|
20544
18616
|
const paths = {
|
|
20545
18617
|
directory,
|
|
20546
|
-
result:
|
|
20547
|
-
report:
|
|
18618
|
+
result: join30(directory, "answers.json"),
|
|
18619
|
+
report: join30(directory, "report.md"),
|
|
20548
18620
|
traces: result.otlpPath
|
|
20549
18621
|
};
|
|
20550
18622
|
const { report, ...machine } = result;
|
|
@@ -20606,9 +18678,9 @@ async function watchSessions(opts) {
|
|
|
20606
18678
|
}
|
|
20607
18679
|
|
|
20608
18680
|
// src/bundle-view.ts
|
|
20609
|
-
import { createHash as
|
|
18681
|
+
import { createHash as createHash16 } from "crypto";
|
|
20610
18682
|
import { mkdir as mkdir7, readdir as readdir20, readFile as readFile16, writeFile as writeFile11 } from "fs/promises";
|
|
20611
|
-
import { dirname as dirname11, join as
|
|
18683
|
+
import { dirname as dirname11, join as join31, resolve as resolve12 } from "path";
|
|
20612
18684
|
var EVIDENCE_ONLY_ALLOW_LIST = [
|
|
20613
18685
|
{
|
|
20614
18686
|
path: "derived/session-index.json",
|
|
@@ -20732,7 +18804,7 @@ async function findSessionTextLeaks(opts) {
|
|
|
20732
18804
|
const index = /* @__PURE__ */ new Map();
|
|
20733
18805
|
const carried = [];
|
|
20734
18806
|
for (const path of opts.carriedPaths) {
|
|
20735
|
-
const signatures = sessionTextSignatures(await readFile16(
|
|
18807
|
+
const signatures = sessionTextSignatures(await readFile16(join31(opts.viewDir, path), "utf8"));
|
|
20736
18808
|
carried.push({ path, signatures: signatures.size });
|
|
20737
18809
|
for (const signature of signatures) if (!index.has(signature)) index.set(signature, path);
|
|
20738
18810
|
}
|
|
@@ -20740,7 +18812,7 @@ async function findSessionTextLeaks(opts) {
|
|
|
20740
18812
|
const comparedSources = [];
|
|
20741
18813
|
let sourceSignatures = 0;
|
|
20742
18814
|
for (const sourcePath of opts.excludedPaths) {
|
|
20743
|
-
const signatures = sessionTextSignatures(await readFile16(
|
|
18815
|
+
const signatures = sessionTextSignatures(await readFile16(join31(opts.sourceDir, sourcePath), "utf8"));
|
|
20744
18816
|
comparedSources.push(sourcePath);
|
|
20745
18817
|
sourceSignatures += signatures.size;
|
|
20746
18818
|
for (const signature of signatures) {
|
|
@@ -20799,17 +18871,17 @@ async function projectSessionBundle(opts) {
|
|
|
20799
18871
|
const candidates = [];
|
|
20800
18872
|
for (const path of await listSessionBundleFiles(sourceDir)) {
|
|
20801
18873
|
if (path === "manifest.json") continue;
|
|
20802
|
-
const bytes = await readFile16(
|
|
20803
|
-
const sha2562 =
|
|
18874
|
+
const bytes = await readFile16(join31(sourceDir, path));
|
|
18875
|
+
const sha2562 = createHash16("sha256").update(bytes).digest("hex");
|
|
20804
18876
|
const recorded = sourceHashes.get(path);
|
|
20805
18877
|
if (!recorded) {
|
|
20806
18878
|
throw new Error(
|
|
20807
|
-
`${
|
|
18879
|
+
`${join31(sourceDir, path)} is not listed in the bundle manifest \u2014 the bundle changed after assembly and cannot be projected; re-run \`traces bundle\``
|
|
20808
18880
|
);
|
|
20809
18881
|
}
|
|
20810
18882
|
if (recorded.sha256 !== sha2562) {
|
|
20811
18883
|
throw new Error(
|
|
20812
|
-
`${
|
|
18884
|
+
`${join31(sourceDir, path)} does not match its manifest hash (${recorded.sha256} recorded, ${sha2562} on disk) \u2014 the bundle changed after assembly and cannot be projected; re-run \`traces bundle\``
|
|
20813
18885
|
);
|
|
20814
18886
|
}
|
|
20815
18887
|
const verdict = classifyEvidenceOnlyPath(path);
|
|
@@ -20845,7 +18917,7 @@ async function projectSessionBundle(opts) {
|
|
|
20845
18917
|
});
|
|
20846
18918
|
continue;
|
|
20847
18919
|
}
|
|
20848
|
-
const target =
|
|
18920
|
+
const target = join31(outDir, candidate.path);
|
|
20849
18921
|
await mkdir7(dirname11(target), { recursive: true });
|
|
20850
18922
|
await writeFile11(target, candidate.bytes);
|
|
20851
18923
|
carriedPaths.push(candidate.path);
|
|
@@ -20862,7 +18934,7 @@ async function projectSessionBundle(opts) {
|
|
|
20862
18934
|
})),
|
|
20863
18935
|
matches: 0
|
|
20864
18936
|
};
|
|
20865
|
-
const sourceManifestBytes = await readFile16(
|
|
18937
|
+
const sourceManifestBytes = await readFile16(join31(sourceDir, "manifest.json"));
|
|
20866
18938
|
const manifest = {
|
|
20867
18939
|
schemaVersion: 2,
|
|
20868
18940
|
kind: "traces.session_bundle",
|
|
@@ -20876,7 +18948,7 @@ async function projectSessionBundle(opts) {
|
|
|
20876
18948
|
knownLimits: [...source.knownLimits, ...EVIDENCE_ONLY_LIMITS],
|
|
20877
18949
|
projection: {
|
|
20878
18950
|
sourceDirectory: sourceDir,
|
|
20879
|
-
sourceManifestSha256:
|
|
18951
|
+
sourceManifestSha256: createHash16("sha256").update(sourceManifestBytes).digest("hex"),
|
|
20880
18952
|
leakCheck
|
|
20881
18953
|
}
|
|
20882
18954
|
};
|
|
@@ -20887,11 +18959,11 @@ async function projectSessionBundle(opts) {
|
|
|
20887
18959
|
// src/upload-state.ts
|
|
20888
18960
|
import { mkdir as mkdir8, readFile as readFile17, writeFile as writeFile12 } from "fs/promises";
|
|
20889
18961
|
import { homedir as homedir12 } from "os";
|
|
20890
|
-
import { dirname as dirname12, join as
|
|
18962
|
+
import { dirname as dirname12, join as join32 } from "path";
|
|
20891
18963
|
import { contentHash } from "@tangle-network/agent-eval";
|
|
20892
18964
|
function stateFile() {
|
|
20893
|
-
const base = process.env.XDG_STATE_HOME ??
|
|
20894
|
-
return
|
|
18965
|
+
const base = process.env.XDG_STATE_HOME ?? join32(homedir12(), ".local", "state");
|
|
18966
|
+
return join32(base, "tangle-traces", "uploaded.json");
|
|
20895
18967
|
}
|
|
20896
18968
|
function uploadKey(harness, sessionId) {
|
|
20897
18969
|
return `${harness}:${sessionId}`;
|
|
@@ -20923,7 +18995,7 @@ function alreadyUploaded(state, harness, sessionId, hash2) {
|
|
|
20923
18995
|
import { readFile as readFile18 } from "fs/promises";
|
|
20924
18996
|
import { createRequire as createRequire2 } from "module";
|
|
20925
18997
|
import { hostname as hostname2 } from "os";
|
|
20926
|
-
import { basename as basename17, join as
|
|
18998
|
+
import { basename as basename17, join as join33 } from "path";
|
|
20927
18999
|
import { hostedClientFromEnv } from "@tangle-network/agent-eval/hosted";
|
|
20928
19000
|
import { REDACTION_VERSION } from "@tangle-network/agent-eval/traces";
|
|
20929
19001
|
var require2 = createRequire2(import.meta.url);
|
|
@@ -20947,7 +19019,7 @@ async function planUpload(opts = {}) {
|
|
|
20947
19019
|
async function gitBranch(cwd) {
|
|
20948
19020
|
if (!cwd) return void 0;
|
|
20949
19021
|
try {
|
|
20950
|
-
const head = await readFile18(
|
|
19022
|
+
const head = await readFile18(join33(cwd, ".git", "HEAD"), "utf8");
|
|
20951
19023
|
const m = head.match(/^ref:\s*refs\/heads\/(.+)$/m);
|
|
20952
19024
|
return m?.[1]?.trim();
|
|
20953
19025
|
} catch {
|
|
@@ -21275,60 +19347,20 @@ export {
|
|
|
21275
19347
|
hashSessionBundleFiles,
|
|
21276
19348
|
writeSessionBundleManifest,
|
|
21277
19349
|
readSessionBundleManifest,
|
|
19350
|
+
verifySessionBundle,
|
|
21278
19351
|
createBundleSourceReader,
|
|
21279
19352
|
assertOutsideSourceBundle,
|
|
21280
19353
|
isChatTrajectoryMessage,
|
|
21281
19354
|
isChatTrajectory,
|
|
21282
19355
|
chatTrajectoryToSpans,
|
|
21283
19356
|
importCodeTraceBench,
|
|
21284
|
-
|
|
21285
|
-
parseObservationOutput,
|
|
21286
|
-
deriveFailureSignature,
|
|
21287
|
-
ingestCodeTraceBenchSteps,
|
|
19357
|
+
DEFAULT_SANDBOX_BASE_URL,
|
|
21288
19358
|
sandboxReplayBackend,
|
|
21289
|
-
|
|
21290
|
-
SandboxCounterfactualRunner,
|
|
21291
|
-
replayVerify,
|
|
21292
|
-
parseReplayVerifyArgs,
|
|
21293
|
-
replayVerifyUsage,
|
|
21294
|
-
cmdReplayVerify,
|
|
21295
|
-
parseCorpusFlag,
|
|
21296
|
-
SUBMIT_ACTION_SIGNATURE,
|
|
21297
|
-
isSubmitAction,
|
|
21298
|
-
readLabelEntries,
|
|
21299
|
-
goldIncorrectSteps,
|
|
21300
|
-
resolveCaseResources,
|
|
21301
|
-
enumerateReplayableCases,
|
|
19359
|
+
assertSandboxReachable,
|
|
21302
19360
|
zaiChatCaller,
|
|
21303
|
-
|
|
21304
|
-
buildFixPrompt,
|
|
21305
|
-
buildRetryFixPrompt,
|
|
21306
|
-
countScriptCommands,
|
|
21307
|
-
extractFixCommand,
|
|
21308
|
-
generateFixCommand,
|
|
21309
|
-
runFixLoop,
|
|
21310
|
-
derivedImageTag,
|
|
21311
|
-
dockerImagePreparer,
|
|
21312
|
-
seededSample,
|
|
21313
|
-
runReplayBatch,
|
|
21314
|
-
renderBatchReport,
|
|
21315
|
-
parseReplayBatchArgs,
|
|
21316
|
-
replayBatchUsage,
|
|
19361
|
+
cmdReplayVerify,
|
|
21317
19362
|
cmdReplayVerifyBatch,
|
|
21318
|
-
|
|
21319
|
-
resolveFindingInvocation,
|
|
21320
|
-
replayVerifyFinding,
|
|
21321
|
-
findingReplayStep,
|
|
21322
|
-
findingTrajectoryId,
|
|
21323
|
-
resolveFindingReplayability,
|
|
21324
|
-
classifyVerdict,
|
|
21325
|
-
DEFAULT_SANDBOX_BASE_URL,
|
|
21326
|
-
assertSandboxReachable,
|
|
21327
|
-
verifyFindings,
|
|
21328
|
-
renderVerifiedFindingsSection,
|
|
21329
|
-
readFindingsFile,
|
|
21330
|
-
verifyFindingsUsage,
|
|
21331
|
-
parseVerifyFindingsArgs,
|
|
19363
|
+
verifyFindingsInSandbox,
|
|
21332
19364
|
cmdVerifyFindings,
|
|
21333
19365
|
writeHodoscopeInput,
|
|
21334
19366
|
hodoscopeAnalyzer,
|