@tangle-network/traces 0.15.1 → 0.15.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -2
- package/dist/{chunk-7DOBU7BJ.js → chunk-SKOW5V5U.js} +535 -2492
- package/dist/cli.js +49 -9
- package/dist/index.d.ts +76 -879
- package/dist/index.js +6 -91
- package/docs/replay-verify.md +6 -5
- package/package.json +5 -5
|
@@ -1017,9 +1017,9 @@ function serializeSpans(spans) {
|
|
|
1017
1017
|
}
|
|
1018
1018
|
async function writeOtlpFile(spans, outPath) {
|
|
1019
1019
|
const { mkdtemp: mkdtemp8, writeFile: writeFile13 } = await import("fs/promises");
|
|
1020
|
-
const { tmpdir:
|
|
1021
|
-
const { join:
|
|
1022
|
-
const path = outPath ??
|
|
1020
|
+
const { tmpdir: tmpdir8 } = await import("os");
|
|
1021
|
+
const { join: join34 } = await import("path");
|
|
1022
|
+
const path = outPath ?? join34(await mkdtemp8(join34(tmpdir8(), "traces-")), "spans.otlp.jsonl");
|
|
1023
1023
|
await writeFile13(path, serializeSpans(spans), "utf8");
|
|
1024
1024
|
return path;
|
|
1025
1025
|
}
|
|
@@ -1130,6 +1130,10 @@ var KIND = {
|
|
|
1130
1130
|
LLM: "llm",
|
|
1131
1131
|
TOOL: "tool",
|
|
1132
1132
|
CHAIN: "custom",
|
|
1133
|
+
EMBEDDING: "custom",
|
|
1134
|
+
RERANKER: "custom",
|
|
1135
|
+
GUARDRAIL: "custom",
|
|
1136
|
+
PROMPT: "custom",
|
|
1133
1137
|
EVALUATOR: "custom",
|
|
1134
1138
|
RETRIEVER: "custom",
|
|
1135
1139
|
UNKNOWN: "custom"
|
|
@@ -1818,7 +1822,7 @@ import {
|
|
|
1818
1822
|
LLM_INPUT_TOKENS as LLM_INPUT_TOKENS2,
|
|
1819
1823
|
LLM_OUTPUT_TOKENS as LLM_OUTPUT_TOKENS2
|
|
1820
1824
|
} from "@tangle-network/agent-eval/trace-attributes";
|
|
1821
|
-
import { deriveHexId as deriveHexId2 } from "@tangle-network/agent-trace-contract";
|
|
1825
|
+
import { ATTR as ATTR3, deriveHexId as deriveHexId2 } from "@tangle-network/agent-trace-contract";
|
|
1822
1826
|
|
|
1823
1827
|
// src/adapters/claude-subagent-sources.ts
|
|
1824
1828
|
import { readdir as readdir4, realpath } from "fs/promises";
|
|
@@ -1984,7 +1988,10 @@ function selectWorkflowBinding(file, bindings, childStartedAt, explicitToolUseId
|
|
|
1984
1988
|
`Cannot link Claude Workflow subagent ${file}: parent call is ambiguous`
|
|
1985
1989
|
);
|
|
1986
1990
|
}
|
|
1987
|
-
return
|
|
1991
|
+
return {
|
|
1992
|
+
binding: latest[0],
|
|
1993
|
+
confidence: explicitToolUseId || bindings.length === 1 ? "correlated" : "heuristic"
|
|
1994
|
+
};
|
|
1988
1995
|
}
|
|
1989
1996
|
|
|
1990
1997
|
// src/adapters/claude-subagent-sources.ts
|
|
@@ -2092,6 +2099,21 @@ async function collectClaudeSubagentSources(ref, bindings, signal) {
|
|
|
2092
2099
|
|
|
2093
2100
|
// src/adapters/claude.ts
|
|
2094
2101
|
var SERVICE2 = "claude-code";
|
|
2102
|
+
var AGENT_TOOL_RENAMED_IN = [2, 1, 63];
|
|
2103
|
+
function modelFacingToolNames(tools, version) {
|
|
2104
|
+
if (typeof version !== "string" || !atLeast(version, AGENT_TOOL_RENAMED_IN)) return [...tools];
|
|
2105
|
+
return tools.map((name) => name === "Task" ? "Agent" : name);
|
|
2106
|
+
}
|
|
2107
|
+
function atLeast(version, min) {
|
|
2108
|
+
const parts = version.split(".").map((p) => Number.parseInt(p, 10));
|
|
2109
|
+
if (parts.length < min.length || parts.some((p) => !Number.isFinite(p))) return false;
|
|
2110
|
+
for (let i = 0; i < min.length; i++) {
|
|
2111
|
+
if (parts[i] !== min[i]) return parts[i] > min[i];
|
|
2112
|
+
}
|
|
2113
|
+
return true;
|
|
2114
|
+
}
|
|
2115
|
+
var TOOL_DEFINITIONS_ATTR = "gen_ai.tool.definitions";
|
|
2116
|
+
var RUN_STATUS_ATTR = "run.status";
|
|
2095
2117
|
var EPOCH = (/* @__PURE__ */ new Date(0)).toISOString();
|
|
2096
2118
|
var HUMAN_ORIGIN = "human";
|
|
2097
2119
|
var ORIGIN_EVIDENCE_ATTR = "traces.claude.actor_evidence";
|
|
@@ -2203,6 +2225,7 @@ function projectClaudeEvent(event) {
|
|
|
2203
2225
|
cacheWriteInputTokens: event.message.usage?.cache_creation_input_tokens ?? null,
|
|
2204
2226
|
content: textOf2(event.message.content) || null,
|
|
2205
2227
|
contentSource: textSources(event.message, "content"),
|
|
2228
|
+
parentToolUseId: event.parent_tool_use_id ?? null,
|
|
2206
2229
|
tools
|
|
2207
2230
|
};
|
|
2208
2231
|
}
|
|
@@ -2233,6 +2256,7 @@ function projectClaudeEvent(event) {
|
|
|
2233
2256
|
timestamp,
|
|
2234
2257
|
prompt: prompt || null,
|
|
2235
2258
|
contentSource: textSources(event.message, "content"),
|
|
2259
|
+
parentToolUseId: event.parent_tool_use_id ?? null,
|
|
2236
2260
|
...prompt ? {
|
|
2237
2261
|
isSidechain: event.isSidechain === true,
|
|
2238
2262
|
isMeta: event.isMeta === true,
|
|
@@ -2352,15 +2376,21 @@ function consumeDistinctClaudeEvent(event, ctx, state, seen, sourcePath) {
|
|
|
2352
2376
|
consumeClaudeEvent(accepted.projection, accepted.uid, ctx, state);
|
|
2353
2377
|
return true;
|
|
2354
2378
|
}
|
|
2379
|
+
function inStreamParent(parentToolUseId, state) {
|
|
2380
|
+
if (!parentToolUseId) return void 0;
|
|
2381
|
+
const parentTool = state.toolSpanByUseId.get(parentToolUseId);
|
|
2382
|
+
return parentTool ? { span: parentTool, confidence: "explicit" } : void 0;
|
|
2383
|
+
}
|
|
2355
2384
|
function consumeClaudeEvent(event, uid, ctx, state) {
|
|
2356
2385
|
if (event.kind === "assistant") {
|
|
2357
2386
|
const messageId = event.messageId ?? uid;
|
|
2358
2387
|
let llmSpan = state.llmSpanByMessageId.get(messageId);
|
|
2359
2388
|
if (!llmSpan) {
|
|
2389
|
+
const parent = inStreamParent(event.parentToolUseId, state);
|
|
2360
2390
|
llmSpan = span({
|
|
2361
2391
|
traceId: ctx.traceId,
|
|
2362
2392
|
spanId: `${ctx.idPrefix}${uid}`,
|
|
2363
|
-
parentSpanId: ctx.rootParent,
|
|
2393
|
+
parentSpanId: parent ? parent.span.span_id : ctx.rootParent,
|
|
2364
2394
|
name: "llm.turn",
|
|
2365
2395
|
kind: "LLM",
|
|
2366
2396
|
startTime: event.timestamp,
|
|
@@ -2373,6 +2403,7 @@ function consumeClaudeEvent(event, uid, ctx, state) {
|
|
|
2373
2403
|
cacheWriteInputTokens: event.cacheWriteInputTokens,
|
|
2374
2404
|
step: state.step
|
|
2375
2405
|
});
|
|
2406
|
+
if (parent) llmSpan.attributes[ATTR3.parentConfidence] = parent.confidence;
|
|
2376
2407
|
state.spans.push(llmSpan);
|
|
2377
2408
|
state.llmSpanByMessageId.set(messageId, llmSpan);
|
|
2378
2409
|
state.step += 1;
|
|
@@ -2444,10 +2475,11 @@ function consumeClaudeEvent(event, uid, ctx, state) {
|
|
|
2444
2475
|
}));
|
|
2445
2476
|
state.sawUserTurn = true;
|
|
2446
2477
|
if (event.originKind) state.sawRecordedOrigin = true;
|
|
2478
|
+
const userParent = inStreamParent(event.parentToolUseId, state);
|
|
2447
2479
|
const prompt = userPromptSpan({
|
|
2448
2480
|
traceId: ctx.traceId,
|
|
2449
2481
|
spanId: `${ctx.idPrefix}${uid}:user`,
|
|
2450
|
-
parentSpanId: ctx.rootParent,
|
|
2482
|
+
parentSpanId: userParent ? userParent.span.span_id : ctx.rootParent,
|
|
2451
2483
|
startTime: event.timestamp,
|
|
2452
2484
|
service: SERVICE2,
|
|
2453
2485
|
agent: ctx.agent,
|
|
@@ -2460,6 +2492,7 @@ function consumeClaudeEvent(event, uid, ctx, state) {
|
|
|
2460
2492
|
prompt.attributes[ORIGIN_RECORDED_ATTR] = true;
|
|
2461
2493
|
prompt.attributes[ORIGIN_KIND_ATTR] = event.originKind;
|
|
2462
2494
|
}
|
|
2495
|
+
if (userParent) prompt.attributes[ATTR3.parentConfidence] = userParent.confidence;
|
|
2463
2496
|
state.spans.push(prompt);
|
|
2464
2497
|
state.promptSpans.push(prompt);
|
|
2465
2498
|
state.step += 1;
|
|
@@ -2788,9 +2821,21 @@ var ClaudeAdapter = class {
|
|
|
2788
2821
|
let selectedTurnId;
|
|
2789
2822
|
let selectedTurnFound = false;
|
|
2790
2823
|
let active = taskScope === "all";
|
|
2824
|
+
let offeredTools;
|
|
2825
|
+
let runResult;
|
|
2791
2826
|
for await (const event of readJsonl(ref.path, sessionJsonlOptions(ref, options))) {
|
|
2792
2827
|
options.signal?.throwIfAborted();
|
|
2793
|
-
|
|
2828
|
+
const eventSessionId = event.sessionId ?? event.session_id;
|
|
2829
|
+
if (!discoveredTraceId && eventSessionId) discoveredTraceId = eventSessionId;
|
|
2830
|
+
if (event.type === "system" && event.subtype === "init" && Array.isArray(event.tools)) {
|
|
2831
|
+
offeredTools = modelFacingToolNames(
|
|
2832
|
+
event.tools.filter((t) => typeof t === "string"),
|
|
2833
|
+
event.claude_code_version
|
|
2834
|
+
);
|
|
2835
|
+
}
|
|
2836
|
+
if (event.type === "result" && typeof event.is_error === "boolean") {
|
|
2837
|
+
runResult = { failed: event.is_error, subtype: event.subtype ?? "unknown" };
|
|
2838
|
+
}
|
|
2794
2839
|
const accepted = distinctClaudeEvent(event, seen, ref.path, `step${state.step}`);
|
|
2795
2840
|
if (!accepted) continue;
|
|
2796
2841
|
indexWorkflowProjection(accepted.projection, workflowIndex);
|
|
@@ -2845,6 +2890,13 @@ var ClaudeAdapter = class {
|
|
|
2845
2890
|
service: SERVICE2,
|
|
2846
2891
|
agent: SERVICE2
|
|
2847
2892
|
});
|
|
2893
|
+
if (offeredTools) {
|
|
2894
|
+
root.attributes[TOOL_DEFINITIONS_ATTR] = JSON.stringify(offeredTools.map((name) => ({ type: "function", name })));
|
|
2895
|
+
}
|
|
2896
|
+
if (runResult) {
|
|
2897
|
+
root.attributes[RUN_STATUS_ATTR] = runResult.failed ? "failed" : "completed";
|
|
2898
|
+
root.status = runResult.failed ? { code: "ERROR", message: `the run ended with ${runResult.subtype}` } : { code: "OK" };
|
|
2899
|
+
}
|
|
2848
2900
|
const spans = [root];
|
|
2849
2901
|
appendAll(spans, main.spans);
|
|
2850
2902
|
await this.foldSubagents(
|
|
@@ -2905,13 +2957,14 @@ var ClaudeAdapter = class {
|
|
|
2905
2957
|
workflowBindings
|
|
2906
2958
|
);
|
|
2907
2959
|
const parsed = await parseClaudeSubagent(ref, traceId, agent, options);
|
|
2908
|
-
const binding = selectWorkflowBinding(
|
|
2960
|
+
const { binding, confidence } = selectWorkflowBinding(
|
|
2909
2961
|
basename5(file),
|
|
2910
2962
|
runBindings,
|
|
2911
2963
|
firstSubagentTimestamp(agent, parsed),
|
|
2912
2964
|
meta.toolUseId
|
|
2913
2965
|
);
|
|
2914
2966
|
agent.parentToolUseId = binding.toolUseId;
|
|
2967
|
+
agent.parentConfidence = confidence;
|
|
2915
2968
|
}
|
|
2916
2969
|
agents.push(agent);
|
|
2917
2970
|
}
|
|
@@ -2968,10 +3021,14 @@ var ClaudeAdapter = class {
|
|
|
2968
3021
|
const parentAgentMissing = Boolean(
|
|
2969
3022
|
agent.meta.parentAgentId && !parsedByAgentId.has(agent.meta.parentAgentId)
|
|
2970
3023
|
);
|
|
3024
|
+
const confidence = parentSpan ? agent.parentConfidence ?? "correlated" : "unknown";
|
|
2971
3025
|
const task = subagentTaskName(agent);
|
|
2972
3026
|
const lifecycle = subagentLifecycleSpan(agent, parsed, traceId, parent, task, subDir);
|
|
2973
3027
|
for (const item of [...parsed.spans, ...lifecycle ? [lifecycle] : []]) {
|
|
2974
|
-
if (item.parent_span_id === `root:${traceId}`
|
|
3028
|
+
if (item.parent_span_id === `root:${traceId}` || item === lifecycle) {
|
|
3029
|
+
item.parent_span_id = parent;
|
|
3030
|
+
item.attributes[ATTR3.parentConfidence] = confidence;
|
|
3031
|
+
}
|
|
2975
3032
|
item.attributes[SUBAGENT_SPAN_ATTR] = true;
|
|
2976
3033
|
if (task) item.attributes[SUBAGENT_TASK_ATTR] = task;
|
|
2977
3034
|
if (agent.parentToolUseId && !parentSpan) {
|
|
@@ -3787,9 +3844,9 @@ function itemCountsJson(counts) {
|
|
|
3787
3844
|
}
|
|
3788
3845
|
function placeInnerSpans(pending, windows) {
|
|
3789
3846
|
return pending.map((entry) => {
|
|
3790
|
-
const { parent, join:
|
|
3847
|
+
const { parent, join: join34 } = joinItem(windows, entry.startMs, entry.endMs);
|
|
3791
3848
|
if (parent) entry.span.parent_span_id = parent.span_id;
|
|
3792
|
-
entry.span.attributes["traces.codex.item_join"] =
|
|
3849
|
+
entry.span.attributes["traces.codex.item_join"] = join34;
|
|
3793
3850
|
return entry.span;
|
|
3794
3851
|
});
|
|
3795
3852
|
}
|
|
@@ -6434,9 +6491,9 @@ function normalizeRemote(url) {
|
|
|
6434
6491
|
}
|
|
6435
6492
|
async function readGit(cwd) {
|
|
6436
6493
|
await stat12(`${cwd}/.git`);
|
|
6437
|
-
const { execFile
|
|
6438
|
-
const { promisify
|
|
6439
|
-
const run =
|
|
6494
|
+
const { execFile } = await import("child_process");
|
|
6495
|
+
const { promisify } = await import("util");
|
|
6496
|
+
const run = promisify(execFile);
|
|
6440
6497
|
const git = async (args) => {
|
|
6441
6498
|
try {
|
|
6442
6499
|
const { stdout } = await run("git", ["-C", cwd, ...args], { timeout: 5e3 });
|
|
@@ -6462,9 +6519,9 @@ async function readGit(cwd) {
|
|
|
6462
6519
|
};
|
|
6463
6520
|
}
|
|
6464
6521
|
async function gitOutput(cwd, args) {
|
|
6465
|
-
const { execFile
|
|
6466
|
-
const { promisify
|
|
6467
|
-
const run =
|
|
6522
|
+
const { execFile } = await import("child_process");
|
|
6523
|
+
const { promisify } = await import("util");
|
|
6524
|
+
const run = promisify(execFile);
|
|
6468
6525
|
try {
|
|
6469
6526
|
const { stdout } = await run("git", ["-C", cwd, ...args], { timeout: 5e3 });
|
|
6470
6527
|
const v = stdout.trim();
|
|
@@ -7530,11 +7587,11 @@ function decodeEvidenceRef(value, label, knownSpanUris) {
|
|
|
7530
7587
|
}
|
|
7531
7588
|
const uri = nonEmptyString3(ref.uri, `${label}.uri`);
|
|
7532
7589
|
if (kind === "span") validateSpanEvidence(uri, knownSpanUris, label);
|
|
7533
|
-
const
|
|
7590
|
+
const excerpt = optionalString(ref.excerpt, `${label}.excerpt`);
|
|
7534
7591
|
return {
|
|
7535
7592
|
kind,
|
|
7536
7593
|
uri,
|
|
7537
|
-
...
|
|
7594
|
+
...excerpt === void 0 ? {} : { excerpt }
|
|
7538
7595
|
};
|
|
7539
7596
|
}
|
|
7540
7597
|
function decodeFinding(value, label, knownSpanUris) {
|
|
@@ -7741,14 +7798,14 @@ function normalizeFinding(row, analystId, index, context) {
|
|
|
7741
7798
|
to: uri
|
|
7742
7799
|
});
|
|
7743
7800
|
}
|
|
7744
|
-
const
|
|
7745
|
-
if (
|
|
7801
|
+
const excerpt = citation.excerpt === void 0 ? void 0 : normalizeExcerpt(citation.excerpt, span2);
|
|
7802
|
+
if (excerpt !== citation.excerpt) {
|
|
7746
7803
|
context.log?.("finding citation normalized: json-escaped excerpt", {
|
|
7747
7804
|
analyst_id: analystId,
|
|
7748
7805
|
uri
|
|
7749
7806
|
});
|
|
7750
7807
|
}
|
|
7751
|
-
return
|
|
7808
|
+
return excerpt === void 0 ? { uri } : { uri, excerpt };
|
|
7752
7809
|
});
|
|
7753
7810
|
const normalized = { ...row, evidence: evidence3 };
|
|
7754
7811
|
const allowed = KIND_EXPECTED_SUBJECTS[analystId];
|
|
@@ -7797,10 +7854,10 @@ function decodeComponent(value) {
|
|
|
7797
7854
|
return value;
|
|
7798
7855
|
}
|
|
7799
7856
|
}
|
|
7800
|
-
function normalizeExcerpt(
|
|
7801
|
-
if (containsText(span2.content,
|
|
7802
|
-
const decoded = decodeJsonEscapes(
|
|
7803
|
-
return decoded !==
|
|
7857
|
+
function normalizeExcerpt(excerpt, span2) {
|
|
7858
|
+
if (containsText(span2.content, excerpt)) return excerpt;
|
|
7859
|
+
const decoded = decodeJsonEscapes(excerpt);
|
|
7860
|
+
return decoded !== excerpt && containsText(span2.content, decoded) ? decoded : excerpt;
|
|
7804
7861
|
}
|
|
7805
7862
|
function decodeJsonEscapes(text2) {
|
|
7806
7863
|
return text2.replace(
|
|
@@ -9118,13 +9175,13 @@ import { OtlpFileTraceStore } from "@tangle-network/agent-eval/traces";
|
|
|
9118
9175
|
import { isUtf8 as isUtf82 } from "buffer";
|
|
9119
9176
|
import { createHash as createHash9 } from "crypto";
|
|
9120
9177
|
import { constants } from "fs";
|
|
9121
|
-
import { lstat, open, realpath as realpath3 } from "fs/promises";
|
|
9178
|
+
import { lstat as lstat2, open, realpath as realpath3 } from "fs/promises";
|
|
9122
9179
|
import { dirname as dirname8, isAbsolute as isAbsolute4, join as join20, relative as relative5, resolve as resolve7, sep as sep2 } from "path";
|
|
9123
9180
|
|
|
9124
9181
|
// src/bundle.ts
|
|
9125
9182
|
import { createHash as createHash8 } from "crypto";
|
|
9126
9183
|
import { readFileSync as readFileSync2 } from "fs";
|
|
9127
|
-
import { copyFile, cp, mkdir as mkdir2, readdir as readdir16, readFile as readFile7, stat as stat14, writeFile as writeFile5 } from "fs/promises";
|
|
9184
|
+
import { copyFile, cp, lstat, mkdir as mkdir2, readdir as readdir16, readFile as readFile7, stat as stat14, writeFile as writeFile5 } from "fs/promises";
|
|
9128
9185
|
import { dirname as dirname7, join as join19, relative as relative4, resolve as resolve6, sep } from "path";
|
|
9129
9186
|
|
|
9130
9187
|
// src/improvement.ts
|
|
@@ -9444,7 +9501,7 @@ function validationExitCode(validation) {
|
|
|
9444
9501
|
|
|
9445
9502
|
// src/loop-analysis.ts
|
|
9446
9503
|
import {
|
|
9447
|
-
ATTR as
|
|
9504
|
+
ATTR as ATTR4,
|
|
9448
9505
|
firstNumberAttr,
|
|
9449
9506
|
firstStringAttr,
|
|
9450
9507
|
ITERATION_ATTR_KEYS,
|
|
@@ -9458,11 +9515,11 @@ function attributes(span2) {
|
|
|
9458
9515
|
return span2.attributes;
|
|
9459
9516
|
}
|
|
9460
9517
|
function outcomeOf(span2) {
|
|
9461
|
-
const raw = firstStringAttr(attributes(span2), [
|
|
9518
|
+
const raw = firstStringAttr(attributes(span2), [ATTR4.outcome]);
|
|
9462
9519
|
return raw !== void 0 && OUTCOMES.includes(raw) ? raw : null;
|
|
9463
9520
|
}
|
|
9464
9521
|
function scoreOf(span2) {
|
|
9465
|
-
return firstNumberAttr(attributes(span2), [
|
|
9522
|
+
return firstNumberAttr(attributes(span2), [ATTR4.score]) ?? null;
|
|
9466
9523
|
}
|
|
9467
9524
|
function iterationOf(span2) {
|
|
9468
9525
|
const value = firstNumberAttr(attributes(span2), ITERATION_ATTR_KEYS);
|
|
@@ -9535,7 +9592,7 @@ function trendOf(iterations) {
|
|
|
9535
9592
|
return {
|
|
9536
9593
|
trend: "unscored",
|
|
9537
9594
|
basis: "none",
|
|
9538
|
-
detail: `${iterations.length} round(s) recorded, ${scored.length + graded.length} graded: the trace carries neither ${
|
|
9595
|
+
detail: `${iterations.length} round(s) recorded, ${scored.length + graded.length} graded: the trace carries neither ${ATTR4.outcome} nor ${ATTR4.score} on enough rounds to compare them`
|
|
9539
9596
|
};
|
|
9540
9597
|
}
|
|
9541
9598
|
function analyzeLoopConvergence(spans) {
|
|
@@ -9546,7 +9603,7 @@ function analyzeLoopConvergence(spans) {
|
|
|
9546
9603
|
const iteration = iterationOf(span2);
|
|
9547
9604
|
if (iteration === null) continue;
|
|
9548
9605
|
iterationSpans += 1;
|
|
9549
|
-
const loopId = firstStringAttr(attributes(span2), [
|
|
9606
|
+
const loopId = firstStringAttr(attributes(span2), [ATTR4.loopId]) ?? `(unnamed loop in ${span2.trace_id})`;
|
|
9550
9607
|
const key = `${span2.trace_id}\0${loopId}`;
|
|
9551
9608
|
const verdict = verdictFor(span2, children);
|
|
9552
9609
|
const bucket = byLoop.get(key) ?? { traceId: span2.trace_id, iterations: [] };
|
|
@@ -10641,8 +10698,8 @@ function renderReport(result, meta) {
|
|
|
10641
10698
|
if (f.recommended_action) lines.push(`- **Fix:** ${f.recommended_action}`);
|
|
10642
10699
|
if (f.validation_plan) lines.push(`- **Check:** ${f.validation_plan}`);
|
|
10643
10700
|
for (const ev of f.evidence_refs.slice(0, 3)) {
|
|
10644
|
-
const
|
|
10645
|
-
lines.push(`- **Evidence:** ${ev.kind} ${ev.uri}${
|
|
10701
|
+
const excerpt = ev.excerpt ? ` \u2014 \`${ev.excerpt.slice(0, 200)}\`` : "";
|
|
10702
|
+
lines.push(`- **Evidence:** ${ev.kind} ${ev.uri}${excerpt}`);
|
|
10646
10703
|
}
|
|
10647
10704
|
lines.push("");
|
|
10648
10705
|
}
|
|
@@ -10966,8 +11023,8 @@ function jsonl(rows) {
|
|
|
10966
11023
|
return rows.length === 0 ? "" : `${rows.map((row) => JSON.stringify(row)).join("\n")}
|
|
10967
11024
|
`;
|
|
10968
11025
|
}
|
|
10969
|
-
function evidence(kind, uri,
|
|
10970
|
-
return
|
|
11026
|
+
function evidence(kind, uri, excerpt) {
|
|
11027
|
+
return excerpt ? { kind, uri, excerpt } : { kind, uri };
|
|
10971
11028
|
}
|
|
10972
11029
|
function totalReactionSignals(reactions) {
|
|
10973
11030
|
return Object.values(reactions.signals).reduce((sum, count2) => sum + count2, 0);
|
|
@@ -11679,8 +11736,8 @@ async function collectContextRoot(root) {
|
|
|
11679
11736
|
if (summary) files.push(summary);
|
|
11680
11737
|
}
|
|
11681
11738
|
}
|
|
11682
|
-
const
|
|
11683
|
-
if (
|
|
11739
|
+
const progress2 = await summarizeFile(join18(evolve, "progress.md"), "other");
|
|
11740
|
+
if (progress2) files.push(progress2);
|
|
11684
11741
|
return { root, files: files.sort((a, b) => a.path.localeCompare(b.path)) };
|
|
11685
11742
|
}
|
|
11686
11743
|
async function listEvolveRootFiles(evolve) {
|
|
@@ -12209,6 +12266,9 @@ var KNOWN_LIMITS = [
|
|
|
12209
12266
|
"progress.md is copied whole; no session-dated section extraction is performed",
|
|
12210
12267
|
"open pull-request state is not captured: the bundler makes no network calls"
|
|
12211
12268
|
];
|
|
12269
|
+
function sha256Hex(bytes) {
|
|
12270
|
+
return createHash8("sha256").update(bytes).digest("hex");
|
|
12271
|
+
}
|
|
12212
12272
|
function bundlePath(...parts) {
|
|
12213
12273
|
return parts.join("/");
|
|
12214
12274
|
}
|
|
@@ -12244,9 +12304,9 @@ function sliceJsonlByWindow(text2, windowStartMs, windowEndMs) {
|
|
|
12244
12304
|
return { kept, totalRows, unparseableTsRows };
|
|
12245
12305
|
}
|
|
12246
12306
|
async function runGitLog(cwd, sinceIso, untilIso) {
|
|
12247
|
-
const { execFile
|
|
12248
|
-
const { promisify
|
|
12249
|
-
const run =
|
|
12307
|
+
const { execFile } = await import("child_process");
|
|
12308
|
+
const { promisify } = await import("util");
|
|
12309
|
+
const run = promisify(execFile);
|
|
12250
12310
|
try {
|
|
12251
12311
|
const { stdout } = await run(
|
|
12252
12312
|
"git",
|
|
@@ -12549,6 +12609,90 @@ async function readSessionBundleManifest(bundleDir) {
|
|
|
12549
12609
|
}
|
|
12550
12610
|
return manifest;
|
|
12551
12611
|
}
|
|
12612
|
+
function unsafeBundlePath(path) {
|
|
12613
|
+
if (path.length === 0) return "empty path";
|
|
12614
|
+
if (path.includes("\\")) return "contains a backslash";
|
|
12615
|
+
if (path.startsWith("/") || /^[A-Za-z]:/.test(path)) return "is absolute";
|
|
12616
|
+
if (path.split("/").some((part) => part === ".." || part === "." || part === "")) {
|
|
12617
|
+
return 'has an empty, "." or ".." segment';
|
|
12618
|
+
}
|
|
12619
|
+
return null;
|
|
12620
|
+
}
|
|
12621
|
+
async function nonRegularEntries(root) {
|
|
12622
|
+
const out = [];
|
|
12623
|
+
const pending = [resolve6(root)];
|
|
12624
|
+
while (pending.length > 0) {
|
|
12625
|
+
const dir = pending.pop();
|
|
12626
|
+
for (const entry of await readdir16(dir, { withFileTypes: true })) {
|
|
12627
|
+
const path = join19(dir, entry.name);
|
|
12628
|
+
if (entry.isDirectory()) pending.push(path);
|
|
12629
|
+
else if (!entry.isFile()) out.push(relative4(resolve6(root), path).split(sep).join("/"));
|
|
12630
|
+
}
|
|
12631
|
+
}
|
|
12632
|
+
return out.sort();
|
|
12633
|
+
}
|
|
12634
|
+
async function verifySessionBundle(bundleDir) {
|
|
12635
|
+
const directory = resolve6(bundleDir);
|
|
12636
|
+
const manifest = await readSessionBundleManifest(directory);
|
|
12637
|
+
const issues = [];
|
|
12638
|
+
const listed = /* @__PURE__ */ new Set();
|
|
12639
|
+
let matched = 0;
|
|
12640
|
+
for (const file of manifest.files) {
|
|
12641
|
+
const unsafe = unsafeBundlePath(file.path);
|
|
12642
|
+
if (unsafe) {
|
|
12643
|
+
issues.push({ kind: "unsafe-path", path: file.path, detail: `manifest path ${unsafe}` });
|
|
12644
|
+
continue;
|
|
12645
|
+
}
|
|
12646
|
+
listed.add(file.path);
|
|
12647
|
+
const info = await lstat(join19(directory, file.path)).catch((error) => {
|
|
12648
|
+
if (isMissingPathError(error)) return null;
|
|
12649
|
+
throw error;
|
|
12650
|
+
});
|
|
12651
|
+
if (!info) {
|
|
12652
|
+
issues.push({ kind: "missing", path: file.path, detail: "listed in the manifest, not on disk" });
|
|
12653
|
+
continue;
|
|
12654
|
+
}
|
|
12655
|
+
if (!info.isFile()) continue;
|
|
12656
|
+
const sha2562 = await fileSha256(join19(directory, file.path));
|
|
12657
|
+
if (info.size !== file.bytes || sha2562 !== file.sha256) {
|
|
12658
|
+
issues.push({
|
|
12659
|
+
kind: "mismatch",
|
|
12660
|
+
path: file.path,
|
|
12661
|
+
detail: `recorded ${file.bytes} bytes sha256 ${file.sha256}, found ${info.size} bytes sha256 ${sha2562}`
|
|
12662
|
+
});
|
|
12663
|
+
continue;
|
|
12664
|
+
}
|
|
12665
|
+
matched += 1;
|
|
12666
|
+
}
|
|
12667
|
+
for (const path of await listSessionBundleFiles(directory)) {
|
|
12668
|
+
if (path === "manifest.json" || listed.has(path)) continue;
|
|
12669
|
+
issues.push({ kind: "unexpected", path, detail: "on disk, not listed in the manifest" });
|
|
12670
|
+
}
|
|
12671
|
+
for (const path of await nonRegularEntries(directory)) {
|
|
12672
|
+
issues.push({ kind: "not-a-file", path, detail: "a symbolic link or special file; a bundle holds regular files only" });
|
|
12673
|
+
}
|
|
12674
|
+
let source;
|
|
12675
|
+
if (manifest.projection) {
|
|
12676
|
+
const sourceManifest = join19(manifest.projection.sourceDirectory, "manifest.json");
|
|
12677
|
+
const bytes = await readFile7(sourceManifest).catch((error) => {
|
|
12678
|
+
if (isMissingPathError(error)) return null;
|
|
12679
|
+
throw error;
|
|
12680
|
+
});
|
|
12681
|
+
source = {
|
|
12682
|
+
directory: manifest.projection.sourceDirectory,
|
|
12683
|
+
status: bytes === null ? "unavailable" : sha256Hex(bytes) === manifest.projection.sourceManifestSha256 ? "match" : "mismatch"
|
|
12684
|
+
};
|
|
12685
|
+
}
|
|
12686
|
+
return {
|
|
12687
|
+
ok: issues.length === 0 && source?.status !== "mismatch",
|
|
12688
|
+
directory,
|
|
12689
|
+
view: manifest.view,
|
|
12690
|
+
listed: manifest.files.length,
|
|
12691
|
+
matched,
|
|
12692
|
+
issues,
|
|
12693
|
+
...source ? { source } : {}
|
|
12694
|
+
};
|
|
12695
|
+
}
|
|
12552
12696
|
var cachedVersion;
|
|
12553
12697
|
function tracesPackageVersion() {
|
|
12554
12698
|
if (cachedVersion) return cachedVersion;
|
|
@@ -12578,7 +12722,7 @@ async function openBundleFile(root, file) {
|
|
|
12578
12722
|
let current = root;
|
|
12579
12723
|
for (const part of relative5(root, path).split(sep2)) {
|
|
12580
12724
|
current = join20(current, part);
|
|
12581
|
-
if ((await
|
|
12725
|
+
if ((await lstat2(current)).isSymbolicLink()) throw new Error("bundle source symlinks are not permitted");
|
|
12582
12726
|
}
|
|
12583
12727
|
if (await realpath3(path) !== path) throw new Error("bundle source resolved outside its declared path");
|
|
12584
12728
|
const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
@@ -12614,9 +12758,9 @@ async function createBundleSourceReader(directory, spans, options = {}) {
|
|
|
12614
12758
|
const maxRecordBytes = options.maxRecordBytes ?? 16 * 1024 * 1024;
|
|
12615
12759
|
if (!Number.isSafeInteger(maxRecordBytes) || maxRecordBytes < 1) throw new Error("maxRecordBytes must be a positive safe integer");
|
|
12616
12760
|
options.signal?.throwIfAborted();
|
|
12617
|
-
if ((await
|
|
12761
|
+
if ((await lstat2(resolve7(directory))).isSymbolicLink()) throw new Error("bundle directory must not be a symlink");
|
|
12618
12762
|
const root = await realpath3(resolve7(directory));
|
|
12619
|
-
if ((await
|
|
12763
|
+
if ((await lstat2(join20(root, "manifest.json"))).isSymbolicLink()) throw new Error("bundle manifest must not be a symlink");
|
|
12620
12764
|
const manifest = await readSessionBundleManifest(root);
|
|
12621
12765
|
if (manifest.view !== "full") throw new Error("original source reads require an explicitly authorized full bundle");
|
|
12622
12766
|
if (!Array.isArray(manifest.files)) throw new Error("bundle manifest has no file receipts");
|
|
@@ -13042,7 +13186,7 @@ import { basename as basename14, dirname as dirname10, join as join23, resolve a
|
|
|
13042
13186
|
import { createHash as createHash11, randomUUID } from "crypto";
|
|
13043
13187
|
import {
|
|
13044
13188
|
link,
|
|
13045
|
-
lstat as
|
|
13189
|
+
lstat as lstat3,
|
|
13046
13190
|
mkdir as mkdir3,
|
|
13047
13191
|
readFile as readFile8,
|
|
13048
13192
|
realpath as realpath4,
|
|
@@ -13211,7 +13355,7 @@ async function optionalContainedFile(root, candidate) {
|
|
|
13211
13355
|
}
|
|
13212
13356
|
async function assertPathMissing(path) {
|
|
13213
13357
|
try {
|
|
13214
|
-
await
|
|
13358
|
+
await lstat3(path);
|
|
13215
13359
|
} catch (error) {
|
|
13216
13360
|
if (isErrno(error, "ENOENT")) return;
|
|
13217
13361
|
throw error;
|
|
@@ -13725,54 +13869,18 @@ function validateRevision(value) {
|
|
|
13725
13869
|
return value;
|
|
13726
13870
|
}
|
|
13727
13871
|
|
|
13728
|
-
// src/replay
|
|
13729
|
-
import {
|
|
13730
|
-
import { join as join24 } from "path";
|
|
13872
|
+
// src/replay.ts
|
|
13873
|
+
import { readFileSync as readFileSync3 } from "fs";
|
|
13731
13874
|
import {
|
|
13732
|
-
|
|
13733
|
-
|
|
13734
|
-
|
|
13735
|
-
|
|
13875
|
+
enumerateReplayableCases,
|
|
13876
|
+
parseCorpusFlag,
|
|
13877
|
+
readFindingsFile,
|
|
13878
|
+
replayVerify,
|
|
13879
|
+
runReplayBatch,
|
|
13880
|
+
verifyFindings
|
|
13881
|
+
} from "@tangle-network/agent-eval/trajectory-replay";
|
|
13736
13882
|
import { Sandbox } from "@tangle-network/sandbox";
|
|
13737
|
-
|
|
13738
|
-
if (!observation) return null;
|
|
13739
|
-
const m = /<returncode>(-?\d+)<\/returncode>/.exec(observation);
|
|
13740
|
-
return m ? Number(m[1]) : null;
|
|
13741
|
-
}
|
|
13742
|
-
function parseObservationOutput(observation) {
|
|
13743
|
-
if (!observation) return "";
|
|
13744
|
-
const m = /<output>\n?([\s\S]*?)\n?<\/output>/.exec(observation);
|
|
13745
|
-
return m ? m[1] : observation;
|
|
13746
|
-
}
|
|
13747
|
-
function deriveFailureSignature(observation) {
|
|
13748
|
-
const line = parseObservationOutput(observation).split("\n").find((l) => /\berror\b/i.test(l));
|
|
13749
|
-
return line ? line.trim().slice(0, 200) : null;
|
|
13750
|
-
}
|
|
13751
|
-
async function ingestCodeTraceBenchSteps(store, steps, caseId) {
|
|
13752
|
-
let tick = 0;
|
|
13753
|
-
const emitter = new TraceEmitter(store, { now: () => ++tick });
|
|
13754
|
-
await emitter.startRun({
|
|
13755
|
-
scenarioId: caseId,
|
|
13756
|
-
tags: { source: "codetracebench-replay", caseId }
|
|
13757
|
-
});
|
|
13758
|
-
for (const step of steps) {
|
|
13759
|
-
if (typeof step.action !== "string" || step.action.length === 0) {
|
|
13760
|
-
throw new Error(`replay-verify: step ${step.step_id} has no action`);
|
|
13761
|
-
}
|
|
13762
|
-
const handle = await emitter.tool({
|
|
13763
|
-
name: `step:${step.step_id}`,
|
|
13764
|
-
toolName: "shell",
|
|
13765
|
-
args: { command: step.action },
|
|
13766
|
-
result: {
|
|
13767
|
-
returncode: parseRecordedReturncode(step.observation),
|
|
13768
|
-
observation: step.observation
|
|
13769
|
-
}
|
|
13770
|
-
});
|
|
13771
|
-
await handle.end();
|
|
13772
|
-
}
|
|
13773
|
-
await emitter.endRun({ pass: true, notes: "recorded trajectory (ingested for replay)" });
|
|
13774
|
-
return { runId: emitter.runId, stepCount: steps.length };
|
|
13775
|
-
}
|
|
13883
|
+
var DEFAULT_SANDBOX_BASE_URL = "http://127.0.0.1:4097";
|
|
13776
13884
|
function sandboxReplayBackend(options) {
|
|
13777
13885
|
const client = new Sandbox({ apiKey: options.apiKey, baseUrl: options.baseUrl });
|
|
13778
13886
|
return {
|
|
@@ -13797,365 +13905,117 @@ function sandboxReplayBackend(options) {
|
|
|
13797
13905
|
}
|
|
13798
13906
|
};
|
|
13799
13907
|
}
|
|
13800
|
-
function
|
|
13801
|
-
|
|
13802
|
-
|
|
13803
|
-
|
|
13804
|
-
|
|
13805
|
-
|
|
13806
|
-
|
|
13807
|
-
|
|
13808
|
-
throw new Error(`replay-verify: span ${span2.name} carries no command`);
|
|
13809
|
-
}
|
|
13810
|
-
return args.command;
|
|
13811
|
-
}
|
|
13812
|
-
function recordedReturncodeOf(span2) {
|
|
13813
|
-
const result = span2.result;
|
|
13814
|
-
return typeof result?.returncode === "number" ? result.returncode : null;
|
|
13815
|
-
}
|
|
13816
|
-
var SandboxCounterfactualRunner = class {
|
|
13817
|
-
constructor(backend, options) {
|
|
13818
|
-
this.backend = backend;
|
|
13819
|
-
this.options = options;
|
|
13820
|
-
}
|
|
13821
|
-
backend;
|
|
13822
|
-
options;
|
|
13823
|
-
lastPrefix = null;
|
|
13824
|
-
lastArm = null;
|
|
13825
|
-
async executeFrom(ctx, emitter) {
|
|
13826
|
-
const { cwd, stepTimeoutMs, prefixLimit, onProgress } = this.options;
|
|
13827
|
-
const session = await this.backend.open();
|
|
13828
|
-
try {
|
|
13829
|
-
const toolSteps = ctx.prefix.filter(
|
|
13830
|
-
(s) => s.span.kind === "tool"
|
|
13831
|
-
);
|
|
13832
|
-
const toExecute = prefixLimit !== void 0 ? toolSteps.slice(0, prefixLimit) : toolSteps;
|
|
13833
|
-
const divergences = [];
|
|
13834
|
-
const prefixStart = Date.now();
|
|
13835
|
-
for (const step of toExecute) {
|
|
13836
|
-
const command = toolCommand(step.span);
|
|
13837
|
-
const expected = recordedReturncodeOf(step.span);
|
|
13838
|
-
const started = Date.now();
|
|
13839
|
-
const result = await session.exec(wrapActionForExec(command, cwd), stepTimeoutMs);
|
|
13840
|
-
const handle = await emitter.sandbox({
|
|
13841
|
-
name: step.span.name,
|
|
13842
|
-
command,
|
|
13843
|
-
exitCode: result.exitCode,
|
|
13844
|
-
wallMs: Date.now() - started
|
|
13845
|
-
});
|
|
13846
|
-
await handle.end();
|
|
13847
|
-
if (expected !== null && expected !== result.exitCode) {
|
|
13848
|
-
divergences.push({
|
|
13849
|
-
step: step.index + 1,
|
|
13850
|
-
expectedReturncode: expected,
|
|
13851
|
-
actualExit: result.exitCode
|
|
13852
|
-
});
|
|
13853
|
-
}
|
|
13854
|
-
onProgress?.(
|
|
13855
|
-
`prefix ${step.span.name}: exit=${result.exitCode}` + (expected !== null && expected !== result.exitCode ? ` (recorded ${expected})` : "")
|
|
13856
|
-
);
|
|
13857
|
-
}
|
|
13858
|
-
this.lastPrefix = {
|
|
13859
|
-
prefixExecuted: toExecute.length,
|
|
13860
|
-
prefixDivergences: divergences,
|
|
13861
|
-
wallMs: Date.now() - prefixStart
|
|
13862
|
-
};
|
|
13863
|
-
if (ctx.mutatedStep.span.kind !== "tool") {
|
|
13864
|
-
throw new Error("replay-verify: mutation target is not a tool span");
|
|
13865
|
-
}
|
|
13866
|
-
const armCommand = toolCommand(ctx.mutatedStep.span);
|
|
13867
|
-
const armStart = Date.now();
|
|
13868
|
-
const armResult = await session.exec(wrapActionForExec(armCommand, cwd), stepTimeoutMs);
|
|
13869
|
-
const armWallMs = Date.now() - armStart;
|
|
13870
|
-
const armHandle = await emitter.sandbox({
|
|
13871
|
-
name: `candidate:${ctx.mutatedStep.span.name}`,
|
|
13872
|
-
command: armCommand,
|
|
13873
|
-
exitCode: armResult.exitCode,
|
|
13874
|
-
wallMs: armWallMs
|
|
13875
|
-
});
|
|
13876
|
-
await armHandle.end();
|
|
13877
|
-
this.lastArm = {
|
|
13878
|
-
command: armCommand,
|
|
13879
|
-
exitCode: armResult.exitCode,
|
|
13880
|
-
stdout: armResult.stdout,
|
|
13881
|
-
stderr: armResult.stderr,
|
|
13882
|
-
wallMs: armWallMs
|
|
13883
|
-
};
|
|
13884
|
-
onProgress?.(`candidate step: exit=${armResult.exitCode} in ${armWallMs}ms`);
|
|
13885
|
-
await emitter.endRun({
|
|
13886
|
-
pass: armResult.exitCode === 0,
|
|
13887
|
-
notes: `prefix ${toExecute.length} steps, ${divergences.length} divergences`
|
|
13888
|
-
});
|
|
13889
|
-
} finally {
|
|
13890
|
-
await session.close();
|
|
13891
|
-
}
|
|
13892
|
-
}
|
|
13893
|
-
};
|
|
13894
|
-
function excerpt(text2, limit = 2400) {
|
|
13895
|
-
if (text2.length <= limit) return text2;
|
|
13896
|
-
const head = text2.slice(0, limit / 2);
|
|
13897
|
-
const tail = text2.slice(-limit / 2);
|
|
13898
|
-
return `${head}
|
|
13899
|
-
\u2026 [${text2.length - limit} chars elided] \u2026
|
|
13900
|
-
${tail}`;
|
|
13901
|
-
}
|
|
13902
|
-
async function replayVerify(options) {
|
|
13903
|
-
const totalStart = Date.now();
|
|
13904
|
-
const steps = JSON.parse(readFileSync3(options.stepsPath, "utf8"));
|
|
13905
|
-
if (!Array.isArray(steps) || steps.length === 0) {
|
|
13906
|
-
throw new Error(`replay-verify: ${options.stepsPath} is not a non-empty steps array`);
|
|
13907
|
-
}
|
|
13908
|
-
const index = options.at - 1;
|
|
13909
|
-
if (index < 0 || index >= steps.length) {
|
|
13910
|
-
throw new Error(`replay-verify: --at ${options.at} out of range [1, ${steps.length}]`);
|
|
13911
|
-
}
|
|
13912
|
-
const target = steps[index];
|
|
13913
|
-
if (target.step_id !== options.at) {
|
|
13908
|
+
function sandboxBackendFactory(options) {
|
|
13909
|
+
return (image) => sandboxReplayBackend({ ...options, image });
|
|
13910
|
+
}
|
|
13911
|
+
async function assertSandboxReachable(baseUrl) {
|
|
13912
|
+
let response;
|
|
13913
|
+
try {
|
|
13914
|
+
response = await fetch(new URL("/health", baseUrl), { signal: AbortSignal.timeout(5e3) });
|
|
13915
|
+
} catch (err) {
|
|
13914
13916
|
throw new Error(
|
|
13915
|
-
`
|
|
13917
|
+
`verify-findings: sandbox API unreachable at ${baseUrl} \u2014 ${err instanceof Error ? err.message : String(err)}. Executing findings requires a running sandbox orchestrator + SDK adapter (see docs/replay-verify.md); start them or pass --base-url.`
|
|
13916
13918
|
);
|
|
13917
13919
|
}
|
|
13918
|
-
|
|
13919
|
-
|
|
13920
|
-
|
|
13921
|
-
const signatureBasis = signature ? "returncode+output-substring" : "returncode-only";
|
|
13922
|
-
const backend = options.backend ?? sandboxReplayBackend({
|
|
13923
|
-
image: options.image,
|
|
13924
|
-
apiKey: requireOption(options.apiKey, "apiKey"),
|
|
13925
|
-
baseUrl: requireOption(options.baseUrl, "baseUrl"),
|
|
13926
|
-
maxLifetimeSeconds: options.maxLifetimeSeconds
|
|
13927
|
-
});
|
|
13928
|
-
const runnerOptions = {
|
|
13929
|
-
cwd: options.cwd,
|
|
13930
|
-
stepTimeoutMs: options.stepTimeoutMs ?? 3e5,
|
|
13931
|
-
prefixLimit: options.prefixLimit,
|
|
13932
|
-
onProgress: options.onProgress
|
|
13933
|
-
};
|
|
13934
|
-
const store = new InMemoryTraceStore2();
|
|
13935
|
-
const { runId } = await ingestCodeTraceBenchSteps(store, steps, caseId);
|
|
13936
|
-
options.onProgress?.(`arm A: replaying ${index} prefix steps then recorded step ${options.at}`);
|
|
13937
|
-
const armARunner = new SandboxCounterfactualRunner(backend, runnerOptions);
|
|
13938
|
-
const armAStart = Date.now();
|
|
13939
|
-
const armAResult = await runCounterfactual(
|
|
13940
|
-
store,
|
|
13941
|
-
runId,
|
|
13942
|
-
{ kind: "custom", at: index, describe: "arm-A identity replay", apply: (s) => s },
|
|
13943
|
-
armARunner
|
|
13944
|
-
);
|
|
13945
|
-
const armAMs = Date.now() - armAStart;
|
|
13946
|
-
const armAExec = requireExec(armARunner, "arm A");
|
|
13947
|
-
const armAOutput = `${armAExec.stdout}
|
|
13948
|
-
${armAExec.stderr}`;
|
|
13949
|
-
const failureSignatureMatch = recordedReturncode !== null && armAExec.exitCode === recordedReturncode && (signature ? armAOutput.includes(signature) : true);
|
|
13950
|
-
let armBRunner = null;
|
|
13951
|
-
let armBExec = null;
|
|
13952
|
-
let armBRunId = null;
|
|
13953
|
-
let armBMs = null;
|
|
13954
|
-
if (options.fixCommand) {
|
|
13955
|
-
const fixCommand = options.fixCommand;
|
|
13956
|
-
options.onProgress?.(`arm B: replaying prefix then corrected step`);
|
|
13957
|
-
armBRunner = new SandboxCounterfactualRunner(backend, runnerOptions);
|
|
13958
|
-
const armBStart = Date.now();
|
|
13959
|
-
const armBResult = await runCounterfactual(
|
|
13960
|
-
store,
|
|
13961
|
-
runId,
|
|
13962
|
-
{
|
|
13963
|
-
kind: "custom",
|
|
13964
|
-
at: index,
|
|
13965
|
-
describe: "arm-B corrected step",
|
|
13966
|
-
apply: (step) => ({
|
|
13967
|
-
...step,
|
|
13968
|
-
span: { ...step.span, args: { command: fixCommand } }
|
|
13969
|
-
})
|
|
13970
|
-
},
|
|
13971
|
-
armBRunner
|
|
13920
|
+
if (!response.ok) {
|
|
13921
|
+
throw new Error(
|
|
13922
|
+
`verify-findings: sandbox API at ${baseUrl} answered /health with HTTP ${response.status} \u2014 refusing to run proofs against degraded infrastructure`
|
|
13972
13923
|
);
|
|
13973
|
-
|
|
13974
|
-
armBRunId = armBResult.counterfactualRunId;
|
|
13975
|
-
armBExec = requireExec(armBRunner, "arm B");
|
|
13976
|
-
}
|
|
13977
|
-
const armBOutput = armBExec ? `${armBExec.stdout}
|
|
13978
|
-
${armBExec.stderr}` : null;
|
|
13979
|
-
const verdict = {
|
|
13980
|
-
case: caseId,
|
|
13981
|
-
image: options.image,
|
|
13982
|
-
driver: options.driverLabel ?? "docker",
|
|
13983
|
-
k: options.at,
|
|
13984
|
-
cwd: options.cwd,
|
|
13985
|
-
recordedReturncode,
|
|
13986
|
-
signature,
|
|
13987
|
-
signatureBasis,
|
|
13988
|
-
prefixExecuted: armARunner.lastPrefix?.prefixExecuted ?? 0,
|
|
13989
|
-
prefixDivergences: armARunner.lastPrefix?.prefixDivergences ?? [],
|
|
13990
|
-
armA: {
|
|
13991
|
-
command: armAExec.command,
|
|
13992
|
-
exitCode: armAExec.exitCode,
|
|
13993
|
-
wallMs: armAExec.wallMs,
|
|
13994
|
-
prefix: armARunner.lastPrefix,
|
|
13995
|
-
failureSignatureMatch
|
|
13996
|
-
},
|
|
13997
|
-
armB: armBExec && armBRunner ? {
|
|
13998
|
-
command: armBExec.command,
|
|
13999
|
-
exitCode: armBExec.exitCode,
|
|
14000
|
-
wallMs: armBExec.wallMs,
|
|
14001
|
-
prefix: armBRunner.lastPrefix,
|
|
14002
|
-
failureVanished: armBExec.exitCode === 0 && (signature && armBOutput ? !armBOutput.includes(signature) : true)
|
|
14003
|
-
} : null,
|
|
14004
|
-
timings: { armAMs, armBMs, totalMs: Date.now() - totalStart },
|
|
14005
|
-
runIds: {
|
|
14006
|
-
original: runId,
|
|
14007
|
-
armA: armAResult.counterfactualRunId,
|
|
14008
|
-
armB: armBRunId
|
|
14009
|
-
}
|
|
14010
|
-
};
|
|
14011
|
-
mkdirSync(options.out, { recursive: true });
|
|
14012
|
-
writeFileSync(join24(options.out, "replay-verdict.json"), `${JSON.stringify(verdict, null, 2)}
|
|
14013
|
-
`);
|
|
14014
|
-
writeFileSync(
|
|
14015
|
-
join24(options.out, "report.md"),
|
|
14016
|
-
renderReport2(verdict, target, armAExec, armBExec)
|
|
14017
|
-
);
|
|
14018
|
-
return verdict;
|
|
14019
|
-
}
|
|
14020
|
-
function requireOption(value, name) {
|
|
14021
|
-
if (value === void 0) throw new Error(`replay-verify: ${name} is required`);
|
|
14022
|
-
return value;
|
|
14023
|
-
}
|
|
14024
|
-
function requireExec(runner, arm) {
|
|
14025
|
-
if (!runner.lastArm) throw new Error(`replay-verify: ${arm} finished without executing step k`);
|
|
14026
|
-
return runner.lastArm;
|
|
13924
|
+
}
|
|
14027
13925
|
}
|
|
14028
|
-
function
|
|
14029
|
-
|
|
14030
|
-
|
|
14031
|
-
|
|
14032
|
-
|
|
14033
|
-
|
|
14034
|
-
|
|
14035
|
-
|
|
14036
|
-
|
|
14037
|
-
|
|
14038
|
-
|
|
14039
|
-
|
|
14040
|
-
|
|
14041
|
-
|
|
14042
|
-
|
|
14043
|
-
|
|
14044
|
-
|
|
14045
|
-
|
|
14046
|
-
|
|
14047
|
-
|
|
14048
|
-
|
|
14049
|
-
|
|
14050
|
-
|
|
13926
|
+
function zaiChatCaller(options) {
|
|
13927
|
+
async function callOnce(system, user, maxTokens) {
|
|
13928
|
+
try {
|
|
13929
|
+
const response = await fetch(`${options.baseUrl.replace(/\/$/, "")}/chat/completions`, {
|
|
13930
|
+
method: "POST",
|
|
13931
|
+
headers: { authorization: `Bearer ${options.apiKey}`, "content-type": "application/json" },
|
|
13932
|
+
body: JSON.stringify({
|
|
13933
|
+
model: options.model,
|
|
13934
|
+
messages: [
|
|
13935
|
+
{ role: "system", content: system },
|
|
13936
|
+
{ role: "user", content: user }
|
|
13937
|
+
],
|
|
13938
|
+
temperature: 0,
|
|
13939
|
+
max_tokens: maxTokens
|
|
13940
|
+
}),
|
|
13941
|
+
signal: AbortSignal.timeout(options.timeoutMs ?? 18e4)
|
|
13942
|
+
});
|
|
13943
|
+
const bodyText = await response.text();
|
|
13944
|
+
if (!response.ok) return { succeeded: false, error: `HTTP ${response.status}: ${bodyText.slice(0, 400)}` };
|
|
13945
|
+
const body = JSON.parse(bodyText);
|
|
13946
|
+
if (body.error) return { succeeded: false, error: String(body.error.message ?? "provider error") };
|
|
13947
|
+
const content = body.choices?.[0]?.message?.content;
|
|
13948
|
+
if (typeof content !== "string" || content.length === 0) {
|
|
13949
|
+
return { succeeded: false, error: `empty completion content: ${bodyText.slice(0, 400)}` };
|
|
13950
|
+
}
|
|
13951
|
+
const promptTokens = body.usage?.prompt_tokens;
|
|
13952
|
+
const completionTokens = body.usage?.completion_tokens;
|
|
13953
|
+
const usage = typeof promptTokens === "number" && typeof completionTokens === "number" ? { promptTokens, completionTokens } : null;
|
|
13954
|
+
return { succeeded: true, value: { content, usage } };
|
|
13955
|
+
} catch (err) {
|
|
13956
|
+
return { succeeded: false, error: err instanceof Error ? err.message : String(err) };
|
|
14051
13957
|
}
|
|
14052
13958
|
}
|
|
14053
|
-
|
|
14054
|
-
|
|
14055
|
-
|
|
14056
|
-
|
|
14057
|
-
|
|
14058
|
-
|
|
14059
|
-
|
|
14060
|
-
|
|
14061
|
-
`Exit ${armA.exitCode} in ${armA.wallMs}ms \u2014 failureSignatureMatch: **${verdict.armA.failureSignatureMatch}**`
|
|
14062
|
-
);
|
|
14063
|
-
lines.push("");
|
|
14064
|
-
lines.push("Recorded observation excerpt:");
|
|
14065
|
-
lines.push("");
|
|
14066
|
-
lines.push("```");
|
|
14067
|
-
lines.push(excerpt(parseObservationOutput(target.observation)));
|
|
14068
|
-
lines.push("```");
|
|
14069
|
-
lines.push("");
|
|
14070
|
-
lines.push("Replayed stdout+stderr excerpt:");
|
|
14071
|
-
lines.push("");
|
|
14072
|
-
lines.push("```");
|
|
14073
|
-
lines.push(excerpt(`${armA.stdout}
|
|
14074
|
-
${armA.stderr}`.trim()));
|
|
14075
|
-
lines.push("```");
|
|
14076
|
-
if (armB && verdict.armB) {
|
|
14077
|
-
lines.push("");
|
|
14078
|
-
lines.push("## Arm B \u2014 corrected step k");
|
|
14079
|
-
lines.push("");
|
|
14080
|
-
lines.push("```sh");
|
|
14081
|
-
lines.push(armB.command);
|
|
14082
|
-
lines.push("```");
|
|
14083
|
-
lines.push("");
|
|
14084
|
-
lines.push(
|
|
14085
|
-
`Exit ${armB.exitCode} in ${armB.wallMs}ms \u2014 failureVanished: **${verdict.armB.failureVanished}** (prefix re-replayed in a fresh sandbox: ${verdict.armB.prefix.prefixExecuted} steps, ${verdict.armB.prefix.prefixDivergences.length} divergences)`
|
|
14086
|
-
);
|
|
14087
|
-
lines.push("");
|
|
14088
|
-
lines.push("Replayed stdout+stderr excerpt:");
|
|
14089
|
-
lines.push("");
|
|
14090
|
-
lines.push("```");
|
|
14091
|
-
lines.push(excerpt(`${armB.stdout}
|
|
14092
|
-
${armB.stderr}`.trim()));
|
|
14093
|
-
lines.push("```");
|
|
14094
|
-
}
|
|
14095
|
-
lines.push("");
|
|
14096
|
-
lines.push("## Timings");
|
|
14097
|
-
lines.push("");
|
|
14098
|
-
lines.push(
|
|
14099
|
-
`arm A ${verdict.timings.armAMs}ms, arm B ${verdict.timings.armBMs ?? "\u2014"}ms, total ${verdict.timings.totalMs}ms.`
|
|
14100
|
-
);
|
|
14101
|
-
lines.push("");
|
|
14102
|
-
return lines.join("\n");
|
|
13959
|
+
return {
|
|
13960
|
+
async complete(system, user) {
|
|
13961
|
+
const baseCap = options.maxTokens ?? 16384;
|
|
13962
|
+
const first = await callOnce(system, user, baseCap);
|
|
13963
|
+
if (first.succeeded || !/finish_reason":"length/.test(first.error)) return first;
|
|
13964
|
+
return callOnce(system, user, baseCap * 2);
|
|
13965
|
+
}
|
|
13966
|
+
};
|
|
14103
13967
|
}
|
|
14104
|
-
function
|
|
13968
|
+
function parseFlags2(command, argv, switches = []) {
|
|
14105
13969
|
const values = /* @__PURE__ */ new Map();
|
|
13970
|
+
const corpora = [];
|
|
13971
|
+
const seen = /* @__PURE__ */ new Set();
|
|
14106
13972
|
for (let i = 0; i < argv.length; i++) {
|
|
14107
13973
|
const arg = argv[i];
|
|
14108
13974
|
if (arg === "--help" || arg === "-h") return "help";
|
|
14109
|
-
if (
|
|
13975
|
+
if (switches.includes(arg)) {
|
|
13976
|
+
seen.add(arg);
|
|
13977
|
+
continue;
|
|
13978
|
+
}
|
|
13979
|
+
if (!arg.startsWith("--")) throw new Error(`${command}: unexpected positional argument: ${arg}`);
|
|
14110
13980
|
const value = argv[++i];
|
|
14111
|
-
if (value === void 0) throw new Error(
|
|
14112
|
-
|
|
13981
|
+
if (value === void 0) throw new Error(`${command}: missing value for ${arg}`);
|
|
13982
|
+
if (arg === "--corpus") corpora.push(parseCorpusFlag(value));
|
|
13983
|
+
else values.set(arg, value);
|
|
14113
13984
|
}
|
|
14114
|
-
|
|
14115
|
-
|
|
14116
|
-
|
|
14117
|
-
|
|
14118
|
-
|
|
14119
|
-
|
|
14120
|
-
|
|
14121
|
-
|
|
14122
|
-
|
|
14123
|
-
|
|
14124
|
-
|
|
13985
|
+
return { values, corpora, switches: seen };
|
|
13986
|
+
}
|
|
13987
|
+
function flagReader(command, values) {
|
|
13988
|
+
return {
|
|
13989
|
+
required(flag) {
|
|
13990
|
+
const v = values.get(flag);
|
|
13991
|
+
if (v === void 0) throw new Error(`${command}: ${flag} is required`);
|
|
13992
|
+
return v;
|
|
13993
|
+
},
|
|
13994
|
+
number(flag) {
|
|
13995
|
+
const v = values.get(flag);
|
|
13996
|
+
if (v === void 0) return void 0;
|
|
13997
|
+
const n = Number(v);
|
|
13998
|
+
if (!Number.isFinite(n)) throw new Error(`${command}: ${flag} must be a number, got ${v}`);
|
|
13999
|
+
return n;
|
|
14000
|
+
}
|
|
14125
14001
|
};
|
|
14126
|
-
|
|
14127
|
-
|
|
14128
|
-
const
|
|
14129
|
-
const
|
|
14130
|
-
|
|
14131
|
-
|
|
14132
|
-
throw new Error(`replay-verify: --at must be a positive step_id, got ${values.get("--at")}`);
|
|
14133
|
-
}
|
|
14134
|
-
let fixCommand = values.get("--fix-command");
|
|
14135
|
-
const fixFile = values.get("--fix-command-file");
|
|
14136
|
-
if (fixCommand !== void 0 && fixFile !== void 0) {
|
|
14137
|
-
throw new Error("replay-verify: pass --fix-command or --fix-command-file, not both");
|
|
14002
|
+
}
|
|
14003
|
+
function sandboxOptions(command, values, requireKey) {
|
|
14004
|
+
const apiKeyEnv = values.get("--api-key-env") ?? "SANDBOX_API_KEY";
|
|
14005
|
+
const apiKey = process.env[apiKeyEnv];
|
|
14006
|
+
if (requireKey && !apiKey) {
|
|
14007
|
+
throw new Error(`${command}: env var ${apiKeyEnv} is empty \u2014 pass --api-key-env or export it`);
|
|
14138
14008
|
}
|
|
14139
|
-
|
|
14009
|
+
const maxLifetime = flagReader(command, values).number("--max-lifetime");
|
|
14140
14010
|
return {
|
|
14141
|
-
|
|
14142
|
-
|
|
14143
|
-
|
|
14144
|
-
fixCommand,
|
|
14145
|
-
cwd,
|
|
14146
|
-
out,
|
|
14147
|
-
caseId: values.get("--case"),
|
|
14148
|
-
signature: values.get("--signature"),
|
|
14149
|
-
stepTimeoutMs: optionalNumber("--step-timeout"),
|
|
14150
|
-
prefixLimit: optionalNumber("--prefix-limit"),
|
|
14151
|
-
driverLabel: values.get("--driver-label"),
|
|
14152
|
-
baseUrl: values.get("--base-url") ?? process.env.SANDBOX_API_URL ?? "http://127.0.0.1:4097",
|
|
14153
|
-
apiKeyEnv: values.get("--api-key-env") ?? "SANDBOX_API_KEY",
|
|
14154
|
-
maxLifetimeSeconds: optionalNumber("--max-lifetime")
|
|
14011
|
+
apiKey: apiKey ?? "",
|
|
14012
|
+
baseUrl: values.get("--base-url") ?? process.env.SANDBOX_API_URL ?? DEFAULT_SANDBOX_BASE_URL,
|
|
14013
|
+
...maxLifetime === void 0 ? {} : { maxLifetimeSeconds: maxLifetime }
|
|
14155
14014
|
};
|
|
14156
14015
|
}
|
|
14157
|
-
|
|
14158
|
-
|
|
14016
|
+
var progress = (message) => process.stderr.write(`${message}
|
|
14017
|
+
`);
|
|
14018
|
+
var REPLAY_VERIFY_USAGE = `traces replay-verify \u2014 replay a trajectory prefix in a sandbox and execute the counterfactual at step k
|
|
14159
14019
|
|
|
14160
14020
|
Usage:
|
|
14161
14021
|
traces replay-verify --steps STEPS.json --image IMG --at K --cwd DIR --out DIR \\
|
|
@@ -14173,1928 +14033,153 @@ Usage:
|
|
|
14173
14033
|
line of the recorded observation containing "error")
|
|
14174
14034
|
--prefix-limit execute at most N prefix steps (weakens fidelity; reported)
|
|
14175
14035
|
--step-timeout per-exec timeout in ms (default 300000)
|
|
14176
|
-
--base-url sandbox API url (default $SANDBOX_API_URL or
|
|
14036
|
+
--base-url sandbox API url (default $SANDBOX_API_URL or ${DEFAULT_SANDBOX_BASE_URL})
|
|
14177
14037
|
--api-key-env env var holding the sandbox API key (default SANDBOX_API_KEY)
|
|
14178
14038
|
|
|
14179
14039
|
Requires a running sandbox orchestrator; only SWE-style trajectories that carry a
|
|
14180
14040
|
docker image are replayable. See docs/replay-verify.md for setup and limits.
|
|
14181
14041
|
`;
|
|
14182
|
-
}
|
|
14183
14042
|
async function cmdReplayVerify(argv) {
|
|
14184
|
-
const parsed =
|
|
14043
|
+
const parsed = parseFlags2("replay-verify", argv);
|
|
14185
14044
|
if (parsed === "help") {
|
|
14186
|
-
process.stdout.write(
|
|
14045
|
+
process.stdout.write(REPLAY_VERIFY_USAGE);
|
|
14187
14046
|
return;
|
|
14188
14047
|
}
|
|
14189
|
-
const
|
|
14190
|
-
|
|
14191
|
-
|
|
14192
|
-
|
|
14193
|
-
);
|
|
14048
|
+
const { values } = parsed;
|
|
14049
|
+
const flags = flagReader("replay-verify", values);
|
|
14050
|
+
const at = Number(flags.required("--at"));
|
|
14051
|
+
if (!Number.isInteger(at) || at < 1) {
|
|
14052
|
+
throw new Error(`replay-verify: --at must be a positive step_id, got ${values.get("--at")}`);
|
|
14053
|
+
}
|
|
14054
|
+
const fixFile = values.get("--fix-command-file");
|
|
14055
|
+
if (values.has("--fix-command") && fixFile !== void 0) {
|
|
14056
|
+
throw new Error("replay-verify: pass --fix-command or --fix-command-file, not both");
|
|
14194
14057
|
}
|
|
14058
|
+
const fixCommand = fixFile === void 0 ? values.get("--fix-command") : readFileSync3(fixFile, "utf8").trimEnd();
|
|
14059
|
+
const image = flags.required("--image");
|
|
14060
|
+
const out = flags.required("--out");
|
|
14195
14061
|
const verdict = await replayVerify({
|
|
14196
|
-
stepsPath:
|
|
14197
|
-
image
|
|
14198
|
-
at
|
|
14199
|
-
fixCommand
|
|
14200
|
-
cwd:
|
|
14201
|
-
out
|
|
14202
|
-
caseId:
|
|
14203
|
-
signature:
|
|
14204
|
-
stepTimeoutMs:
|
|
14205
|
-
prefixLimit:
|
|
14206
|
-
driverLabel:
|
|
14207
|
-
|
|
14208
|
-
|
|
14209
|
-
maxLifetimeSeconds: parsed.maxLifetimeSeconds,
|
|
14210
|
-
onProgress: (message) => process.stderr.write(`${message}
|
|
14211
|
-
`)
|
|
14062
|
+
stepsPath: flags.required("--steps"),
|
|
14063
|
+
image,
|
|
14064
|
+
at,
|
|
14065
|
+
fixCommand,
|
|
14066
|
+
cwd: flags.required("--cwd"),
|
|
14067
|
+
out,
|
|
14068
|
+
caseId: values.get("--case"),
|
|
14069
|
+
signature: values.get("--signature"),
|
|
14070
|
+
stepTimeoutMs: flags.number("--step-timeout"),
|
|
14071
|
+
prefixLimit: flags.number("--prefix-limit"),
|
|
14072
|
+
driverLabel: values.get("--driver-label"),
|
|
14073
|
+
backend: sandboxReplayBackend({ ...sandboxOptions("replay-verify", values, true), image }),
|
|
14074
|
+
onProgress: progress
|
|
14212
14075
|
});
|
|
14213
14076
|
process.stdout.write(`${JSON.stringify(verdict, null, 2)}
|
|
14214
14077
|
`);
|
|
14215
14078
|
const verdictLine = verdict.armB ? `armA.failureSignatureMatch=${verdict.armA.failureSignatureMatch} armB.failureVanished=${verdict.armB.failureVanished}` : `armA.failureSignatureMatch=${verdict.armA.failureSignatureMatch}`;
|
|
14216
|
-
|
|
14217
|
-
`replay-verify: ${verdictLine} (${verdict.prefixDivergences.length}/${verdict.prefixExecuted} prefix divergences) \u2192 ${
|
|
14218
|
-
`
|
|
14079
|
+
progress(
|
|
14080
|
+
`replay-verify: ${verdictLine} (${verdict.prefixDivergences.length}/${verdict.prefixExecuted} prefix divergences) \u2192 ${out}`
|
|
14219
14081
|
);
|
|
14220
14082
|
}
|
|
14083
|
+
var REPLAY_BATCH_USAGE = `traces replay-verify-batch \u2014 measure replayability and fix-flip rates across gold-labeled corpora
|
|
14221
14084
|
|
|
14222
|
-
|
|
14223
|
-
|
|
14224
|
-
|
|
14225
|
-
|
|
14226
|
-
|
|
14227
|
-
|
|
14228
|
-
|
|
14229
|
-
|
|
14230
|
-
|
|
14231
|
-
|
|
14232
|
-
|
|
14233
|
-
|
|
14234
|
-
|
|
14235
|
-
|
|
14236
|
-
|
|
14237
|
-
|
|
14238
|
-
|
|
14239
|
-
|
|
14240
|
-
|
|
14241
|
-
|
|
14242
|
-
|
|
14243
|
-
|
|
14244
|
-
|
|
14245
|
-
|
|
14246
|
-
|
|
14247
|
-
|
|
14248
|
-
|
|
14249
|
-
|
|
14250
|
-
|
|
14251
|
-
const ids = /* @__PURE__ */ new Set();
|
|
14252
|
-
for (const stage of entry.incorrect_stages ?? []) {
|
|
14253
|
-
for (const id of stage.incorrect_step_ids ?? []) ids.add(id);
|
|
14254
|
-
}
|
|
14255
|
-
return [...ids].sort((a, b) => a - b);
|
|
14256
|
-
}
|
|
14257
|
-
function findRawTrajFiles(sweRawDir) {
|
|
14258
|
-
if (!existsSync2(sweRawDir)) return [];
|
|
14259
|
-
const entries = readdirSync2(sweRawDir, { recursive: true, encoding: "utf8" });
|
|
14260
|
-
return entries.filter((relative7) => relative7.endsWith(".traj.json")).map((relative7) => join25(sweRawDir, relative7)).sort();
|
|
14261
|
-
}
|
|
14262
|
-
function cwdFromPwdObservation(steps) {
|
|
14263
|
-
for (const step of steps) {
|
|
14264
|
-
if (step.action.trim() !== "pwd") continue;
|
|
14265
|
-
const line = parseObservationOutput(step.observation).split("\n").map((l) => l.trim()).find((l) => l.startsWith("/"));
|
|
14266
|
-
if (line) return line;
|
|
14085
|
+
Usage:
|
|
14086
|
+
traces replay-verify-batch \\
|
|
14087
|
+
--corpus NAME=<labels.json>::<prepared-dir> [--corpus ...] \\
|
|
14088
|
+
--out DIR [--fix none|generate|loop] [--fix-attempts 3] [--enumerate-only] \\
|
|
14089
|
+
[--fix-model glm-5.2] [--fix-base-url URL] [--fix-api-key-env ZAI_GLM_API_KEY] \\
|
|
14090
|
+
[--max-fix-cases 30] [--seed 17] [--step-timeout MS] [--prefix-limit N] \\
|
|
14091
|
+
[--case-filter SUBSTRING] [--case-limit N] \\
|
|
14092
|
+
[--base-url URL] [--api-key-env SANDBOX_API_KEY] [--max-lifetime SECONDS]
|
|
14093
|
+
|
|
14094
|
+
Replayable = the raw trajectory carries info.docker_config.base_image AND the
|
|
14095
|
+
labels mark at least one gold incorrect step that is a real mid-trajectory
|
|
14096
|
+
action. Excluded cases and image pull failures are report rows, never
|
|
14097
|
+
skipped silently. --enumerate-only prints the enumeration without touching a
|
|
14098
|
+
sandbox.
|
|
14099
|
+
|
|
14100
|
+
--fix generate runs ONE LLM call per arm-A-reproduced case (cap
|
|
14101
|
+
--max-fix-cases, seeded sample beyond it) and executes the corrected command
|
|
14102
|
+
as arm B in a fresh sandbox. --fix loop feeds each failed attempt's REAL
|
|
14103
|
+
executed output into the next prompt, up to --fix-attempts attempts.
|
|
14104
|
+
|
|
14105
|
+
Outputs batch-report.json, batch-report.md, cases.jsonl (incremental), and one
|
|
14106
|
+
directory per case with the full replay-verify artifacts.
|
|
14107
|
+
See docs/replay-verify.md for orchestrator setup and honest limits.
|
|
14108
|
+
`;
|
|
14109
|
+
async function cmdReplayVerifyBatch(argv) {
|
|
14110
|
+
const parsed = parseFlags2("replay-verify-batch", argv, ["--enumerate-only"]);
|
|
14111
|
+
if (parsed === "help") {
|
|
14112
|
+
process.stdout.write(REPLAY_BATCH_USAGE);
|
|
14113
|
+
return;
|
|
14267
14114
|
}
|
|
14268
|
-
|
|
14269
|
-
|
|
14270
|
-
|
|
14271
|
-
|
|
14272
|
-
if (existsSync2(taskPath)) {
|
|
14273
|
-
const text2 = readFileSync4(taskPath, "utf8").trim();
|
|
14274
|
-
if (text2.length > 0) return text2;
|
|
14115
|
+
const { values, corpora } = parsed;
|
|
14116
|
+
const flags = flagReader("replay-verify-batch", values);
|
|
14117
|
+
if (corpora.length === 0) {
|
|
14118
|
+
throw new Error("replay-verify-batch: at least one --corpus name=<labels>::<prepared> is required");
|
|
14275
14119
|
}
|
|
14276
|
-
|
|
14277
|
-
|
|
14278
|
-
|
|
14120
|
+
if (parsed.switches.has("--enumerate-only")) {
|
|
14121
|
+
const enumeration = enumerateReplayableCases(corpora);
|
|
14122
|
+
const replayable = enumeration.replayable.map((c) => ({
|
|
14123
|
+
corpus: c.corpus,
|
|
14124
|
+
trajId: c.trajId,
|
|
14125
|
+
image: c.image,
|
|
14126
|
+
cwd: c.cwd,
|
|
14127
|
+
cwdSource: c.cwdSource,
|
|
14128
|
+
k: c.k,
|
|
14129
|
+
submitGoldsSkipped: c.submitGoldsSkipped,
|
|
14130
|
+
recordedReturncodeAtK: c.recordedReturncodeAtK,
|
|
14131
|
+
goldIncorrectSteps: c.goldIncorrectSteps
|
|
14132
|
+
}));
|
|
14133
|
+
const report2 = { labelEntries: enumeration.labelEntryCount, replayable, excluded: enumeration.excluded };
|
|
14134
|
+
process.stdout.write(`${JSON.stringify(report2, null, 2)}
|
|
14135
|
+
`);
|
|
14136
|
+
return;
|
|
14279
14137
|
}
|
|
14280
|
-
|
|
14281
|
-
|
|
14282
|
-
|
|
14283
|
-
|
|
14284
|
-
const rawFiles = findRawTrajFiles(sweRawDir);
|
|
14285
|
-
if (rawFiles.length === 0) {
|
|
14286
|
-
return { resolved: false, reason: "no-swe-raw-trajectory", detail: sweRawDir };
|
|
14138
|
+
const out = flags.required("--out");
|
|
14139
|
+
const fix = values.get("--fix") ?? "none";
|
|
14140
|
+
if (fix !== "none" && fix !== "generate" && fix !== "loop") {
|
|
14141
|
+
throw new Error(`replay-verify-batch: --fix must be none, generate, or loop, got ${fix}`);
|
|
14287
14142
|
}
|
|
14288
|
-
|
|
14289
|
-
|
|
14143
|
+
const fixAttempts = flags.number("--fix-attempts") ?? 3;
|
|
14144
|
+
if (!Number.isInteger(fixAttempts) || fixAttempts < 1) {
|
|
14145
|
+
throw new Error(`replay-verify-batch: --fix-attempts must be a positive integer, got ${fixAttempts}`);
|
|
14290
14146
|
}
|
|
14291
|
-
|
|
14292
|
-
|
|
14293
|
-
|
|
14294
|
-
|
|
14295
|
-
|
|
14296
|
-
|
|
14297
|
-
|
|
14298
|
-
|
|
14299
|
-
|
|
14147
|
+
const fixModel = values.get("--fix-model") ?? "glm-5.2";
|
|
14148
|
+
let fixCaller;
|
|
14149
|
+
if (fix !== "none") {
|
|
14150
|
+
const fixApiKeyEnv = values.get("--fix-api-key-env") ?? "ZAI_GLM_API_KEY";
|
|
14151
|
+
const fixApiKey = process.env[fixApiKeyEnv];
|
|
14152
|
+
if (!fixApiKey) {
|
|
14153
|
+
throw new Error(`replay-verify-batch: env var ${fixApiKeyEnv} is empty \u2014 pass --fix-api-key-env or export it`);
|
|
14154
|
+
}
|
|
14155
|
+
fixCaller = zaiChatCaller({
|
|
14156
|
+
baseUrl: values.get("--fix-base-url") ?? "https://api.z.ai/api/coding/paas/v4",
|
|
14157
|
+
apiKey: fixApiKey,
|
|
14158
|
+
model: fixModel
|
|
14159
|
+
});
|
|
14300
14160
|
}
|
|
14301
|
-
const
|
|
14302
|
-
const image = dockerConfig?.base_image;
|
|
14303
|
-
if (typeof image !== "string" || image.length === 0) {
|
|
14304
|
-
return { resolved: false, reason: "no-docker-image", detail: rawFiles[0] };
|
|
14305
|
-
}
|
|
14306
|
-
const stepsPath = join25(corpus.preparedDir, "normalized", trajId, "steps.json");
|
|
14307
|
-
if (!existsSync2(stepsPath)) {
|
|
14308
|
-
return { resolved: false, reason: "missing-steps-json", detail: stepsPath };
|
|
14309
|
-
}
|
|
14310
|
-
const steps = JSON.parse(readFileSync4(stepsPath, "utf8"));
|
|
14311
|
-
const runConfigCwd = raw.info?.config?.environment?.cwd;
|
|
14312
|
-
const dockerCwd = dockerConfig?.cwd;
|
|
14313
|
-
let cwd;
|
|
14314
|
-
let cwdSource;
|
|
14315
|
-
if (typeof runConfigCwd === "string" && runConfigCwd.length > 0) {
|
|
14316
|
-
cwd = runConfigCwd;
|
|
14317
|
-
cwdSource = "run-config";
|
|
14318
|
-
} else if (typeof dockerCwd === "string" && dockerCwd.length > 0) {
|
|
14319
|
-
cwd = dockerCwd;
|
|
14320
|
-
cwdSource = "docker-config";
|
|
14321
|
-
} else {
|
|
14322
|
-
const observed = cwdFromPwdObservation(steps);
|
|
14323
|
-
if (!observed) return { resolved: false, reason: "cwd-underivable" };
|
|
14324
|
-
cwd = observed;
|
|
14325
|
-
cwdSource = "pwd-observation";
|
|
14326
|
-
}
|
|
14327
|
-
const recordedTimeout = raw.info?.config?.environment?.timeout;
|
|
14328
|
-
return {
|
|
14329
|
-
resolved: true,
|
|
14330
|
-
resources: {
|
|
14331
|
-
corpus: corpus.name,
|
|
14332
|
-
trajId,
|
|
14333
|
-
stepsPath,
|
|
14334
|
-
steps,
|
|
14335
|
-
taskStatement: taskStatementOf(corpus.preparedDir, trajId, raw),
|
|
14336
|
-
image,
|
|
14337
|
-
cwd,
|
|
14338
|
-
cwdSource,
|
|
14339
|
-
recordedStepTimeoutMs: typeof recordedTimeout === "number" && recordedTimeout > 0 ? recordedTimeout * 1e3 : null
|
|
14340
|
-
}
|
|
14341
|
-
};
|
|
14342
|
-
}
|
|
14343
|
-
function enumerateReplayableCases(corpora) {
|
|
14344
|
-
const replayable = [];
|
|
14345
|
-
const excluded = [];
|
|
14346
|
-
let labelEntryCount = 0;
|
|
14347
|
-
for (const corpus of corpora) {
|
|
14348
|
-
for (const entry of readLabelEntries(corpus.labelsPath)) {
|
|
14349
|
-
labelEntryCount += 1;
|
|
14350
|
-
const trajId = entry.traj_id;
|
|
14351
|
-
const resolution = resolveCaseResources(corpus, trajId);
|
|
14352
|
-
if (!resolution.resolved) {
|
|
14353
|
-
excluded.push({ corpus: corpus.name, trajId, reason: resolution.reason, detail: resolution.detail });
|
|
14354
|
-
continue;
|
|
14355
|
-
}
|
|
14356
|
-
const gold = goldIncorrectSteps(entry);
|
|
14357
|
-
if (gold.length === 0) {
|
|
14358
|
-
excluded.push({ corpus: corpus.name, trajId, reason: "no-gold-incorrect-step" });
|
|
14359
|
-
continue;
|
|
14360
|
-
}
|
|
14361
|
-
const resources = resolution.resources;
|
|
14362
|
-
let submitGoldsSkipped = 0;
|
|
14363
|
-
let target = null;
|
|
14364
|
-
let missingGoldId = null;
|
|
14365
|
-
for (const goldId of gold) {
|
|
14366
|
-
const step = resources.steps.find((s) => s.step_id === goldId);
|
|
14367
|
-
if (!step) {
|
|
14368
|
-
missingGoldId = goldId;
|
|
14369
|
-
break;
|
|
14370
|
-
}
|
|
14371
|
-
if (isSubmitAction(step.action)) {
|
|
14372
|
-
submitGoldsSkipped += 1;
|
|
14373
|
-
continue;
|
|
14374
|
-
}
|
|
14375
|
-
target = step;
|
|
14376
|
-
break;
|
|
14377
|
-
}
|
|
14378
|
-
if (missingGoldId !== null) {
|
|
14379
|
-
excluded.push({
|
|
14380
|
-
corpus: corpus.name,
|
|
14381
|
-
trajId,
|
|
14382
|
-
reason: "gold-step-outside-steps",
|
|
14383
|
-
detail: `k=${missingGoldId}, steps=${resources.steps.length}`
|
|
14384
|
-
});
|
|
14385
|
-
continue;
|
|
14386
|
-
}
|
|
14387
|
-
if (!target) {
|
|
14388
|
-
excluded.push({
|
|
14389
|
-
corpus: corpus.name,
|
|
14390
|
-
trajId,
|
|
14391
|
-
reason: "gold-only-submit-step",
|
|
14392
|
-
detail: `${submitGoldsSkipped} gold step(s), all submit commands`
|
|
14393
|
-
});
|
|
14394
|
-
continue;
|
|
14395
|
-
}
|
|
14396
|
-
replayable.push({
|
|
14397
|
-
...resources,
|
|
14398
|
-
goldIncorrectSteps: gold,
|
|
14399
|
-
k: target.step_id,
|
|
14400
|
-
submitGoldsSkipped,
|
|
14401
|
-
recordedReturncodeAtK: parseRecordedReturncode(target.observation)
|
|
14402
|
-
});
|
|
14403
|
-
}
|
|
14404
|
-
}
|
|
14405
|
-
return { replayable, excluded, labelEntryCount };
|
|
14406
|
-
}
|
|
14407
|
-
|
|
14408
|
-
// src/replay-fix.ts
|
|
14409
|
-
function zaiChatCaller(options) {
|
|
14410
|
-
const fetchImpl = options.fetchImpl ?? fetch;
|
|
14411
|
-
return {
|
|
14412
|
-
async complete(system, user) {
|
|
14413
|
-
const baseCap = options.maxTokens ?? 16384;
|
|
14414
|
-
let lastError = "empty completion content";
|
|
14415
|
-
for (const cap of [baseCap, baseCap * 2]) {
|
|
14416
|
-
const outcome = await callOnce(cap);
|
|
14417
|
-
if (outcome.succeeded) return outcome;
|
|
14418
|
-
lastError = outcome.error;
|
|
14419
|
-
if (!/finish_reason":"length/.test(outcome.error)) return outcome;
|
|
14420
|
-
}
|
|
14421
|
-
return { succeeded: false, error: lastError };
|
|
14422
|
-
async function callOnce(maxTokens) {
|
|
14423
|
-
const controller = new AbortController();
|
|
14424
|
-
const timer = setTimeout(() => controller.abort(), options.timeoutMs ?? 18e4);
|
|
14425
|
-
try {
|
|
14426
|
-
const response = await fetchImpl(`${options.baseUrl.replace(/\/$/, "")}/chat/completions`, {
|
|
14427
|
-
method: "POST",
|
|
14428
|
-
headers: {
|
|
14429
|
-
authorization: `Bearer ${options.apiKey}`,
|
|
14430
|
-
"content-type": "application/json"
|
|
14431
|
-
},
|
|
14432
|
-
body: JSON.stringify({
|
|
14433
|
-
model: options.model,
|
|
14434
|
-
messages: [
|
|
14435
|
-
{ role: "system", content: system },
|
|
14436
|
-
{ role: "user", content: user }
|
|
14437
|
-
],
|
|
14438
|
-
temperature: 0,
|
|
14439
|
-
max_tokens: maxTokens
|
|
14440
|
-
}),
|
|
14441
|
-
signal: controller.signal
|
|
14442
|
-
});
|
|
14443
|
-
const bodyText = await response.text();
|
|
14444
|
-
if (!response.ok) {
|
|
14445
|
-
return { succeeded: false, error: `HTTP ${response.status}: ${bodyText.slice(0, 400)}` };
|
|
14446
|
-
}
|
|
14447
|
-
const body = JSON.parse(bodyText);
|
|
14448
|
-
if (body.error) {
|
|
14449
|
-
return { succeeded: false, error: String(body.error.message ?? "provider error") };
|
|
14450
|
-
}
|
|
14451
|
-
const content = body.choices?.[0]?.message?.content;
|
|
14452
|
-
if (typeof content !== "string" || content.length === 0) {
|
|
14453
|
-
return { succeeded: false, error: `empty completion content: ${bodyText.slice(0, 400)}` };
|
|
14454
|
-
}
|
|
14455
|
-
const promptTokens = body.usage?.prompt_tokens;
|
|
14456
|
-
const completionTokens = body.usage?.completion_tokens;
|
|
14457
|
-
const usage = typeof promptTokens === "number" && typeof completionTokens === "number" ? { promptTokens, completionTokens } : null;
|
|
14458
|
-
return { succeeded: true, value: { content, usage } };
|
|
14459
|
-
} catch (err) {
|
|
14460
|
-
return { succeeded: false, error: err instanceof Error ? err.message : String(err) };
|
|
14461
|
-
} finally {
|
|
14462
|
-
clearTimeout(timer);
|
|
14463
|
-
}
|
|
14464
|
-
}
|
|
14465
|
-
}
|
|
14466
|
-
};
|
|
14467
|
-
}
|
|
14468
|
-
function clipText(text2, limit) {
|
|
14469
|
-
if (text2.length <= limit) return text2;
|
|
14470
|
-
const half = Math.floor(limit / 2);
|
|
14471
|
-
return `${text2.slice(0, half)}
|
|
14472
|
-
\u2026 [${text2.length - limit} chars elided] \u2026
|
|
14473
|
-
${text2.slice(-half)}`;
|
|
14474
|
-
}
|
|
14475
|
-
function renderStep(step, marker) {
|
|
14476
|
-
const rc = parseRecordedReturncode(step.observation);
|
|
14477
|
-
const output = clipText(parseObservationOutput(step.observation).trim(), 1600);
|
|
14478
|
-
return [
|
|
14479
|
-
`### step ${step.step_id}${marker}`,
|
|
14480
|
-
"```sh",
|
|
14481
|
-
clipText(step.action, 2e3),
|
|
14482
|
-
"```",
|
|
14483
|
-
`returncode: ${rc ?? "none recorded"}`,
|
|
14484
|
-
output.length > 0 ? `output:
|
|
14485
|
-
\`\`\`
|
|
14486
|
-
${output}
|
|
14487
|
-
\`\`\`` : "output: (empty)"
|
|
14488
|
-
].join("\n");
|
|
14489
|
-
}
|
|
14490
|
-
function promptBody(input) {
|
|
14491
|
-
const radius = input.contextRadius ?? 3;
|
|
14492
|
-
const target = input.steps.find((s) => s.step_id === input.k);
|
|
14493
|
-
if (!target) throw new Error(`replay-fix: no step with step_id ${input.k}`);
|
|
14494
|
-
const context = input.steps.filter(
|
|
14495
|
-
(s) => s.step_id !== input.k && Math.abs(s.step_id - input.k) <= radius
|
|
14496
|
-
);
|
|
14497
|
-
return [
|
|
14498
|
-
"## Task the agent was solving",
|
|
14499
|
-
input.taskStatement ? clipText(input.taskStatement, 4e3) : "(no task statement recorded)",
|
|
14500
|
-
"",
|
|
14501
|
-
"## Surrounding steps",
|
|
14502
|
-
...context.map((s) => renderStep(s, "")),
|
|
14503
|
-
"",
|
|
14504
|
-
"## Failing step to correct",
|
|
14505
|
-
renderStep(target, " (INCORRECT \u2014 correct this one)")
|
|
14506
|
-
];
|
|
14507
|
-
}
|
|
14508
|
-
function buildFixPrompt(input) {
|
|
14509
|
-
const system = [
|
|
14510
|
-
"You repair one failed shell command from a recorded coding-agent trajectory.",
|
|
14511
|
-
"The trajectory replays inside the original docker image; every command runs as a fresh /bin/sh subshell from a fixed working directory.",
|
|
14512
|
-
"You are given the failing step, its recorded output, surrounding steps, and the task statement.",
|
|
14513
|
-
"Reply with exactly ONE corrected shell command (compound commands with && or pipes are fine) inside a single ```sh fenced block.",
|
|
14514
|
-
"The corrected command must accomplish the failing step's intent and exit 0. No prose outside the fenced block."
|
|
14515
|
-
].join("\n");
|
|
14516
|
-
const user = [
|
|
14517
|
-
...promptBody(input),
|
|
14518
|
-
"",
|
|
14519
|
-
"Output the single corrected replacement for the failing step now."
|
|
14520
|
-
].join("\n");
|
|
14521
|
-
return { system, user };
|
|
14522
|
-
}
|
|
14523
|
-
function renderFailedAttempt(prior) {
|
|
14524
|
-
if (prior.command === null) {
|
|
14525
|
-
return [
|
|
14526
|
-
`### attempt ${prior.attempt}`,
|
|
14527
|
-
`model call failed before producing a command: ${prior.llmError ?? "unknown error"}`
|
|
14528
|
-
].join("\n");
|
|
14529
|
-
}
|
|
14530
|
-
const stdout = (prior.stdoutTail ?? "").trim();
|
|
14531
|
-
const stderr = (prior.stderrTail ?? "").trim();
|
|
14532
|
-
return [
|
|
14533
|
-
`### attempt ${prior.attempt}`,
|
|
14534
|
-
"```sh",
|
|
14535
|
-
clipText(prior.command, 2e3),
|
|
14536
|
-
"```",
|
|
14537
|
-
`exit code: ${prior.exitCode ?? "not executed"}`,
|
|
14538
|
-
stdout.length > 0 ? `stdout:
|
|
14539
|
-
\`\`\`
|
|
14540
|
-
${stdout}
|
|
14541
|
-
\`\`\`` : "stdout: (empty)",
|
|
14542
|
-
stderr.length > 0 ? `stderr:
|
|
14543
|
-
\`\`\`
|
|
14544
|
-
${stderr}
|
|
14545
|
-
\`\`\`` : "stderr: (empty)"
|
|
14546
|
-
].join("\n");
|
|
14547
|
-
}
|
|
14548
|
-
function buildRetryFixPrompt(input, priorAttempts, maxScriptCommands = 5) {
|
|
14549
|
-
if (priorAttempts.length === 0) {
|
|
14550
|
-
throw new Error("replay-fix: buildRetryFixPrompt requires at least one prior attempt");
|
|
14551
|
-
}
|
|
14552
|
-
const system = [
|
|
14553
|
-
"You repair one failed shell command from a recorded coding-agent trajectory.",
|
|
14554
|
-
"The trajectory replays inside the original docker image; every command runs as a fresh /bin/sh subshell from a fixed working directory.",
|
|
14555
|
-
"Earlier corrected commands were executed for real and failed; their actual output is included below.",
|
|
14556
|
-
`Reply with a corrected fix inside a single \`\`\`sh fenced block: either one command, or a short script of at most ${maxScriptCommands} commands (one per line).`,
|
|
14557
|
-
"The whole block executes as ONE /bin/sh unit from the fixed working directory and must exit 0.",
|
|
14558
|
-
"Do not repeat a command that already failed. Keep reasoning brief. No prose outside the fenced block."
|
|
14559
|
-
].join("\n");
|
|
14560
|
-
const user = [
|
|
14561
|
-
...promptBody(input),
|
|
14562
|
-
"",
|
|
14563
|
-
"## Previous fix attempts (executed for real \u2014 all failed)",
|
|
14564
|
-
...priorAttempts.map((prior) => renderFailedAttempt(prior)),
|
|
14565
|
-
"",
|
|
14566
|
-
`Output a corrected fix now \u2014 one \`\`\`sh block, at most ${maxScriptCommands} commands.`
|
|
14567
|
-
].join("\n");
|
|
14568
|
-
return { system, user };
|
|
14569
|
-
}
|
|
14570
|
-
function countScriptCommands(script) {
|
|
14571
|
-
return script.split("\n").filter((line) => line.trim().length > 0 && !line.trim().startsWith("#")).length;
|
|
14572
|
-
}
|
|
14573
|
-
function extractFixCommand(content) {
|
|
14574
|
-
const blocks = [...content.matchAll(/```(?:sh|bash|shell)?\n([\s\S]*?)```/g)];
|
|
14575
|
-
const candidate = blocks.length > 0 ? blocks[blocks.length - 1][1] : content;
|
|
14576
|
-
const command = candidate.trim();
|
|
14577
|
-
if (command.length === 0 || command.includes("```")) return null;
|
|
14578
|
-
return command;
|
|
14579
|
-
}
|
|
14580
|
-
async function generateFixCommand(caller, input) {
|
|
14581
|
-
const { system, user } = buildFixPrompt(input);
|
|
14582
|
-
const outcome = await caller.complete(system, user);
|
|
14583
|
-
if (!outcome.succeeded) return outcome;
|
|
14584
|
-
const command = extractFixCommand(outcome.value.content);
|
|
14585
|
-
if (command === null) {
|
|
14586
|
-
return {
|
|
14587
|
-
succeeded: false,
|
|
14588
|
-
error: `completion carried no usable command: ${clipText(outcome.value.content, 300)}`
|
|
14589
|
-
};
|
|
14590
|
-
}
|
|
14591
|
-
return { succeeded: true, value: { command, usage: outcome.value.usage } };
|
|
14592
|
-
}
|
|
14593
|
-
|
|
14594
|
-
// src/replay-fix-loop.ts
|
|
14595
|
-
function toFailedAttempt(record2) {
|
|
14596
|
-
return {
|
|
14597
|
-
attempt: record2.attempt,
|
|
14598
|
-
command: record2.command,
|
|
14599
|
-
exitCode: record2.exitCode,
|
|
14600
|
-
stdoutTail: record2.stdoutTail,
|
|
14601
|
-
stderrTail: record2.stderrTail,
|
|
14602
|
-
llmError: record2.llmError ?? record2.armBError
|
|
14603
|
-
};
|
|
14604
|
-
}
|
|
14605
|
-
async function runFixLoop(caller, input, executor, options) {
|
|
14606
|
-
if (!Number.isInteger(options.maxAttempts) || options.maxAttempts < 1) {
|
|
14607
|
-
throw new Error(
|
|
14608
|
-
`replay-fix-loop: maxAttempts must be a positive integer, got ${options.maxAttempts}`
|
|
14609
|
-
);
|
|
14610
|
-
}
|
|
14611
|
-
const scriptCap = options.maxScriptCommands ?? 5;
|
|
14612
|
-
const tailChars = options.outputTailChars ?? 1600;
|
|
14613
|
-
const attempts = [];
|
|
14614
|
-
let llmCalls = 0;
|
|
14615
|
-
let llmFailures = 0;
|
|
14616
|
-
let promptTokens = 0;
|
|
14617
|
-
let completionTokens = 0;
|
|
14618
|
-
const unexecuted = (attempt, llmError, usage, command, started) => ({
|
|
14619
|
-
attempt,
|
|
14620
|
-
command,
|
|
14621
|
-
llmError,
|
|
14622
|
-
usage,
|
|
14623
|
-
executed: false,
|
|
14624
|
-
exitCode: null,
|
|
14625
|
-
prefixExecuted: null,
|
|
14626
|
-
prefixDivergences: null,
|
|
14627
|
-
failureVanished: null,
|
|
14628
|
-
stdoutTail: null,
|
|
14629
|
-
stderrTail: null,
|
|
14630
|
-
armBError: null,
|
|
14631
|
-
wallMs: Date.now() - started
|
|
14632
|
-
});
|
|
14633
|
-
for (let attempt = 1; attempt <= options.maxAttempts; attempt++) {
|
|
14634
|
-
const started = Date.now();
|
|
14635
|
-
const prompt = attempt === 1 ? buildFixPrompt(input) : buildRetryFixPrompt(input, attempts.map(toFailedAttempt), scriptCap);
|
|
14636
|
-
llmCalls += 1;
|
|
14637
|
-
const outcome = await caller.complete(prompt.system, prompt.user);
|
|
14638
|
-
if (!outcome.succeeded) {
|
|
14639
|
-
llmFailures += 1;
|
|
14640
|
-
attempts.push(unexecuted(attempt, outcome.error, null, null, started));
|
|
14641
|
-
options.onProgress?.(
|
|
14642
|
-
`fix-loop attempt ${attempt}: LLM failed \u2014 ${outcome.error.slice(0, 160)}`
|
|
14643
|
-
);
|
|
14644
|
-
continue;
|
|
14645
|
-
}
|
|
14646
|
-
const usage = outcome.value.usage;
|
|
14647
|
-
promptTokens += usage?.promptTokens ?? 0;
|
|
14648
|
-
completionTokens += usage?.completionTokens ?? 0;
|
|
14649
|
-
const command = extractFixCommand(outcome.value.content);
|
|
14650
|
-
if (command === null) {
|
|
14651
|
-
llmFailures += 1;
|
|
14652
|
-
attempts.push(
|
|
14653
|
-
unexecuted(
|
|
14654
|
-
attempt,
|
|
14655
|
-
`completion carried no usable command: ${clipText(outcome.value.content, 300)}`,
|
|
14656
|
-
usage,
|
|
14657
|
-
null,
|
|
14658
|
-
started
|
|
14659
|
-
)
|
|
14660
|
-
);
|
|
14661
|
-
continue;
|
|
14662
|
-
}
|
|
14663
|
-
if (attempt > 1) {
|
|
14664
|
-
const commandLines = countScriptCommands(command);
|
|
14665
|
-
if (commandLines > scriptCap) {
|
|
14666
|
-
llmFailures += 1;
|
|
14667
|
-
attempts.push(
|
|
14668
|
-
unexecuted(
|
|
14669
|
-
attempt,
|
|
14670
|
-
`script exceeds ${scriptCap} command lines (${commandLines})`,
|
|
14671
|
-
usage,
|
|
14672
|
-
null,
|
|
14673
|
-
started
|
|
14674
|
-
)
|
|
14675
|
-
);
|
|
14676
|
-
options.onProgress?.(
|
|
14677
|
-
`fix-loop attempt ${attempt}: rejected script with ${commandLines} command lines`
|
|
14678
|
-
);
|
|
14679
|
-
continue;
|
|
14680
|
-
}
|
|
14681
|
-
}
|
|
14682
|
-
let execution;
|
|
14683
|
-
try {
|
|
14684
|
-
execution = await executor(command, attempt);
|
|
14685
|
-
} catch (err) {
|
|
14686
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
14687
|
-
attempts.push({
|
|
14688
|
-
...unexecuted(attempt, "", usage, command, started),
|
|
14689
|
-
llmError: null,
|
|
14690
|
-
armBError: message.slice(0, 500)
|
|
14691
|
-
});
|
|
14692
|
-
options.onProgress?.(
|
|
14693
|
-
`fix-loop attempt ${attempt}: sandbox error \u2014 ${message.slice(0, 160)}`
|
|
14694
|
-
);
|
|
14695
|
-
return {
|
|
14696
|
-
flipped: false,
|
|
14697
|
-
flippedAtAttempt: null,
|
|
14698
|
-
attempts,
|
|
14699
|
-
aborted: true,
|
|
14700
|
-
llmCalls,
|
|
14701
|
-
llmFailures,
|
|
14702
|
-
promptTokens,
|
|
14703
|
-
completionTokens
|
|
14704
|
-
};
|
|
14705
|
-
}
|
|
14706
|
-
attempts.push({
|
|
14707
|
-
attempt,
|
|
14708
|
-
command,
|
|
14709
|
-
llmError: null,
|
|
14710
|
-
usage,
|
|
14711
|
-
executed: true,
|
|
14712
|
-
exitCode: execution.exitCode,
|
|
14713
|
-
prefixExecuted: execution.prefixExecuted,
|
|
14714
|
-
prefixDivergences: execution.prefixDivergences,
|
|
14715
|
-
failureVanished: execution.failureVanished,
|
|
14716
|
-
stdoutTail: clipText(execution.stdout, tailChars),
|
|
14717
|
-
stderrTail: clipText(execution.stderr, tailChars),
|
|
14718
|
-
armBError: null,
|
|
14719
|
-
wallMs: Date.now() - started
|
|
14720
|
-
});
|
|
14721
|
-
options.onProgress?.(
|
|
14722
|
-
`fix-loop attempt ${attempt}: exit=${execution.exitCode} failureVanished=${execution.failureVanished}`
|
|
14723
|
-
);
|
|
14724
|
-
if (execution.failureVanished) {
|
|
14725
|
-
return {
|
|
14726
|
-
flipped: true,
|
|
14727
|
-
flippedAtAttempt: attempt,
|
|
14728
|
-
attempts,
|
|
14729
|
-
aborted: false,
|
|
14730
|
-
llmCalls,
|
|
14731
|
-
llmFailures,
|
|
14732
|
-
promptTokens,
|
|
14733
|
-
completionTokens
|
|
14734
|
-
};
|
|
14735
|
-
}
|
|
14736
|
-
}
|
|
14737
|
-
return {
|
|
14738
|
-
flipped: false,
|
|
14739
|
-
flippedAtAttempt: null,
|
|
14740
|
-
attempts,
|
|
14741
|
-
aborted: false,
|
|
14742
|
-
llmCalls,
|
|
14743
|
-
llmFailures,
|
|
14744
|
-
promptTokens,
|
|
14745
|
-
completionTokens
|
|
14746
|
-
};
|
|
14747
|
-
}
|
|
14748
|
-
|
|
14749
|
-
// src/replay-batch.ts
|
|
14750
|
-
import { execFile } from "child_process";
|
|
14751
|
-
import { createHash as createHash12 } from "crypto";
|
|
14752
|
-
import { appendFileSync, mkdirSync as mkdirSync2, mkdtempSync, writeFileSync as writeFileSync2 } from "fs";
|
|
14753
|
-
import { tmpdir as tmpdir5 } from "os";
|
|
14754
|
-
import { join as join26 } from "path";
|
|
14755
|
-
import { promisify } from "util";
|
|
14756
|
-
import {
|
|
14757
|
-
InMemoryTraceStore as InMemoryTraceStore3,
|
|
14758
|
-
runCounterfactual as runCounterfactual2
|
|
14759
|
-
} from "@tangle-network/agent-eval";
|
|
14760
|
-
var execFileAsync = promisify(execFile);
|
|
14761
|
-
function derivedImageTag(image, cwd) {
|
|
14762
|
-
const digest = createHash12("sha256").update(`${image}
|
|
14763
|
-
${cwd}`).digest("hex").slice(0, 12);
|
|
14764
|
-
return `ctb-replay:${digest}-uid1000`;
|
|
14765
|
-
}
|
|
14766
|
-
function dockerImagePreparer(options = {}) {
|
|
14767
|
-
const pullTimeoutMs = options.pullTimeoutMs ?? 12e5;
|
|
14768
|
-
const buildTimeoutMs = options.buildTimeoutMs ?? 9e5;
|
|
14769
|
-
const imageExists = async (tag) => {
|
|
14770
|
-
try {
|
|
14771
|
-
await execFileAsync("docker", ["image", "inspect", tag], { maxBuffer: 8 * 1024 * 1024 });
|
|
14772
|
-
return true;
|
|
14773
|
-
} catch {
|
|
14774
|
-
return false;
|
|
14775
|
-
}
|
|
14776
|
-
};
|
|
14777
|
-
const errorTail = (err) => {
|
|
14778
|
-
const raw = err && typeof err === "object" && "stderr" in err && typeof err.stderr === "string" && err.stderr.length > 0 ? err.stderr : err instanceof Error ? err.message : String(err);
|
|
14779
|
-
return raw.trim().split("\n").slice(-3).join(" | ").slice(0, 400);
|
|
14780
|
-
};
|
|
14781
|
-
return {
|
|
14782
|
-
async ensure(image, cwd) {
|
|
14783
|
-
if (cwd === "/") {
|
|
14784
|
-
if (await imageExists(image)) {
|
|
14785
|
-
return { succeeded: true, value: { derivedImage: image, pulled: false, built: false } };
|
|
14786
|
-
}
|
|
14787
|
-
try {
|
|
14788
|
-
await execFileAsync("docker", ["pull", image], {
|
|
14789
|
-
timeout: pullTimeoutMs,
|
|
14790
|
-
maxBuffer: 32 * 1024 * 1024
|
|
14791
|
-
});
|
|
14792
|
-
} catch (err) {
|
|
14793
|
-
return { succeeded: false, error: `pull ${image}: ${errorTail(err)}` };
|
|
14794
|
-
}
|
|
14795
|
-
return { succeeded: true, value: { derivedImage: image, pulled: true, built: false } };
|
|
14796
|
-
}
|
|
14797
|
-
const derived = derivedImageTag(image, cwd);
|
|
14798
|
-
if (await imageExists(derived)) {
|
|
14799
|
-
return { succeeded: true, value: { derivedImage: derived, pulled: false, built: false } };
|
|
14800
|
-
}
|
|
14801
|
-
let pulled = false;
|
|
14802
|
-
if (!await imageExists(image)) {
|
|
14803
|
-
try {
|
|
14804
|
-
await execFileAsync("docker", ["pull", image], {
|
|
14805
|
-
timeout: pullTimeoutMs,
|
|
14806
|
-
maxBuffer: 32 * 1024 * 1024
|
|
14807
|
-
});
|
|
14808
|
-
pulled = true;
|
|
14809
|
-
} catch (err) {
|
|
14810
|
-
return { succeeded: false, error: `pull ${image}: ${errorTail(err)}` };
|
|
14811
|
-
}
|
|
14812
|
-
}
|
|
14813
|
-
const contextDir = mkdtempSync(join26(tmpdir5(), "ctb-replay-image-"));
|
|
14814
|
-
const quotedCwd = `'${cwd.replaceAll("'", `'\\''`)}'`;
|
|
14815
|
-
writeFileSync2(
|
|
14816
|
-
join26(contextDir, "Dockerfile"),
|
|
14817
|
-
`FROM ${image}
|
|
14818
|
-
RUN chown -R 1000:1000 ${quotedCwd}
|
|
14819
|
-
`
|
|
14820
|
-
);
|
|
14821
|
-
try {
|
|
14822
|
-
await execFileAsync("docker", ["build", "-t", derived, contextDir], {
|
|
14823
|
-
timeout: buildTimeoutMs,
|
|
14824
|
-
maxBuffer: 32 * 1024 * 1024
|
|
14825
|
-
});
|
|
14826
|
-
} catch (err) {
|
|
14827
|
-
return { succeeded: false, error: `build ${derived} from ${image}: ${errorTail(err)}` };
|
|
14828
|
-
}
|
|
14829
|
-
return { succeeded: true, value: { derivedImage: derived, pulled, built: true } };
|
|
14830
|
-
}
|
|
14831
|
-
};
|
|
14832
|
-
}
|
|
14833
|
-
function mulberry32(seed) {
|
|
14834
|
-
let state = seed >>> 0;
|
|
14835
|
-
return () => {
|
|
14836
|
-
state = state + 1831565813 >>> 0;
|
|
14837
|
-
let t = state;
|
|
14838
|
-
t = Math.imul(t ^ t >>> 15, t | 1);
|
|
14839
|
-
t ^= t + Math.imul(t ^ t >>> 7, t | 61);
|
|
14840
|
-
return ((t ^ t >>> 14) >>> 0) / 4294967296;
|
|
14841
|
-
};
|
|
14842
|
-
}
|
|
14843
|
-
function seededSample(items, size, seed) {
|
|
14844
|
-
const pool = [...items];
|
|
14845
|
-
const random = mulberry32(seed);
|
|
14846
|
-
for (let i = pool.length - 1; i > 0; i--) {
|
|
14847
|
-
const j = Math.floor(random() * (i + 1));
|
|
14848
|
-
[pool[i], pool[j]] = [pool[j], pool[i]];
|
|
14849
|
-
}
|
|
14850
|
-
return new Set(pool.slice(0, size));
|
|
14851
|
-
}
|
|
14852
|
-
function caseOutDirName(row) {
|
|
14853
|
-
return `${row.corpus}--${row.trajId}`.replaceAll(/[^A-Za-z0-9._-]/g, "_").slice(0, 180);
|
|
14854
|
-
}
|
|
14855
|
-
async function executeArmB(replayCase, fixCommand, backend, signature, stepTimeoutMs, prefixLimit, onProgress) {
|
|
14856
|
-
const store = new InMemoryTraceStore3();
|
|
14857
|
-
const { runId } = await ingestCodeTraceBenchSteps(store, replayCase.steps, replayCase.trajId);
|
|
14858
|
-
const runner = new SandboxCounterfactualRunner(backend, {
|
|
14859
|
-
cwd: replayCase.cwd,
|
|
14860
|
-
stepTimeoutMs,
|
|
14861
|
-
prefixLimit,
|
|
14862
|
-
onProgress
|
|
14863
|
-
});
|
|
14864
|
-
await runCounterfactual2(
|
|
14865
|
-
store,
|
|
14866
|
-
runId,
|
|
14867
|
-
{
|
|
14868
|
-
kind: "custom",
|
|
14869
|
-
at: replayCase.k - 1,
|
|
14870
|
-
describe: "arm-B corrected step",
|
|
14871
|
-
apply: (step) => ({
|
|
14872
|
-
...step,
|
|
14873
|
-
span: { ...step.span, args: { command: fixCommand } }
|
|
14874
|
-
})
|
|
14875
|
-
},
|
|
14876
|
-
runner
|
|
14877
|
-
);
|
|
14878
|
-
const exec = runner.lastArm;
|
|
14879
|
-
if (!exec) throw new Error("arm B finished without executing the corrected step");
|
|
14880
|
-
const output = `${exec.stdout}
|
|
14881
|
-
${exec.stderr}`;
|
|
14882
|
-
return {
|
|
14883
|
-
exitCode: exec.exitCode,
|
|
14884
|
-
prefixExecuted: runner.lastPrefix?.prefixExecuted ?? 0,
|
|
14885
|
-
prefixDivergences: runner.lastPrefix?.prefixDivergences.length ?? 0,
|
|
14886
|
-
failureVanished: exec.exitCode === 0 && (signature ? !output.includes(signature) : true),
|
|
14887
|
-
stdout: exec.stdout,
|
|
14888
|
-
stderr: exec.stderr
|
|
14889
|
-
};
|
|
14890
|
-
}
|
|
14891
|
-
function rate(numerator, denominator) {
|
|
14892
|
-
return denominator > 0 ? numerator / denominator : null;
|
|
14893
|
-
}
|
|
14894
|
-
async function runReplayBatch(options) {
|
|
14895
|
-
const onProgress = options.onProgress ?? (() => {
|
|
14896
|
-
});
|
|
14897
|
-
const enumeration = enumerateReplayableCases(options.corpora);
|
|
14898
|
-
let selected = enumeration.replayable;
|
|
14899
|
-
if (options.caseFilter) {
|
|
14900
|
-
selected = selected.filter((c) => c.trajId.includes(options.caseFilter));
|
|
14901
|
-
}
|
|
14902
|
-
if (options.caseLimit !== void 0) selected = selected.slice(0, options.caseLimit);
|
|
14903
|
-
onProgress(
|
|
14904
|
-
`enumerated ${enumeration.labelEntryCount} label entries \u2192 ${enumeration.replayable.length} replayable, ${enumeration.excluded.length} excluded; executing ${selected.length}`
|
|
14905
|
-
);
|
|
14906
|
-
const preparer = options.preparer ?? dockerImagePreparer();
|
|
14907
|
-
const backendFactory = options.backendFactory ?? ((derivedImage) => sandboxReplayBackend({
|
|
14908
|
-
image: derivedImage,
|
|
14909
|
-
apiKey: requireOption2(options.apiKey, "apiKey"),
|
|
14910
|
-
baseUrl: requireOption2(options.baseUrl, "baseUrl"),
|
|
14911
|
-
maxLifetimeSeconds: options.maxLifetimeSeconds ?? 3600
|
|
14912
|
-
}));
|
|
14913
|
-
mkdirSync2(options.out, { recursive: true });
|
|
14914
|
-
const progressPath = join26(options.out, "cases.jsonl");
|
|
14915
|
-
const rows = [];
|
|
14916
|
-
const pullFailures = [];
|
|
14917
|
-
const verdictByTraj = /* @__PURE__ */ new Map();
|
|
14918
|
-
for (const [index, replayCase] of selected.entries()) {
|
|
14919
|
-
const caseStart = Date.now();
|
|
14920
|
-
const label = `[${index + 1}/${selected.length}] ${replayCase.corpus}/${replayCase.trajId}`;
|
|
14921
|
-
onProgress(`${label}: preparing image ${replayCase.image}`);
|
|
14922
|
-
const preparation = await preparer.ensure(replayCase.image, replayCase.cwd);
|
|
14923
|
-
const base = {
|
|
14924
|
-
corpus: replayCase.corpus,
|
|
14925
|
-
trajId: replayCase.trajId,
|
|
14926
|
-
image: replayCase.image,
|
|
14927
|
-
cwd: replayCase.cwd,
|
|
14928
|
-
cwdSource: replayCase.cwdSource,
|
|
14929
|
-
k: replayCase.k,
|
|
14930
|
-
stepCount: replayCase.steps.length,
|
|
14931
|
-
goldIncorrectSteps: replayCase.goldIncorrectSteps,
|
|
14932
|
-
submitGoldsSkipped: replayCase.submitGoldsSkipped,
|
|
14933
|
-
recordedReturncodeAtK: replayCase.recordedReturncodeAtK
|
|
14934
|
-
};
|
|
14935
|
-
if (!preparation.succeeded) {
|
|
14936
|
-
pullFailures.push({
|
|
14937
|
-
corpus: replayCase.corpus,
|
|
14938
|
-
trajId: replayCase.trajId,
|
|
14939
|
-
image: replayCase.image,
|
|
14940
|
-
error: preparation.error
|
|
14941
|
-
});
|
|
14942
|
-
const row = {
|
|
14943
|
-
...base,
|
|
14944
|
-
derivedImage: null,
|
|
14945
|
-
signature: null,
|
|
14946
|
-
status: "image-unavailable",
|
|
14947
|
-
error: preparation.error,
|
|
14948
|
-
imagePulled: false,
|
|
14949
|
-
imageBuilt: false,
|
|
14950
|
-
prefixExecuted: null,
|
|
14951
|
-
prefixDivergences: null,
|
|
14952
|
-
prefixDivergencePct: null,
|
|
14953
|
-
armAExit: null,
|
|
14954
|
-
armAReturncodeMatch: false,
|
|
14955
|
-
armASignatureMatch: false,
|
|
14956
|
-
replayed: false,
|
|
14957
|
-
fix: null,
|
|
14958
|
-
wallMs: Date.now() - caseStart
|
|
14959
|
-
};
|
|
14960
|
-
rows.push(row);
|
|
14961
|
-
appendFileSync(progressPath, `${JSON.stringify(row)}
|
|
14962
|
-
`);
|
|
14963
|
-
onProgress(`${label}: image unavailable \u2014 ${preparation.error}`);
|
|
14964
|
-
continue;
|
|
14965
|
-
}
|
|
14966
|
-
const { derivedImage, pulled, built } = preparation.value;
|
|
14967
|
-
const stepTimeoutMs = options.stepTimeoutMs ?? replayCase.recordedStepTimeoutMs ?? 12e4;
|
|
14968
|
-
const caseOut = join26(options.out, caseOutDirName(replayCase));
|
|
14969
|
-
onProgress(`${label}: arm A on ${derivedImage} (k=${replayCase.k}, timeout ${stepTimeoutMs}ms)`);
|
|
14970
|
-
try {
|
|
14971
|
-
const verdict = await replayVerify({
|
|
14972
|
-
stepsPath: replayCase.stepsPath,
|
|
14973
|
-
image: derivedImage,
|
|
14974
|
-
at: replayCase.k,
|
|
14975
|
-
cwd: replayCase.cwd,
|
|
14976
|
-
out: caseOut,
|
|
14977
|
-
caseId: replayCase.trajId,
|
|
14978
|
-
stepTimeoutMs,
|
|
14979
|
-
prefixLimit: options.prefixLimit,
|
|
14980
|
-
backend: backendFactory(derivedImage),
|
|
14981
|
-
onProgress: (message) => onProgress(`${label}: ${message}`)
|
|
14982
|
-
});
|
|
14983
|
-
verdictByTraj.set(replayCase.trajId, verdict);
|
|
14984
|
-
const divergencePct = verdict.prefixExecuted > 0 ? verdict.prefixDivergences.length / verdict.prefixExecuted * 100 : 0;
|
|
14985
|
-
const returncodeMatch = verdict.recordedReturncode !== null && verdict.armA.exitCode === verdict.recordedReturncode;
|
|
14986
|
-
const row = {
|
|
14987
|
-
...base,
|
|
14988
|
-
derivedImage,
|
|
14989
|
-
signature: verdict.signature,
|
|
14990
|
-
status: "ok",
|
|
14991
|
-
error: null,
|
|
14992
|
-
imagePulled: pulled,
|
|
14993
|
-
imageBuilt: built,
|
|
14994
|
-
prefixExecuted: verdict.prefixExecuted,
|
|
14995
|
-
prefixDivergences: verdict.prefixDivergences.length,
|
|
14996
|
-
prefixDivergencePct: Number(divergencePct.toFixed(1)),
|
|
14997
|
-
armAExit: verdict.armA.exitCode,
|
|
14998
|
-
armAReturncodeMatch: returncodeMatch,
|
|
14999
|
-
armASignatureMatch: verdict.armA.failureSignatureMatch,
|
|
15000
|
-
replayed: divergencePct <= 10 && returncodeMatch,
|
|
15001
|
-
fix: null,
|
|
15002
|
-
wallMs: Date.now() - caseStart
|
|
15003
|
-
};
|
|
15004
|
-
rows.push(row);
|
|
15005
|
-
appendFileSync(progressPath, `${JSON.stringify(row)}
|
|
15006
|
-
`);
|
|
15007
|
-
onProgress(
|
|
15008
|
-
`${label}: armA exit=${verdict.armA.exitCode} rcMatch=${returncodeMatch} divergences=${verdict.prefixDivergences.length}/${verdict.prefixExecuted}`
|
|
15009
|
-
);
|
|
15010
|
-
} catch (err) {
|
|
15011
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
15012
|
-
const row = {
|
|
15013
|
-
...base,
|
|
15014
|
-
derivedImage,
|
|
15015
|
-
signature: null,
|
|
15016
|
-
status: "replay-error",
|
|
15017
|
-
error: message.slice(0, 500),
|
|
15018
|
-
imagePulled: pulled,
|
|
15019
|
-
imageBuilt: built,
|
|
15020
|
-
prefixExecuted: null,
|
|
15021
|
-
prefixDivergences: null,
|
|
15022
|
-
prefixDivergencePct: null,
|
|
15023
|
-
armAExit: null,
|
|
15024
|
-
armAReturncodeMatch: false,
|
|
15025
|
-
armASignatureMatch: false,
|
|
15026
|
-
replayed: false,
|
|
15027
|
-
fix: null,
|
|
15028
|
-
wallMs: Date.now() - caseStart
|
|
15029
|
-
};
|
|
15030
|
-
rows.push(row);
|
|
15031
|
-
appendFileSync(progressPath, `${JSON.stringify(row)}
|
|
15032
|
-
`);
|
|
15033
|
-
onProgress(`${label}: replay error \u2014 ${message.slice(0, 200)}`);
|
|
15034
|
-
}
|
|
15035
|
-
}
|
|
15036
|
-
let llm = null;
|
|
15037
|
-
if (options.fix !== "none") {
|
|
15038
|
-
const caller = options.fixCaller;
|
|
15039
|
-
if (!caller) throw new Error(`replay-verify-batch: fix=${options.fix} requires a fixCaller`);
|
|
15040
|
-
const fixAttempts = options.fixAttempts ?? 3;
|
|
15041
|
-
const maxFixCases = options.maxFixCases ?? 30;
|
|
15042
|
-
const seed = options.seed ?? 17;
|
|
15043
|
-
const eligible = rows.filter((r) => r.status === "ok" && r.replayed);
|
|
15044
|
-
const sampled = eligible.length > maxFixCases ? seededSample(eligible, maxFixCases, seed) : new Set(eligible);
|
|
15045
|
-
if (eligible.length > maxFixCases) {
|
|
15046
|
-
onProgress(
|
|
15047
|
-
`fix phase: ${eligible.length} eligible > cap ${maxFixCases}; seeded sample (seed=${seed})`
|
|
15048
|
-
);
|
|
15049
|
-
}
|
|
15050
|
-
let calls = 0;
|
|
15051
|
-
let failures = 0;
|
|
15052
|
-
let promptTokens = 0;
|
|
15053
|
-
let completionTokens = 0;
|
|
15054
|
-
for (const row of eligible) {
|
|
15055
|
-
const index = rows.indexOf(row);
|
|
15056
|
-
if (!sampled.has(row)) {
|
|
15057
|
-
rows[index] = {
|
|
15058
|
-
...row,
|
|
15059
|
-
fix: {
|
|
15060
|
-
attempted: false,
|
|
15061
|
-
sampledOut: true,
|
|
15062
|
-
command: null,
|
|
15063
|
-
llmError: null,
|
|
15064
|
-
usage: null,
|
|
15065
|
-
armBExit: null,
|
|
15066
|
-
armBPrefixExecuted: null,
|
|
15067
|
-
armBPrefixDivergences: null,
|
|
15068
|
-
failureVanished: null,
|
|
15069
|
-
armBError: null,
|
|
15070
|
-
attempts: null,
|
|
15071
|
-
flippedAtAttempt: null
|
|
15072
|
-
}
|
|
15073
|
-
};
|
|
15074
|
-
continue;
|
|
15075
|
-
}
|
|
15076
|
-
const replayCase = selected.find((c) => c.trajId === row.trajId && c.corpus === row.corpus);
|
|
15077
|
-
const label = `fix ${row.corpus}/${row.trajId}`;
|
|
15078
|
-
const signature = verdictByTraj.get(row.trajId)?.signature ?? null;
|
|
15079
|
-
const stepTimeoutMs = options.stepTimeoutMs ?? replayCase.recordedStepTimeoutMs ?? 12e4;
|
|
15080
|
-
const promptInput = {
|
|
15081
|
-
taskStatement: replayCase.taskStatement,
|
|
15082
|
-
steps: replayCase.steps,
|
|
15083
|
-
k: replayCase.k
|
|
15084
|
-
};
|
|
15085
|
-
const caseOut = join26(options.out, caseOutDirName(row));
|
|
15086
|
-
if (options.fix === "loop") {
|
|
15087
|
-
onProgress(`${label}: fix loop (budget ${fixAttempts} attempts)`);
|
|
15088
|
-
const result = await runFixLoop(
|
|
15089
|
-
caller,
|
|
15090
|
-
promptInput,
|
|
15091
|
-
async (command, attempt) => {
|
|
15092
|
-
onProgress(
|
|
15093
|
-
`${label}: attempt ${attempt} arm B \u2014 ${command.split("\n")[0].slice(0, 120)}`
|
|
15094
|
-
);
|
|
15095
|
-
const armB = await executeArmB(
|
|
15096
|
-
replayCase,
|
|
15097
|
-
command,
|
|
15098
|
-
backendFactory(row.derivedImage),
|
|
15099
|
-
signature,
|
|
15100
|
-
stepTimeoutMs,
|
|
15101
|
-
options.prefixLimit,
|
|
15102
|
-
(message) => onProgress(`${label}: attempt ${attempt}: ${message}`)
|
|
15103
|
-
);
|
|
15104
|
-
mkdirSync2(caseOut, { recursive: true });
|
|
15105
|
-
writeFileSync2(
|
|
15106
|
-
join26(caseOut, `armB-attempt${attempt}-result.json`),
|
|
15107
|
-
`${JSON.stringify({ command, ...armB }, null, 2)}
|
|
15108
|
-
`
|
|
15109
|
-
);
|
|
15110
|
-
writeFileSync2(
|
|
15111
|
-
join26(caseOut, "armB-result.json"),
|
|
15112
|
-
`${JSON.stringify({ command, attempt, ...armB }, null, 2)}
|
|
15113
|
-
`
|
|
15114
|
-
);
|
|
15115
|
-
return armB;
|
|
15116
|
-
},
|
|
15117
|
-
{ maxAttempts: fixAttempts, onProgress: (message) => onProgress(`${label}: ${message}`) }
|
|
15118
|
-
);
|
|
15119
|
-
calls += result.llmCalls;
|
|
15120
|
-
failures += result.llmFailures;
|
|
15121
|
-
promptTokens += result.promptTokens;
|
|
15122
|
-
completionTokens += result.completionTokens;
|
|
15123
|
-
const usage = result.promptTokens + result.completionTokens > 0 ? { promptTokens: result.promptTokens, completionTokens: result.completionTokens } : null;
|
|
15124
|
-
const flippedRecord = result.flippedAtAttempt !== null ? result.attempts.find((a) => a.attempt === result.flippedAtAttempt) : null;
|
|
15125
|
-
const summary = flippedRecord ?? [...result.attempts].reverse().find((a) => a.executed) ?? null;
|
|
15126
|
-
const lastRecord = result.attempts.at(-1) ?? null;
|
|
15127
|
-
rows[index] = {
|
|
15128
|
-
...row,
|
|
15129
|
-
fix: summary ? {
|
|
15130
|
-
attempted: true,
|
|
15131
|
-
sampledOut: false,
|
|
15132
|
-
command: summary.command,
|
|
15133
|
-
llmError: null,
|
|
15134
|
-
usage,
|
|
15135
|
-
armBExit: summary.exitCode,
|
|
15136
|
-
armBPrefixExecuted: summary.prefixExecuted,
|
|
15137
|
-
armBPrefixDivergences: summary.prefixDivergences,
|
|
15138
|
-
failureVanished: summary.failureVanished,
|
|
15139
|
-
armBError: null,
|
|
15140
|
-
attempts: result.attempts,
|
|
15141
|
-
flippedAtAttempt: result.flippedAtAttempt
|
|
15142
|
-
} : result.aborted ? {
|
|
15143
|
-
attempted: true,
|
|
15144
|
-
sampledOut: false,
|
|
15145
|
-
command: lastRecord?.command ?? null,
|
|
15146
|
-
llmError: null,
|
|
15147
|
-
usage,
|
|
15148
|
-
armBExit: null,
|
|
15149
|
-
armBPrefixExecuted: null,
|
|
15150
|
-
armBPrefixDivergences: null,
|
|
15151
|
-
failureVanished: null,
|
|
15152
|
-
armBError: lastRecord?.armBError ?? "sandbox error",
|
|
15153
|
-
attempts: result.attempts,
|
|
15154
|
-
flippedAtAttempt: null
|
|
15155
|
-
} : {
|
|
15156
|
-
attempted: true,
|
|
15157
|
-
sampledOut: false,
|
|
15158
|
-
command: null,
|
|
15159
|
-
llmError: lastRecord?.llmError ?? "no attempt produced a runnable fix",
|
|
15160
|
-
usage,
|
|
15161
|
-
armBExit: null,
|
|
15162
|
-
armBPrefixExecuted: null,
|
|
15163
|
-
armBPrefixDivergences: null,
|
|
15164
|
-
failureVanished: null,
|
|
15165
|
-
armBError: null,
|
|
15166
|
-
attempts: result.attempts,
|
|
15167
|
-
flippedAtAttempt: null
|
|
15168
|
-
}
|
|
15169
|
-
};
|
|
15170
|
-
onProgress(
|
|
15171
|
-
`${label}: loop done \u2014 flipped=${result.flipped}` + (result.flippedAtAttempt !== null ? ` at attempt ${result.flippedAtAttempt}` : "") + ` (${result.llmCalls} calls, ${result.attempts.filter((a) => a.executed).length} arms)`
|
|
15172
|
-
);
|
|
15173
|
-
continue;
|
|
15174
|
-
}
|
|
15175
|
-
onProgress(`${label}: generating corrected command`);
|
|
15176
|
-
calls += 1;
|
|
15177
|
-
const generated = await generateFixCommand(caller, promptInput);
|
|
15178
|
-
if (!generated.succeeded) {
|
|
15179
|
-
failures += 1;
|
|
15180
|
-
rows[index] = {
|
|
15181
|
-
...row,
|
|
15182
|
-
fix: {
|
|
15183
|
-
attempted: true,
|
|
15184
|
-
sampledOut: false,
|
|
15185
|
-
command: null,
|
|
15186
|
-
llmError: generated.error,
|
|
15187
|
-
usage: null,
|
|
15188
|
-
armBExit: null,
|
|
15189
|
-
armBPrefixExecuted: null,
|
|
15190
|
-
armBPrefixDivergences: null,
|
|
15191
|
-
failureVanished: null,
|
|
15192
|
-
armBError: null,
|
|
15193
|
-
attempts: null,
|
|
15194
|
-
flippedAtAttempt: null
|
|
15195
|
-
}
|
|
15196
|
-
};
|
|
15197
|
-
onProgress(`${label}: LLM failed \u2014 ${generated.error.slice(0, 200)}`);
|
|
15198
|
-
continue;
|
|
15199
|
-
}
|
|
15200
|
-
promptTokens += generated.value.usage?.promptTokens ?? 0;
|
|
15201
|
-
completionTokens += generated.value.usage?.completionTokens ?? 0;
|
|
15202
|
-
onProgress(`${label}: arm B \u2014 ${generated.value.command.split("\n")[0].slice(0, 120)}`);
|
|
15203
|
-
try {
|
|
15204
|
-
const armB = await executeArmB(
|
|
15205
|
-
replayCase,
|
|
15206
|
-
generated.value.command,
|
|
15207
|
-
backendFactory(row.derivedImage),
|
|
15208
|
-
signature,
|
|
15209
|
-
stepTimeoutMs,
|
|
15210
|
-
options.prefixLimit,
|
|
15211
|
-
(message) => onProgress(`${label}: ${message}`)
|
|
15212
|
-
);
|
|
15213
|
-
rows[index] = {
|
|
15214
|
-
...row,
|
|
15215
|
-
fix: {
|
|
15216
|
-
attempted: true,
|
|
15217
|
-
sampledOut: false,
|
|
15218
|
-
command: generated.value.command,
|
|
15219
|
-
llmError: null,
|
|
15220
|
-
usage: generated.value.usage,
|
|
15221
|
-
armBExit: armB.exitCode,
|
|
15222
|
-
armBPrefixExecuted: armB.prefixExecuted,
|
|
15223
|
-
armBPrefixDivergences: armB.prefixDivergences,
|
|
15224
|
-
failureVanished: armB.failureVanished,
|
|
15225
|
-
armBError: null,
|
|
15226
|
-
attempts: null,
|
|
15227
|
-
flippedAtAttempt: armB.failureVanished ? 1 : null
|
|
15228
|
-
}
|
|
15229
|
-
};
|
|
15230
|
-
mkdirSync2(caseOut, { recursive: true });
|
|
15231
|
-
writeFileSync2(
|
|
15232
|
-
join26(caseOut, "armB-result.json"),
|
|
15233
|
-
`${JSON.stringify({ command: generated.value.command, ...armB }, null, 2)}
|
|
15234
|
-
`
|
|
15235
|
-
);
|
|
15236
|
-
onProgress(`${label}: armB exit=${armB.exitCode} failureVanished=${armB.failureVanished}`);
|
|
15237
|
-
} catch (err) {
|
|
15238
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
15239
|
-
rows[index] = {
|
|
15240
|
-
...row,
|
|
15241
|
-
fix: {
|
|
15242
|
-
attempted: true,
|
|
15243
|
-
sampledOut: false,
|
|
15244
|
-
command: generated.value.command,
|
|
15245
|
-
llmError: null,
|
|
15246
|
-
usage: generated.value.usage,
|
|
15247
|
-
armBExit: null,
|
|
15248
|
-
armBPrefixExecuted: null,
|
|
15249
|
-
armBPrefixDivergences: null,
|
|
15250
|
-
failureVanished: null,
|
|
15251
|
-
armBError: message.slice(0, 500),
|
|
15252
|
-
attempts: null,
|
|
15253
|
-
flippedAtAttempt: null
|
|
15254
|
-
}
|
|
15255
|
-
};
|
|
15256
|
-
onProgress(`${label}: arm B error \u2014 ${message.slice(0, 200)}`);
|
|
15257
|
-
}
|
|
15258
|
-
}
|
|
15259
|
-
llm = {
|
|
15260
|
-
model: options.fixModelLabel ?? "unknown",
|
|
15261
|
-
calls,
|
|
15262
|
-
failures,
|
|
15263
|
-
promptTokens,
|
|
15264
|
-
completionTokens
|
|
15265
|
-
};
|
|
15266
|
-
}
|
|
15267
|
-
const executedRows = rows.filter((r) => r.status === "ok");
|
|
15268
|
-
const replayed = executedRows.filter((r) => r.replayed);
|
|
15269
|
-
const signatureStrict = executedRows.filter(
|
|
15270
|
-
(r) => r.replayed && r.armASignatureMatch
|
|
15271
|
-
);
|
|
15272
|
-
const armBExecuted = rows.filter((r) => r.fix?.attempted && r.fix.failureVanished !== null);
|
|
15273
|
-
const flipped = armBExecuted.filter((r) => r.fix.failureVanished === true);
|
|
15274
|
-
const armBNonzeroRc = armBExecuted.filter(
|
|
15275
|
-
(r) => r.recordedReturncodeAtK !== null && r.recordedReturncodeAtK !== 0
|
|
15276
|
-
);
|
|
15277
|
-
const flippedNonzeroRc = armBNonzeroRc.filter((r) => r.fix.failureVanished === true);
|
|
15278
|
-
const attempt1Executed = rows.filter(
|
|
15279
|
-
(r) => r.fix?.attempts?.find((a) => a.attempt === 1)?.executed === true
|
|
15280
|
-
);
|
|
15281
|
-
const flippedAt1 = rows.filter((r) => r.fix?.flippedAtAttempt === 1);
|
|
15282
|
-
const flipsByAttempt = {};
|
|
15283
|
-
for (const row of rows) {
|
|
15284
|
-
const at = row.fix?.flippedAtAttempt;
|
|
15285
|
-
if (typeof at === "number") flipsByAttempt[String(at)] = (flipsByAttempt[String(at)] ?? 0) + 1;
|
|
15286
|
-
}
|
|
15287
|
-
const excludedByReason = {};
|
|
15288
|
-
for (const excluded of enumeration.excluded) {
|
|
15289
|
-
excludedByReason[excluded.reason] = (excludedByReason[excluded.reason] ?? 0) + 1;
|
|
15290
|
-
}
|
|
15291
|
-
const submitGoldsByCorpus = {};
|
|
15292
|
-
const submitEntry = (corpus) => submitGoldsByCorpus[corpus] ??= { submitOnlyCases: 0, goldsSkippedWithinReplayable: 0 };
|
|
15293
|
-
for (const excluded of enumeration.excluded) {
|
|
15294
|
-
if (excluded.reason === "gold-only-submit-step") submitEntry(excluded.corpus).submitOnlyCases += 1;
|
|
15295
|
-
}
|
|
15296
|
-
for (const replayCase of enumeration.replayable) {
|
|
15297
|
-
if (replayCase.submitGoldsSkipped > 0) {
|
|
15298
|
-
submitEntry(replayCase.corpus).goldsSkippedWithinReplayable += replayCase.submitGoldsSkipped;
|
|
15299
|
-
}
|
|
15300
|
-
}
|
|
15301
|
-
const report = {
|
|
15302
|
-
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
15303
|
-
corpora: options.corpora.map((c) => ({
|
|
15304
|
-
name: c.name,
|
|
15305
|
-
labelsPath: c.labelsPath,
|
|
15306
|
-
preparedDir: c.preparedDir
|
|
15307
|
-
})),
|
|
15308
|
-
totals: {
|
|
15309
|
-
labelEntries: enumeration.labelEntryCount,
|
|
15310
|
-
replayable: enumeration.replayable.length,
|
|
15311
|
-
executed: selected.length,
|
|
15312
|
-
excludedByReason,
|
|
15313
|
-
submitGoldsByCorpus
|
|
15314
|
-
},
|
|
15315
|
-
headline: {
|
|
15316
|
-
replayabilityRate: {
|
|
15317
|
-
numerator: replayed.length,
|
|
15318
|
-
denominator: selected.length,
|
|
15319
|
-
value: rate(replayed.length, selected.length)
|
|
15320
|
-
},
|
|
15321
|
-
signatureStrictRate: {
|
|
15322
|
-
numerator: signatureStrict.length,
|
|
15323
|
-
denominator: selected.length,
|
|
15324
|
-
value: rate(signatureStrict.length, selected.length)
|
|
15325
|
-
},
|
|
15326
|
-
fixFlipRate: options.fix !== "none" ? {
|
|
15327
|
-
numerator: flipped.length,
|
|
15328
|
-
denominator: armBExecuted.length,
|
|
15329
|
-
value: rate(flipped.length, armBExecuted.length)
|
|
15330
|
-
} : null,
|
|
15331
|
-
fixFlipRateNonzeroRc: options.fix !== "none" ? {
|
|
15332
|
-
numerator: flippedNonzeroRc.length,
|
|
15333
|
-
denominator: armBNonzeroRc.length,
|
|
15334
|
-
value: rate(flippedNonzeroRc.length, armBNonzeroRc.length)
|
|
15335
|
-
} : null,
|
|
15336
|
-
fixFlipAttempt1: options.fix === "loop" ? {
|
|
15337
|
-
numerator: flippedAt1.length,
|
|
15338
|
-
denominator: attempt1Executed.length,
|
|
15339
|
-
value: rate(flippedAt1.length, attempt1Executed.length)
|
|
15340
|
-
} : null,
|
|
15341
|
-
flipsByAttempt: options.fix === "loop" ? flipsByAttempt : null
|
|
15342
|
-
},
|
|
15343
|
-
llm,
|
|
15344
|
-
excluded: enumeration.excluded,
|
|
15345
|
-
pullFailures,
|
|
15346
|
-
cases: rows
|
|
15347
|
-
};
|
|
15348
|
-
writeFileSync2(join26(options.out, "batch-report.json"), `${JSON.stringify(report, null, 2)}
|
|
15349
|
-
`);
|
|
15350
|
-
writeFileSync2(join26(options.out, "batch-report.md"), renderBatchReport(report));
|
|
15351
|
-
return report;
|
|
15352
|
-
}
|
|
15353
|
-
function requireOption2(value, name) {
|
|
15354
|
-
if (value === void 0) throw new Error(`replay-verify-batch: ${name} is required`);
|
|
15355
|
-
return value;
|
|
15356
|
-
}
|
|
15357
|
-
function pct(value) {
|
|
15358
|
-
return value === null ? "\u2014" : `${(value * 100).toFixed(1)}%`;
|
|
15359
|
-
}
|
|
15360
|
-
function renderBatchReport(report) {
|
|
15361
|
-
const lines = [];
|
|
15362
|
-
const { headline, totals } = report;
|
|
15363
|
-
lines.push("# Replay-verify batch report");
|
|
15364
|
-
lines.push("");
|
|
15365
|
-
lines.push(`Generated ${report.generatedAt}.`);
|
|
15366
|
-
lines.push("");
|
|
15367
|
-
lines.push("## Headline");
|
|
15368
|
-
lines.push("");
|
|
15369
|
-
lines.push(
|
|
15370
|
-
`- **Replayability rate: ${pct(headline.replayabilityRate.value)}** (${headline.replayabilityRate.numerator}/${headline.replayabilityRate.denominator} replayable cases where the prefix replayed with \u226410% returncode divergence AND arm A reproduced the recorded returncode at the gold step k).`
|
|
15371
|
-
);
|
|
15372
|
-
lines.push(
|
|
15373
|
-
`- Signature-strict rate: ${pct(headline.signatureStrictRate.value)} (${headline.signatureStrictRate.numerator}/${headline.signatureStrictRate.denominator}; additionally requires the recorded error substring in arm A output).`
|
|
15374
|
-
);
|
|
15375
|
-
if (headline.fixFlipRate) {
|
|
15376
|
-
lines.push(
|
|
15377
|
-
`- **Fix-flip rate: ${pct(headline.fixFlipRate.value)}** (${headline.fixFlipRate.numerator}/${headline.fixFlipRate.denominator} arm-B-executed cases where the generated fix made the failure vanish).`
|
|
15378
|
-
);
|
|
15379
|
-
}
|
|
15380
|
-
if (headline.fixFlipRateNonzeroRc) {
|
|
15381
|
-
lines.push(
|
|
15382
|
-
`- Fix-flip rate on recorded-rc\u22600 cases: ${pct(headline.fixFlipRateNonzeroRc.value)} (${headline.fixFlipRateNonzeroRc.numerator}/${headline.fixFlipRateNonzeroRc.denominator}; real recorded failures \u2014 a gold step recorded with rc 0 flips vacuously).`
|
|
15383
|
-
);
|
|
15384
|
-
}
|
|
15385
|
-
if (headline.fixFlipAttempt1) {
|
|
15386
|
-
lines.push(
|
|
15387
|
-
`- Fix-flip@1: ${pct(headline.fixFlipAttempt1.value)} (${headline.fixFlipAttempt1.numerator}/${headline.fixFlipAttempt1.denominator} cases whose attempt 1 executed \u2014 the one-shot-comparable number).`
|
|
15388
|
-
);
|
|
15389
|
-
}
|
|
15390
|
-
if (headline.flipsByAttempt && Object.keys(headline.flipsByAttempt).length > 0) {
|
|
15391
|
-
const parts = Object.entries(headline.flipsByAttempt).sort((a, b) => Number(a[0]) - Number(b[0])).map(([attempt, count2]) => `attempt ${attempt}: ${count2}`);
|
|
15392
|
-
lines.push(`- Flips by attempt: ${parts.join(", ")}.`);
|
|
15393
|
-
}
|
|
15394
|
-
lines.push("");
|
|
15395
|
-
lines.push("## Enumeration");
|
|
15396
|
-
lines.push("");
|
|
15397
|
-
lines.push(
|
|
15398
|
-
`${totals.labelEntries} label entries across ${report.corpora.length} corpora \u2192 ${totals.replayable} replayable (SWE-style docker image + \u22651 gold incorrect step), ${totals.executed} executed.`
|
|
15399
|
-
);
|
|
15400
|
-
lines.push("");
|
|
15401
|
-
lines.push("| exclusion reason | count |");
|
|
15402
|
-
lines.push("| --- | --- |");
|
|
15403
|
-
for (const [reason, count2] of Object.entries(totals.excludedByReason).sort((a, b) => b[1] - a[1])) {
|
|
15404
|
-
lines.push(`| ${reason} | ${count2} |`);
|
|
15405
|
-
}
|
|
15406
|
-
const submitEntries = Object.entries(totals.submitGoldsByCorpus);
|
|
15407
|
-
if (submitEntries.length > 0) {
|
|
15408
|
-
lines.push("");
|
|
15409
|
-
lines.push(
|
|
15410
|
-
"Submit-command golds are never counterfactual targets (a gold on the submit step marks a bad submit decision, not a failed command):"
|
|
15411
|
-
);
|
|
15412
|
-
lines.push("");
|
|
15413
|
-
lines.push("| corpus | cases excluded (all golds = submit) | golds skipped within replayable cases |");
|
|
15414
|
-
lines.push("| --- | --- | --- |");
|
|
15415
|
-
for (const [corpus, stats] of submitEntries.sort((a, b) => a[0].localeCompare(b[0]))) {
|
|
15416
|
-
lines.push(`| ${corpus} | ${stats.submitOnlyCases} | ${stats.goldsSkippedWithinReplayable} |`);
|
|
15417
|
-
}
|
|
15418
|
-
}
|
|
15419
|
-
if (report.pullFailures.length > 0) {
|
|
15420
|
-
lines.push("");
|
|
15421
|
-
lines.push("## Image pull/build failures");
|
|
15422
|
-
lines.push("");
|
|
15423
|
-
lines.push("| corpus | trajectory | image | error |");
|
|
15424
|
-
lines.push("| --- | --- | --- | --- |");
|
|
15425
|
-
for (const failure2 of report.pullFailures) {
|
|
15426
|
-
lines.push(
|
|
15427
|
-
`| ${failure2.corpus} | ${failure2.trajId} | \`${failure2.image}\` | ${failure2.error.replaceAll("|", "\\|")} |`
|
|
15428
|
-
);
|
|
15429
|
-
}
|
|
15430
|
-
}
|
|
15431
|
-
lines.push("");
|
|
15432
|
-
lines.push("## Per-case results");
|
|
15433
|
-
lines.push("");
|
|
15434
|
-
lines.push(
|
|
15435
|
-
"| corpus | trajectory | k | rc@k | prefix | div | div% | armA exit | rc match | sig match | replayed | fix | armB exit | vanished | wall s |"
|
|
15436
|
-
);
|
|
15437
|
-
lines.push(
|
|
15438
|
-
"| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |"
|
|
15439
|
-
);
|
|
15440
|
-
for (const row of report.cases) {
|
|
15441
|
-
const fix = row.fix;
|
|
15442
|
-
const fixCell = !fix ? "\u2014" : fix.sampledOut ? "sampled-out" : fix.llmError ? "llm-failed" : fix.armBError ? "armB-error" : fix.attempts ? fix.flippedAtAttempt !== null ? `flip@${fix.flippedAtAttempt}` : `exhausted(${fix.attempts.length})` : "generated";
|
|
15443
|
-
lines.push(
|
|
15444
|
-
[
|
|
15445
|
-
row.corpus,
|
|
15446
|
-
row.trajId.length > 48 ? `${row.trajId.slice(0, 45)}\u2026` : row.trajId,
|
|
15447
|
-
row.k,
|
|
15448
|
-
row.recordedReturncodeAtK ?? "null",
|
|
15449
|
-
row.prefixExecuted ?? "\u2014",
|
|
15450
|
-
row.prefixDivergences ?? "\u2014",
|
|
15451
|
-
row.prefixDivergencePct ?? "\u2014",
|
|
15452
|
-
row.status === "ok" ? row.armAExit : row.status,
|
|
15453
|
-
row.armAReturncodeMatch ? "yes" : "no",
|
|
15454
|
-
row.armASignatureMatch ? "yes" : "no",
|
|
15455
|
-
row.replayed ? "**yes**" : "no",
|
|
15456
|
-
fixCell,
|
|
15457
|
-
fix?.armBExit ?? "\u2014",
|
|
15458
|
-
fix?.failureVanished === null || fix === null ? "\u2014" : fix.failureVanished ? "**yes**" : "no",
|
|
15459
|
-
(row.wallMs / 1e3).toFixed(1)
|
|
15460
|
-
].join(" | ")
|
|
15461
|
-
);
|
|
15462
|
-
}
|
|
15463
|
-
if (report.llm) {
|
|
15464
|
-
lines.push("");
|
|
15465
|
-
lines.push("## LLM fix generation");
|
|
15466
|
-
lines.push("");
|
|
15467
|
-
lines.push(
|
|
15468
|
-
`Model ${report.llm.model}: ${report.llm.calls} calls (${report.llm.failures} failed), ${report.llm.promptTokens} prompt + ${report.llm.completionTokens} completion tokens.`
|
|
15469
|
-
);
|
|
15470
|
-
}
|
|
15471
|
-
lines.push("");
|
|
15472
|
-
return lines.join("\n");
|
|
15473
|
-
}
|
|
15474
|
-
function parseReplayBatchArgs(argv) {
|
|
15475
|
-
const corpora = [];
|
|
15476
|
-
const values = /* @__PURE__ */ new Map();
|
|
15477
|
-
let enumerateOnly = false;
|
|
15478
|
-
for (let i = 0; i < argv.length; i++) {
|
|
15479
|
-
const arg = argv[i];
|
|
15480
|
-
if (arg === "--help" || arg === "-h") return "help";
|
|
15481
|
-
if (arg === "--enumerate-only") {
|
|
15482
|
-
enumerateOnly = true;
|
|
15483
|
-
continue;
|
|
15484
|
-
}
|
|
15485
|
-
if (!arg.startsWith("--")) throw new Error(`unexpected positional argument: ${arg}`);
|
|
15486
|
-
const value = argv[++i];
|
|
15487
|
-
if (value === void 0) throw new Error(`missing value for ${arg}`);
|
|
15488
|
-
if (arg === "--corpus") corpora.push(parseCorpusFlag(value));
|
|
15489
|
-
else values.set(arg, value);
|
|
15490
|
-
}
|
|
15491
|
-
if (corpora.length === 0) {
|
|
15492
|
-
throw new Error("replay-verify-batch: at least one --corpus name=<labels>::<prepared> is required");
|
|
15493
|
-
}
|
|
15494
|
-
const out = values.get("--out");
|
|
15495
|
-
if (!out && !enumerateOnly) throw new Error("replay-verify-batch: --out is required");
|
|
15496
|
-
const optionalNumber = (flag) => {
|
|
15497
|
-
const v = values.get(flag);
|
|
15498
|
-
if (v === void 0) return void 0;
|
|
15499
|
-
const n = Number(v);
|
|
15500
|
-
if (!Number.isFinite(n)) throw new Error(`replay-verify-batch: ${flag} must be a number, got ${v}`);
|
|
15501
|
-
return n;
|
|
15502
|
-
};
|
|
15503
|
-
const fix = values.get("--fix") ?? "none";
|
|
15504
|
-
if (fix !== "none" && fix !== "generate" && fix !== "loop") {
|
|
15505
|
-
throw new Error(`replay-verify-batch: --fix must be none, generate, or loop, got ${fix}`);
|
|
15506
|
-
}
|
|
15507
|
-
const fixAttempts = optionalNumber("--fix-attempts") ?? 3;
|
|
15508
|
-
if (!Number.isInteger(fixAttempts) || fixAttempts < 1) {
|
|
15509
|
-
throw new Error(
|
|
15510
|
-
`replay-verify-batch: --fix-attempts must be a positive integer, got ${values.get("--fix-attempts")}`
|
|
15511
|
-
);
|
|
15512
|
-
}
|
|
15513
|
-
return {
|
|
14161
|
+
const report = await runReplayBatch({
|
|
15514
14162
|
corpora,
|
|
15515
|
-
out
|
|
14163
|
+
out,
|
|
15516
14164
|
fix,
|
|
15517
14165
|
fixAttempts,
|
|
15518
|
-
fixModel: values.get("--fix-model") ?? "glm-5.2",
|
|
15519
|
-
fixBaseUrl: values.get("--fix-base-url") ?? "https://api.z.ai/api/coding/paas/v4",
|
|
15520
|
-
fixApiKeyEnv: values.get("--fix-api-key-env") ?? "ZAI_GLM_API_KEY",
|
|
15521
|
-
maxFixCases: optionalNumber("--max-fix-cases") ?? 30,
|
|
15522
|
-
seed: optionalNumber("--seed") ?? 17,
|
|
15523
|
-
stepTimeoutMs: optionalNumber("--step-timeout"),
|
|
15524
|
-
prefixLimit: optionalNumber("--prefix-limit"),
|
|
15525
|
-
caseFilter: values.get("--case-filter"),
|
|
15526
|
-
caseLimit: optionalNumber("--case-limit"),
|
|
15527
|
-
baseUrl: values.get("--base-url") ?? process.env.SANDBOX_API_URL ?? "http://127.0.0.1:4097",
|
|
15528
|
-
apiKeyEnv: values.get("--api-key-env") ?? "SANDBOX_API_KEY",
|
|
15529
|
-
maxLifetimeSeconds: optionalNumber("--max-lifetime"),
|
|
15530
|
-
enumerateOnly
|
|
15531
|
-
};
|
|
15532
|
-
}
|
|
15533
|
-
function replayBatchUsage() {
|
|
15534
|
-
return `traces replay-verify-batch \u2014 measure replayability and fix-flip rates across gold-labeled corpora
|
|
15535
|
-
|
|
15536
|
-
Usage:
|
|
15537
|
-
traces replay-verify-batch \\
|
|
15538
|
-
--corpus NAME=<labels.json>::<prepared-dir> [--corpus ...] \\
|
|
15539
|
-
--out DIR [--fix none|generate|loop] [--fix-attempts 3] [--enumerate-only] \\
|
|
15540
|
-
[--fix-model glm-5.2] [--fix-base-url URL] [--fix-api-key-env ZAI_GLM_API_KEY] \\
|
|
15541
|
-
[--max-fix-cases 30] [--seed 17] [--step-timeout MS] [--prefix-limit N] \\
|
|
15542
|
-
[--case-filter SUBSTRING] [--case-limit N] \\
|
|
15543
|
-
[--base-url URL] [--api-key-env SANDBOX_API_KEY] [--max-lifetime SECONDS]
|
|
15544
|
-
|
|
15545
|
-
Replayable = the raw trajectory carries info.docker_config.base_image AND the
|
|
15546
|
-
labels mark at least one gold incorrect step that is a real mid-trajectory
|
|
15547
|
-
action. k = the first such gold step; golds whose action is the agent's
|
|
15548
|
-
submit command (COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT) are skipped and
|
|
15549
|
-
counted, and cases whose golds are all submit steps are excluded as
|
|
15550
|
-
gold-only-submit-step. Excluded cases and image pull failures are report
|
|
15551
|
-
rows, never skipped silently. --enumerate-only prints the enumeration
|
|
15552
|
-
without touching a sandbox.
|
|
15553
|
-
|
|
15554
|
-
--fix generate runs ONE LLM call per arm-A-reproduced case (cap
|
|
15555
|
-
--max-fix-cases, seeded sample beyond it) and executes the corrected command
|
|
15556
|
-
as arm B in a fresh sandbox.
|
|
15557
|
-
|
|
15558
|
-
--fix loop iterates: attempt 1 is the one-shot prompt; when an arm fails
|
|
15559
|
-
(nonzero exit or the failure signature persists) or the model call fails,
|
|
15560
|
-
the next prompt carries every prior command with its REAL executed
|
|
15561
|
-
stdout/stderr, up to --fix-attempts attempts (default 3). Retries may
|
|
15562
|
-
answer with a short script (<=5 commands) executed as one /bin/sh unit.
|
|
15563
|
-
Every attempt runs in its own fresh sandbox with the same replayed prefix.
|
|
15564
|
-
|
|
15565
|
-
Outputs batch-report.json, batch-report.md, cases.jsonl (incremental), and one
|
|
15566
|
-
directory per case with the full replay-verify artifacts.
|
|
15567
|
-
See docs/replay-verify.md for orchestrator setup and honest limits.
|
|
15568
|
-
`;
|
|
15569
|
-
}
|
|
15570
|
-
async function cmdReplayVerifyBatch(argv) {
|
|
15571
|
-
const parsed = parseReplayBatchArgs(argv);
|
|
15572
|
-
if (parsed === "help") {
|
|
15573
|
-
process.stdout.write(replayBatchUsage());
|
|
15574
|
-
return;
|
|
15575
|
-
}
|
|
15576
|
-
if (parsed.enumerateOnly) {
|
|
15577
|
-
const enumeration = enumerateReplayableCases(parsed.corpora);
|
|
15578
|
-
process.stdout.write(
|
|
15579
|
-
`${JSON.stringify(
|
|
15580
|
-
{
|
|
15581
|
-
labelEntries: enumeration.labelEntryCount,
|
|
15582
|
-
replayable: enumeration.replayable.map((c) => ({
|
|
15583
|
-
corpus: c.corpus,
|
|
15584
|
-
trajId: c.trajId,
|
|
15585
|
-
image: c.image,
|
|
15586
|
-
cwd: c.cwd,
|
|
15587
|
-
cwdSource: c.cwdSource,
|
|
15588
|
-
k: c.k,
|
|
15589
|
-
submitGoldsSkipped: c.submitGoldsSkipped,
|
|
15590
|
-
recordedReturncodeAtK: c.recordedReturncodeAtK,
|
|
15591
|
-
goldIncorrectSteps: c.goldIncorrectSteps
|
|
15592
|
-
})),
|
|
15593
|
-
excluded: enumeration.excluded
|
|
15594
|
-
},
|
|
15595
|
-
null,
|
|
15596
|
-
2
|
|
15597
|
-
)}
|
|
15598
|
-
`
|
|
15599
|
-
);
|
|
15600
|
-
return;
|
|
15601
|
-
}
|
|
15602
|
-
const apiKey = process.env[parsed.apiKeyEnv];
|
|
15603
|
-
if (!apiKey) {
|
|
15604
|
-
throw new Error(
|
|
15605
|
-
`replay-verify-batch: env var ${parsed.apiKeyEnv} is empty \u2014 pass --api-key-env or export it`
|
|
15606
|
-
);
|
|
15607
|
-
}
|
|
15608
|
-
let fixCaller;
|
|
15609
|
-
if (parsed.fix !== "none") {
|
|
15610
|
-
const fixApiKey = process.env[parsed.fixApiKeyEnv];
|
|
15611
|
-
if (!fixApiKey) {
|
|
15612
|
-
throw new Error(
|
|
15613
|
-
`replay-verify-batch: env var ${parsed.fixApiKeyEnv} is empty \u2014 pass --fix-api-key-env or export it`
|
|
15614
|
-
);
|
|
15615
|
-
}
|
|
15616
|
-
fixCaller = zaiChatCaller({
|
|
15617
|
-
baseUrl: parsed.fixBaseUrl,
|
|
15618
|
-
apiKey: fixApiKey,
|
|
15619
|
-
model: parsed.fixModel
|
|
15620
|
-
});
|
|
15621
|
-
}
|
|
15622
|
-
const report = await runReplayBatch({
|
|
15623
|
-
corpora: parsed.corpora,
|
|
15624
|
-
out: parsed.out,
|
|
15625
|
-
baseUrl: parsed.baseUrl,
|
|
15626
|
-
apiKey,
|
|
15627
|
-
fix: parsed.fix,
|
|
15628
|
-
fixAttempts: parsed.fixAttempts,
|
|
15629
14166
|
fixCaller,
|
|
15630
|
-
fixModelLabel:
|
|
15631
|
-
maxFixCases:
|
|
15632
|
-
seed:
|
|
15633
|
-
stepTimeoutMs:
|
|
15634
|
-
prefixLimit:
|
|
15635
|
-
caseFilter:
|
|
15636
|
-
caseLimit:
|
|
15637
|
-
|
|
15638
|
-
onProgress:
|
|
15639
|
-
`)
|
|
14167
|
+
fixModelLabel: fixModel,
|
|
14168
|
+
maxFixCases: flags.number("--max-fix-cases") ?? 30,
|
|
14169
|
+
seed: flags.number("--seed") ?? 17,
|
|
14170
|
+
stepTimeoutMs: flags.number("--step-timeout"),
|
|
14171
|
+
prefixLimit: flags.number("--prefix-limit"),
|
|
14172
|
+
caseFilter: values.get("--case-filter"),
|
|
14173
|
+
caseLimit: flags.number("--case-limit"),
|
|
14174
|
+
backendFactory: sandboxBackendFactory(sandboxOptions("replay-verify-batch", values, true)),
|
|
14175
|
+
onProgress: progress
|
|
15640
14176
|
});
|
|
15641
14177
|
const { replayabilityRate, fixFlipRate } = report.headline;
|
|
15642
|
-
|
|
15643
|
-
`replay-verify-batch: replayability ${replayabilityRate.numerator}/${replayabilityRate.denominator}` + (fixFlipRate ? `, fix-flip ${fixFlipRate.numerator}/${fixFlipRate.denominator}` : "") + ` \u2192 ${
|
|
15644
|
-
`
|
|
15645
|
-
);
|
|
15646
|
-
}
|
|
15647
|
-
|
|
15648
|
-
// src/replay-wire.ts
|
|
15649
|
-
var SUBJECT_PATTERN = /^incorrect-steps-(\d+)-(\d+)-(escaped|unescaped)-consequence-(\d+)$/;
|
|
15650
|
-
function parseIncorrectStepsSubject(subject) {
|
|
15651
|
-
const match = SUBJECT_PATTERN.exec(subject);
|
|
15652
|
-
if (!match) return null;
|
|
15653
|
-
return {
|
|
15654
|
-
firstStep: Number(match[1]),
|
|
15655
|
-
lastStep: Number(match[2]),
|
|
15656
|
-
escapeStatus: match[3],
|
|
15657
|
-
consequenceStep: Number(match[4])
|
|
15658
|
-
};
|
|
15659
|
-
}
|
|
15660
|
-
function resolveFindingInvocation(finding2, corpora) {
|
|
15661
|
-
const subject = parseIncorrectStepsSubject(finding2.subject);
|
|
15662
|
-
if (!subject) {
|
|
15663
|
-
throw new Error(
|
|
15664
|
-
`replay-wire: subject '${finding2.subject}' is not incorrect-steps-<first>-<last>-<escaped|unescaped>-consequence-<step>`
|
|
15665
|
-
);
|
|
15666
|
-
}
|
|
15667
|
-
const failures = [];
|
|
15668
|
-
for (const corpus of corpora) {
|
|
15669
|
-
const resolution = resolveCaseResources(corpus, finding2.trajId);
|
|
15670
|
-
if (resolution.resolved) {
|
|
15671
|
-
const at = subject.firstStep;
|
|
15672
|
-
const step = resolution.resources.steps.find((s) => s.step_id === at);
|
|
15673
|
-
if (!step) {
|
|
15674
|
-
throw new Error(
|
|
15675
|
-
`replay-wire: finding step ${at} is outside ${finding2.trajId} (${resolution.resources.steps.length} steps)`
|
|
15676
|
-
);
|
|
15677
|
-
}
|
|
15678
|
-
return { resources: resolution.resources, subject, at };
|
|
15679
|
-
}
|
|
15680
|
-
failures.push(`${corpus.name}: ${resolution.reason}`);
|
|
15681
|
-
}
|
|
15682
|
-
throw new Error(
|
|
15683
|
-
`replay-wire: trajectory ${finding2.trajId} is not replayable in any corpus \u2014 ${failures.join("; ")}`
|
|
15684
|
-
);
|
|
15685
|
-
}
|
|
15686
|
-
async function replayVerifyFinding(finding2, options) {
|
|
15687
|
-
if (options.fixCaller && options.fixCommand !== void 0) {
|
|
15688
|
-
throw new Error("replay-wire: pass fixCaller or fixCommand, not both");
|
|
15689
|
-
}
|
|
15690
|
-
const invocation = resolveFindingInvocation(finding2, options.corpora);
|
|
15691
|
-
let fixCommand = options.fixCommand ?? null;
|
|
15692
|
-
if (options.fixCaller) {
|
|
15693
|
-
const generated = await generateFixCommand(options.fixCaller, {
|
|
15694
|
-
taskStatement: invocation.resources.taskStatement,
|
|
15695
|
-
steps: invocation.resources.steps,
|
|
15696
|
-
k: invocation.at
|
|
15697
|
-
});
|
|
15698
|
-
if (!generated.succeeded) {
|
|
15699
|
-
throw new Error(`replay-wire: fix generation failed \u2014 ${generated.error}`);
|
|
15700
|
-
}
|
|
15701
|
-
fixCommand = generated.value.command;
|
|
15702
|
-
}
|
|
15703
|
-
const verdict = await replayVerify({
|
|
15704
|
-
stepsPath: invocation.resources.stepsPath,
|
|
15705
|
-
image: invocation.resources.image,
|
|
15706
|
-
at: invocation.at,
|
|
15707
|
-
fixCommand: fixCommand ?? void 0,
|
|
15708
|
-
cwd: invocation.resources.cwd,
|
|
15709
|
-
out: options.out,
|
|
15710
|
-
caseId: invocation.resources.trajId,
|
|
15711
|
-
stepTimeoutMs: options.stepTimeoutMs ?? invocation.resources.recordedStepTimeoutMs ?? void 0,
|
|
15712
|
-
prefixLimit: options.prefixLimit,
|
|
15713
|
-
backend: options.backend,
|
|
15714
|
-
apiKey: options.apiKey,
|
|
15715
|
-
baseUrl: options.baseUrl,
|
|
15716
|
-
onProgress: options.onProgress
|
|
15717
|
-
});
|
|
15718
|
-
return { invocation, fixCommand, verdict };
|
|
15719
|
-
}
|
|
15720
|
-
|
|
15721
|
-
// src/analyze-verify.ts
|
|
15722
|
-
import { mkdirSync as mkdirSync3, readFileSync as readFileSync5, writeFileSync as writeFileSync3 } from "fs";
|
|
15723
|
-
import { join as join27 } from "path";
|
|
15724
|
-
var SINGLE_STEP_SUBJECT = /^incorrect-step-(\d+)$/;
|
|
15725
|
-
function findingReplayStep(finding2) {
|
|
15726
|
-
const fromMetadata = finding2.metadata?.block_first_step;
|
|
15727
|
-
if (typeof fromMetadata === "number" && Number.isInteger(fromMetadata) && fromMetadata >= 1) {
|
|
15728
|
-
return fromMetadata;
|
|
15729
|
-
}
|
|
15730
|
-
const subject = finding2.subject ?? "";
|
|
15731
|
-
const wire = parseIncorrectStepsSubject(subject);
|
|
15732
|
-
if (wire) return wire.firstStep;
|
|
15733
|
-
const single = SINGLE_STEP_SUBJECT.exec(subject);
|
|
15734
|
-
if (single) return Number(single[1]);
|
|
15735
|
-
return null;
|
|
15736
|
-
}
|
|
15737
|
-
var TRACE_EVIDENCE_URI = /^trace:\/\/([^/]+)\//;
|
|
15738
|
-
function findingTrajectoryId(finding2) {
|
|
15739
|
-
for (const ref of finding2.evidence_refs ?? []) {
|
|
15740
|
-
if (typeof ref.uri !== "string") continue;
|
|
15741
|
-
const match = TRACE_EVIDENCE_URI.exec(ref.uri);
|
|
15742
|
-
if (match) return match[1];
|
|
15743
|
-
}
|
|
15744
|
-
return null;
|
|
15745
|
-
}
|
|
15746
|
-
function checkStep(steps, at) {
|
|
15747
|
-
const step = steps.find((s) => s.step_id === at);
|
|
15748
|
-
if (!step) {
|
|
15749
|
-
return { ok: false, reason: `step ${at} is outside the trajectory (${steps.length} steps)` };
|
|
15750
|
-
}
|
|
15751
|
-
if (isSubmitAction(step.action)) {
|
|
15752
|
-
return {
|
|
15753
|
-
ok: false,
|
|
15754
|
-
reason: `step ${at} is the submit action \u2014 a submit decision has no executable failure to replay`
|
|
15755
|
-
};
|
|
15756
|
-
}
|
|
15757
|
-
const recordedReturncode = parseRecordedReturncode(step.observation);
|
|
15758
|
-
if (recordedReturncode === null) {
|
|
15759
|
-
return {
|
|
15760
|
-
ok: false,
|
|
15761
|
-
reason: `step ${at} recorded no returncode \u2014 there is no executable failure signature to reproduce`
|
|
15762
|
-
};
|
|
15763
|
-
}
|
|
15764
|
-
return { ok: true, recordedReturncode };
|
|
15765
|
-
}
|
|
15766
|
-
function resolveFindingReplayability(finding2, source) {
|
|
15767
|
-
const at = findingReplayStep(finding2);
|
|
15768
|
-
if (at === null) {
|
|
15769
|
-
return {
|
|
15770
|
-
replayable: false,
|
|
15771
|
-
reason: `subject '${finding2.subject ?? "(none)"}' names no trajectory step (expected incorrect-step-<n>, incorrect-steps-<f>-<l>-\u2026, or metadata.block_first_step)`
|
|
15772
|
-
};
|
|
15773
|
-
}
|
|
15774
|
-
if (source.kind === "direct") {
|
|
15775
|
-
const trajId2 = findingTrajectoryId(finding2);
|
|
15776
|
-
if (trajId2 && source.caseId && trajId2 !== source.caseId) {
|
|
15777
|
-
return {
|
|
15778
|
-
replayable: false,
|
|
15779
|
-
reason: `finding cites trajectory '${trajId2}' but the supplied steps are case '${source.caseId}'`
|
|
15780
|
-
};
|
|
15781
|
-
}
|
|
15782
|
-
let steps;
|
|
15783
|
-
try {
|
|
15784
|
-
steps = JSON.parse(readFileSync5(source.stepsPath, "utf8"));
|
|
15785
|
-
} catch (err) {
|
|
15786
|
-
throw new Error(
|
|
15787
|
-
`verify-findings: cannot read steps file ${source.stepsPath} \u2014 ${err instanceof Error ? err.message : String(err)}`
|
|
15788
|
-
);
|
|
15789
|
-
}
|
|
15790
|
-
if (!Array.isArray(steps) || steps.length === 0) {
|
|
15791
|
-
throw new Error(`verify-findings: ${source.stepsPath} is not a non-empty steps array`);
|
|
15792
|
-
}
|
|
15793
|
-
const step = checkStep(steps, at);
|
|
15794
|
-
if (!step.ok) return { replayable: false, reason: step.reason };
|
|
15795
|
-
return {
|
|
15796
|
-
replayable: true,
|
|
15797
|
-
resolved: {
|
|
15798
|
-
caseId: source.caseId ?? trajId2 ?? source.stepsPath,
|
|
15799
|
-
stepsPath: source.stepsPath,
|
|
15800
|
-
image: source.image,
|
|
15801
|
-
cwd: source.cwd,
|
|
15802
|
-
at,
|
|
15803
|
-
recordedReturncode: step.recordedReturncode,
|
|
15804
|
-
recordedStepTimeoutMs: null
|
|
15805
|
-
}
|
|
15806
|
-
};
|
|
15807
|
-
}
|
|
15808
|
-
const trajId = findingTrajectoryId(finding2);
|
|
15809
|
-
if (!trajId) {
|
|
15810
|
-
return {
|
|
15811
|
-
replayable: false,
|
|
15812
|
-
reason: "finding carries no trace://<trajectory>/ evidence ref naming its trajectory"
|
|
15813
|
-
};
|
|
15814
|
-
}
|
|
15815
|
-
const failures = [];
|
|
15816
|
-
for (const corpus of source.corpora) {
|
|
15817
|
-
const resolution = resolveCaseResources(corpus, trajId);
|
|
15818
|
-
if (!resolution.resolved) {
|
|
15819
|
-
failures.push(`${corpus.name}: ${resolution.reason}${resolution.detail ? ` (${resolution.detail})` : ""}`);
|
|
15820
|
-
continue;
|
|
15821
|
-
}
|
|
15822
|
-
const step = checkStep(resolution.resources.steps, at);
|
|
15823
|
-
if (!step.ok) return { replayable: false, reason: step.reason };
|
|
15824
|
-
return {
|
|
15825
|
-
replayable: true,
|
|
15826
|
-
resolved: {
|
|
15827
|
-
caseId: trajId,
|
|
15828
|
-
stepsPath: resolution.resources.stepsPath,
|
|
15829
|
-
image: resolution.resources.image,
|
|
15830
|
-
cwd: resolution.resources.cwd,
|
|
15831
|
-
at,
|
|
15832
|
-
recordedReturncode: step.recordedReturncode,
|
|
15833
|
-
recordedStepTimeoutMs: resolution.resources.recordedStepTimeoutMs
|
|
15834
|
-
}
|
|
15835
|
-
};
|
|
15836
|
-
}
|
|
15837
|
-
return {
|
|
15838
|
-
replayable: false,
|
|
15839
|
-
reason: `trajectory ${trajId} is not replayable in any corpus \u2014 ${failures.join("; ")}`
|
|
15840
|
-
};
|
|
15841
|
-
}
|
|
15842
|
-
function classifyVerdict(verdict) {
|
|
15843
|
-
if (!verdict.armA.failureSignatureMatch) return "divergent";
|
|
15844
|
-
if (verdict.armB?.failureVanished) return "fix-flipped";
|
|
15845
|
-
return "reproduced";
|
|
15846
|
-
}
|
|
15847
|
-
var DEFAULT_SANDBOX_BASE_URL = "http://127.0.0.1:4097";
|
|
15848
|
-
async function assertSandboxReachable(baseUrl) {
|
|
15849
|
-
let response;
|
|
15850
|
-
try {
|
|
15851
|
-
response = await fetch(new URL("/health", baseUrl), { signal: AbortSignal.timeout(5e3) });
|
|
15852
|
-
} catch (err) {
|
|
15853
|
-
throw new Error(
|
|
15854
|
-
`verify-findings: sandbox API unreachable at ${baseUrl} \u2014 ${err instanceof Error ? err.message : String(err)}. Executing findings requires a running sandbox orchestrator + SDK adapter (see docs/replay-verify.md); start them or pass --base-url.`
|
|
15855
|
-
);
|
|
15856
|
-
}
|
|
15857
|
-
if (!response.ok) {
|
|
15858
|
-
throw new Error(
|
|
15859
|
-
`verify-findings: sandbox API at ${baseUrl} answered /health with HTTP ${response.status} \u2014 refusing to run proofs against degraded infrastructure`
|
|
15860
|
-
);
|
|
15861
|
-
}
|
|
15862
|
-
}
|
|
15863
|
-
function receiptExecution(verdict) {
|
|
15864
|
-
return {
|
|
15865
|
-
image: verdict.image,
|
|
15866
|
-
cwd: verdict.cwd,
|
|
15867
|
-
recordedReturncode: verdict.recordedReturncode,
|
|
15868
|
-
signature: verdict.signature,
|
|
15869
|
-
signatureBasis: verdict.signatureBasis,
|
|
15870
|
-
armA: {
|
|
15871
|
-
command: verdict.armA.command,
|
|
15872
|
-
exitCode: verdict.armA.exitCode,
|
|
15873
|
-
wallMs: verdict.armA.wallMs,
|
|
15874
|
-
failureSignatureMatch: verdict.armA.failureSignatureMatch
|
|
15875
|
-
},
|
|
15876
|
-
armB: verdict.armB ? {
|
|
15877
|
-
command: verdict.armB.command,
|
|
15878
|
-
exitCode: verdict.armB.exitCode,
|
|
15879
|
-
wallMs: verdict.armB.wallMs,
|
|
15880
|
-
failureVanished: verdict.armB.failureVanished
|
|
15881
|
-
} : null,
|
|
15882
|
-
prefixExecuted: verdict.prefixExecuted,
|
|
15883
|
-
prefixDivergences: verdict.prefixDivergences.length,
|
|
15884
|
-
totalMs: verdict.timings.totalMs
|
|
15885
|
-
};
|
|
15886
|
-
}
|
|
15887
|
-
function writeReceipt(receiptDir, finding2, verification, execution) {
|
|
15888
|
-
const receipt = {
|
|
15889
|
-
schema_version: "1.0.0",
|
|
15890
|
-
produced_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
15891
|
-
finding_id: verification.finding_id,
|
|
15892
|
-
analyst_id: finding2.analyst_id ?? null,
|
|
15893
|
-
subject: verification.subject,
|
|
15894
|
-
claim: typeof finding2.claim === "string" ? finding2.claim.slice(0, 600) : null,
|
|
15895
|
-
trajectory_id: verification.trajectory_id,
|
|
15896
|
-
step: verification.step,
|
|
15897
|
-
verified: verification.verified,
|
|
15898
|
-
reason: verification.reason,
|
|
15899
|
-
execution,
|
|
15900
|
-
verdict_path: verification.verdict_path,
|
|
15901
|
-
deduplicated_with: verification.deduplicated_with
|
|
15902
|
-
};
|
|
15903
|
-
writeFileSync3(join27(receiptDir, "receipt.json"), `${JSON.stringify(receipt, null, 2)}
|
|
15904
|
-
`);
|
|
15905
|
-
}
|
|
15906
|
-
function receiptDirName(index, finding2) {
|
|
15907
|
-
const id = typeof finding2.finding_id === "string" && finding2.finding_id.length > 0 ? finding2.finding_id.replace(/[^A-Za-z0-9_-]/g, "_") : "finding";
|
|
15908
|
-
return `${String(index + 1).padStart(3, "0")}-${id}`;
|
|
15909
|
-
}
|
|
15910
|
-
async function verifyFindings(findings, options) {
|
|
15911
|
-
if (findings.length === 0) throw new Error("verify-findings: no findings to verify");
|
|
15912
|
-
mkdirSync3(options.out, { recursive: true });
|
|
15913
|
-
const resolutions = findings.map((finding2) => resolveFindingReplayability(finding2, options.source));
|
|
15914
|
-
const baseUrl = options.baseUrl ?? DEFAULT_SANDBOX_BASE_URL;
|
|
15915
|
-
if (resolutions.some((resolution) => resolution.replayable) && !options.backend) {
|
|
15916
|
-
if (!options.apiKey) {
|
|
15917
|
-
throw new Error(
|
|
15918
|
-
"verify-findings: replayable findings need a sandbox API key \u2014 export SANDBOX_API_KEY or pass --api-key-env"
|
|
15919
|
-
);
|
|
15920
|
-
}
|
|
15921
|
-
await assertSandboxReachable(baseUrl);
|
|
15922
|
-
}
|
|
15923
|
-
const preparer = options.source.kind === "corpus" && !options.backend ? options.source.preparer ?? dockerImagePreparer() : null;
|
|
15924
|
-
const preparedImages = /* @__PURE__ */ new Map();
|
|
15925
|
-
const executedByKey = /* @__PURE__ */ new Map();
|
|
15926
|
-
const verifications = [];
|
|
15927
|
-
let executions = 0;
|
|
15928
|
-
for (let index = 0; index < findings.length; index++) {
|
|
15929
|
-
const finding2 = findings[index];
|
|
15930
|
-
const resolution = resolutions[index];
|
|
15931
|
-
const receiptDir = join27(options.out, receiptDirName(index, finding2));
|
|
15932
|
-
mkdirSync3(receiptDir, { recursive: true });
|
|
15933
|
-
const identity2 = {
|
|
15934
|
-
finding_id: finding2.finding_id ?? null,
|
|
15935
|
-
subject: finding2.subject ?? null,
|
|
15936
|
-
trajectory_id: findingTrajectoryId(finding2)
|
|
15937
|
-
};
|
|
15938
|
-
if (!resolution.replayable) {
|
|
15939
|
-
const verification2 = {
|
|
15940
|
-
...identity2,
|
|
15941
|
-
step: findingReplayStep(finding2),
|
|
15942
|
-
verified: "not-replayable",
|
|
15943
|
-
reason: resolution.reason,
|
|
15944
|
-
receipt: receiptDir,
|
|
15945
|
-
verdict_path: null,
|
|
15946
|
-
deduplicated_with: null
|
|
15947
|
-
};
|
|
15948
|
-
writeReceipt(receiptDir, finding2, verification2, null);
|
|
15949
|
-
verifications.push(verification2);
|
|
15950
|
-
options.onProgress?.(`${identity2.finding_id ?? `finding ${index + 1}`}: not-replayable \u2014 ${resolution.reason}`);
|
|
15951
|
-
continue;
|
|
15952
|
-
}
|
|
15953
|
-
const resolved = resolution.resolved;
|
|
15954
|
-
const dedupeKey = `${resolved.caseId}::${resolved.at}::${options.fixCommand ?? ""}`;
|
|
15955
|
-
const prior = executedByKey.get(dedupeKey);
|
|
15956
|
-
if (prior) {
|
|
15957
|
-
const verification2 = {
|
|
15958
|
-
...identity2,
|
|
15959
|
-
step: resolved.at,
|
|
15960
|
-
verified: prior.verified,
|
|
15961
|
-
reason: prior.reason,
|
|
15962
|
-
receipt: receiptDir,
|
|
15963
|
-
verdict_path: prior.verdict_path,
|
|
15964
|
-
deduplicated_with: prior.receipt
|
|
15965
|
-
};
|
|
15966
|
-
const priorVerdict = prior.verdict_path ? JSON.parse(readFileSync5(prior.verdict_path, "utf8")) : null;
|
|
15967
|
-
writeReceipt(receiptDir, finding2, verification2, priorVerdict ? receiptExecution(priorVerdict) : null);
|
|
15968
|
-
verifications.push(verification2);
|
|
15969
|
-
options.onProgress?.(
|
|
15970
|
-
`${identity2.finding_id ?? `finding ${index + 1}`}: ${prior.verified} (shares proof with ${prior.finding_id ?? prior.receipt})`
|
|
15971
|
-
);
|
|
15972
|
-
continue;
|
|
15973
|
-
}
|
|
15974
|
-
let image = resolved.image;
|
|
15975
|
-
if (preparer) {
|
|
15976
|
-
const preparationKey = `${resolved.image}::${resolved.cwd}`;
|
|
15977
|
-
let preparation = preparedImages.get(preparationKey);
|
|
15978
|
-
if (!preparation) {
|
|
15979
|
-
const ensured = await preparer.ensure(resolved.image, resolved.cwd);
|
|
15980
|
-
preparation = ensured.succeeded ? { succeeded: true, image: ensured.value.derivedImage } : { succeeded: false, error: ensured.error };
|
|
15981
|
-
preparedImages.set(preparationKey, preparation);
|
|
15982
|
-
}
|
|
15983
|
-
if (!preparation.succeeded) {
|
|
15984
|
-
const verification2 = {
|
|
15985
|
-
...identity2,
|
|
15986
|
-
step: resolved.at,
|
|
15987
|
-
verified: "not-replayable",
|
|
15988
|
-
reason: `replay image could not be prepared \u2014 ${preparation.error}`,
|
|
15989
|
-
receipt: receiptDir,
|
|
15990
|
-
verdict_path: null,
|
|
15991
|
-
deduplicated_with: null
|
|
15992
|
-
};
|
|
15993
|
-
writeReceipt(receiptDir, finding2, verification2, null);
|
|
15994
|
-
verifications.push(verification2);
|
|
15995
|
-
options.onProgress?.(`${identity2.finding_id ?? `finding ${index + 1}`}: not-replayable \u2014 image preparation failed`);
|
|
15996
|
-
continue;
|
|
15997
|
-
}
|
|
15998
|
-
image = preparation.image;
|
|
15999
|
-
}
|
|
16000
|
-
options.onProgress?.(
|
|
16001
|
-
`${identity2.finding_id ?? `finding ${index + 1}`}: executing arm A at step ${resolved.at} of ${resolved.caseId} on ${image}`
|
|
16002
|
-
);
|
|
16003
|
-
const verdict = await replayVerify({
|
|
16004
|
-
stepsPath: resolved.stepsPath,
|
|
16005
|
-
image,
|
|
16006
|
-
at: resolved.at,
|
|
16007
|
-
fixCommand: options.fixCommand,
|
|
16008
|
-
cwd: resolved.cwd,
|
|
16009
|
-
out: receiptDir,
|
|
16010
|
-
caseId: resolved.caseId,
|
|
16011
|
-
stepTimeoutMs: options.stepTimeoutMs ?? resolved.recordedStepTimeoutMs ?? void 0,
|
|
16012
|
-
prefixLimit: options.prefixLimit,
|
|
16013
|
-
backend: options.backend,
|
|
16014
|
-
apiKey: options.apiKey,
|
|
16015
|
-
baseUrl,
|
|
16016
|
-
maxLifetimeSeconds: options.maxLifetimeSeconds,
|
|
16017
|
-
onProgress: options.onProgress
|
|
16018
|
-
});
|
|
16019
|
-
executions += 1;
|
|
16020
|
-
const verification = {
|
|
16021
|
-
...identity2,
|
|
16022
|
-
step: resolved.at,
|
|
16023
|
-
verified: classifyVerdict(verdict),
|
|
16024
|
-
reason: null,
|
|
16025
|
-
receipt: receiptDir,
|
|
16026
|
-
verdict_path: join27(receiptDir, "replay-verdict.json"),
|
|
16027
|
-
deduplicated_with: null
|
|
16028
|
-
};
|
|
16029
|
-
executedByKey.set(dedupeKey, verification);
|
|
16030
|
-
writeReceipt(receiptDir, finding2, verification, receiptExecution(verdict));
|
|
16031
|
-
verifications.push(verification);
|
|
16032
|
-
options.onProgress?.(`${identity2.finding_id ?? `finding ${index + 1}`}: ${verification.verified}`);
|
|
16033
|
-
}
|
|
16034
|
-
const counts = {
|
|
16035
|
-
reproduced: 0,
|
|
16036
|
-
"fix-flipped": 0,
|
|
16037
|
-
divergent: 0,
|
|
16038
|
-
"not-replayable": 0
|
|
16039
|
-
};
|
|
16040
|
-
for (const verification of verifications) counts[verification.verified] += 1;
|
|
16041
|
-
const run = { out: options.out, verifications, counts, executions };
|
|
16042
|
-
writeFileSync3(
|
|
16043
|
-
join27(options.out, "verifications.json"),
|
|
16044
|
-
`${JSON.stringify({ schema_version: "1.0.0", ...run }, null, 2)}
|
|
16045
|
-
`
|
|
16046
|
-
);
|
|
16047
|
-
return run;
|
|
16048
|
-
}
|
|
16049
|
-
function verdictCell(verification) {
|
|
16050
|
-
switch (verification.verified) {
|
|
16051
|
-
case "reproduced":
|
|
16052
|
-
return "**VERIFIED** \u2014 reproduced";
|
|
16053
|
-
case "fix-flipped":
|
|
16054
|
-
return "**VERIFIED** \u2014 fix-flipped";
|
|
16055
|
-
case "divergent":
|
|
16056
|
-
return "DIVERGENT \u2014 recorded failure did not reproduce";
|
|
16057
|
-
case "not-replayable":
|
|
16058
|
-
return `UNVERIFIABLE \u2014 ${verification.reason ?? "no reason recorded"}`;
|
|
16059
|
-
}
|
|
16060
|
-
}
|
|
16061
|
-
function renderVerifiedFindingsSection(run) {
|
|
16062
|
-
const lines = ["## Verified findings (executed replay)", ""];
|
|
16063
|
-
const total = run.verifications.length;
|
|
16064
|
-
lines.push(
|
|
16065
|
-
`${total} finding(s) \u2192 ${run.counts.reproduced} reproduced, ${run.counts["fix-flipped"]} fix-flipped, ${run.counts.divergent} divergent, ${run.counts["not-replayable"]} not replayable (${run.executions} sandbox execution(s); findings accusing the same step share one proof).`
|
|
16066
|
-
);
|
|
16067
|
-
lines.push("");
|
|
16068
|
-
lines.push("| Finding | Subject | Step | Verdict | Receipt |");
|
|
16069
|
-
lines.push("|---|---|---:|---|---|");
|
|
16070
|
-
for (const verification of run.verifications) {
|
|
16071
|
-
const shared = verification.deduplicated_with ? " (shared proof)" : "";
|
|
16072
|
-
lines.push(
|
|
16073
|
-
`| \`${verification.finding_id ?? "\u2014"}\` | \`${verification.subject ?? "\u2014"}\` | ${verification.step ?? "\u2014"} | ${verdictCell(verification)} | \`${verification.receipt}\`${shared} |`
|
|
16074
|
-
);
|
|
16075
|
-
}
|
|
16076
|
-
lines.push("");
|
|
16077
|
-
lines.push(
|
|
16078
|
-
"VERIFIED = the accused step was re-executed in a sandbox after replaying the trajectory prefix, and the recorded failure signature reproduced (fix-flipped: a corrected command additionally made it vanish). Each receipt directory carries receipt.json and, when executed, replay-verdict.json + report.md with real stdout/stderr."
|
|
14178
|
+
progress(
|
|
14179
|
+
`replay-verify-batch: replayability ${replayabilityRate.numerator}/${replayabilityRate.denominator}` + (fixFlipRate ? `, fix-flip ${fixFlipRate.numerator}/${fixFlipRate.denominator}` : "") + ` \u2192 ${out}`
|
|
16079
14180
|
);
|
|
16080
|
-
lines.push("");
|
|
16081
|
-
return lines.join("\n");
|
|
16082
|
-
}
|
|
16083
|
-
function readFindingsFile(path) {
|
|
16084
|
-
const parsed = JSON.parse(readFileSync5(path, "utf8"));
|
|
16085
|
-
const array = Array.isArray(parsed) ? parsed : parsed && typeof parsed === "object" && Array.isArray(parsed.findings) ? parsed.findings : null;
|
|
16086
|
-
if (!array) {
|
|
16087
|
-
throw new Error(`${path} is neither a findings array nor an object with a findings array`);
|
|
16088
|
-
}
|
|
16089
|
-
for (const entry of array) {
|
|
16090
|
-
if (!entry || typeof entry !== "object") {
|
|
16091
|
-
throw new Error(`${path}: every finding must be an object, got ${JSON.stringify(entry)}`);
|
|
16092
|
-
}
|
|
16093
|
-
}
|
|
16094
|
-
return array;
|
|
16095
14181
|
}
|
|
16096
|
-
|
|
16097
|
-
return `traces verify-findings \u2014 execute analyst findings as sandbox replay proofs
|
|
14182
|
+
var VERIFY_FINDINGS_USAGE = `traces verify-findings \u2014 execute analyst findings as sandbox replay proofs
|
|
16098
14183
|
|
|
16099
14184
|
Usage:
|
|
16100
14185
|
traces verify-findings --findings FINDINGS.json --out DIR \\
|
|
@@ -16111,7 +14196,7 @@ Usage:
|
|
|
16111
14196
|
verify against one trajectory; the image must be replay-ready
|
|
16112
14197
|
(uid-1000 derived), exactly like traces replay-verify
|
|
16113
14198
|
--case trajectory id of --steps; findings citing other trajectories
|
|
16114
|
-
are
|
|
14199
|
+
are marked not-replayable
|
|
16115
14200
|
--corpus resolve each finding's trajectory in CodeTraceBench corpora;
|
|
16116
14201
|
images are derived via the batch uid-1000 preparer (docker)
|
|
16117
14202
|
--fix-command corrected step for arm B; a reproduced finding whose fix
|
|
@@ -16124,110 +14209,83 @@ with a receipt directory carrying the executed evidence or the precise reason.
|
|
|
16124
14209
|
Requires a running sandbox orchestrator when any finding is replayable \u2014 infra
|
|
16125
14210
|
absence is an error, never a silent skip. See docs/replay-verify.md for setup.
|
|
16126
14211
|
`;
|
|
14212
|
+
function verifyFindingsInSandbox(findings, options) {
|
|
14213
|
+
if (findings.length === 0) throw new Error("verify-findings: no findings to verify");
|
|
14214
|
+
const { sandbox, ...rest } = options;
|
|
14215
|
+
return verifyFindings(findings, {
|
|
14216
|
+
...rest,
|
|
14217
|
+
backendFactory: sandboxBackendFactory(sandbox),
|
|
14218
|
+
preflight: async () => {
|
|
14219
|
+
if (!sandbox.apiKey) {
|
|
14220
|
+
throw new Error(
|
|
14221
|
+
"verify-findings: replayable findings need a sandbox API key \u2014 export SANDBOX_API_KEY or pass --api-key-env"
|
|
14222
|
+
);
|
|
14223
|
+
}
|
|
14224
|
+
await assertSandboxReachable(sandbox.baseUrl);
|
|
14225
|
+
},
|
|
14226
|
+
onProgress: progress
|
|
14227
|
+
});
|
|
16127
14228
|
}
|
|
16128
|
-
function
|
|
16129
|
-
const
|
|
16130
|
-
|
|
16131
|
-
|
|
16132
|
-
|
|
16133
|
-
if (arg === "--help" || arg === "-h") return "help";
|
|
16134
|
-
if (!arg.startsWith("--")) throw new Error(`unexpected positional argument: ${arg}`);
|
|
16135
|
-
const value = argv[++i];
|
|
16136
|
-
if (value === void 0) throw new Error(`missing value for ${arg}`);
|
|
16137
|
-
if (arg === "--corpus") {
|
|
16138
|
-
corpora.push(parseCorpusFlag(value));
|
|
16139
|
-
continue;
|
|
16140
|
-
}
|
|
16141
|
-
values.set(arg, value);
|
|
14229
|
+
async function cmdVerifyFindings(argv) {
|
|
14230
|
+
const parsed = parseFlags2("verify-findings", argv);
|
|
14231
|
+
if (parsed === "help") {
|
|
14232
|
+
process.stdout.write(VERIFY_FINDINGS_USAGE);
|
|
14233
|
+
return;
|
|
16142
14234
|
}
|
|
16143
|
-
const
|
|
16144
|
-
|
|
16145
|
-
|
|
16146
|
-
|
|
16147
|
-
};
|
|
16148
|
-
const optionalNumber = (flag) => {
|
|
16149
|
-
const v = values.get(flag);
|
|
16150
|
-
if (v === void 0) return void 0;
|
|
16151
|
-
const n = Number(v);
|
|
16152
|
-
if (!Number.isFinite(n)) throw new Error(`verify-findings: ${flag} must be a number, got ${v}`);
|
|
16153
|
-
return n;
|
|
16154
|
-
};
|
|
16155
|
-
const findingsPath = required("--findings");
|
|
16156
|
-
const out = required("--out");
|
|
14235
|
+
const { values, corpora } = parsed;
|
|
14236
|
+
const flags = flagReader("verify-findings", values);
|
|
14237
|
+
const findingsPath = flags.required("--findings");
|
|
14238
|
+
const out = flags.required("--out");
|
|
16157
14239
|
const direct = values.has("--steps") || values.has("--image") || values.has("--cwd");
|
|
16158
14240
|
if (direct && corpora.length > 0) {
|
|
16159
14241
|
throw new Error("verify-findings: pass --steps/--image/--cwd or --corpus, not both");
|
|
16160
14242
|
}
|
|
16161
14243
|
let source;
|
|
16162
14244
|
if (direct) {
|
|
14245
|
+
const caseId = values.get("--case");
|
|
16163
14246
|
source = {
|
|
16164
14247
|
kind: "direct",
|
|
16165
|
-
stepsPath: required("--steps"),
|
|
16166
|
-
image: required("--image"),
|
|
16167
|
-
cwd: required("--cwd"),
|
|
16168
|
-
...
|
|
14248
|
+
stepsPath: flags.required("--steps"),
|
|
14249
|
+
image: flags.required("--image"),
|
|
14250
|
+
cwd: flags.required("--cwd"),
|
|
14251
|
+
...caseId === void 0 ? {} : { caseId }
|
|
16169
14252
|
};
|
|
16170
14253
|
} else if (corpora.length > 0) {
|
|
16171
14254
|
source = { kind: "corpus", corpora };
|
|
16172
14255
|
} else {
|
|
16173
14256
|
throw new Error("verify-findings: a replay source is required \u2014 --steps/--image/--cwd or --corpus");
|
|
16174
14257
|
}
|
|
16175
|
-
|
|
16176
|
-
findingsPath,
|
|
14258
|
+
const run = await verifyFindingsInSandbox(readFindingsFile(findingsPath), {
|
|
16177
14259
|
source,
|
|
16178
14260
|
out,
|
|
14261
|
+
sandbox: sandboxOptions("verify-findings", values, false),
|
|
16179
14262
|
fixCommand: values.get("--fix-command"),
|
|
16180
|
-
stepTimeoutMs:
|
|
16181
|
-
prefixLimit:
|
|
16182
|
-
baseUrl: values.get("--base-url") ?? process.env.SANDBOX_API_URL ?? DEFAULT_SANDBOX_BASE_URL,
|
|
16183
|
-
apiKeyEnv: values.get("--api-key-env") ?? "SANDBOX_API_KEY",
|
|
16184
|
-
maxLifetimeSeconds: optionalNumber("--max-lifetime")
|
|
16185
|
-
};
|
|
16186
|
-
}
|
|
16187
|
-
async function cmdVerifyFindings(argv) {
|
|
16188
|
-
const parsed = parseVerifyFindingsArgs(argv);
|
|
16189
|
-
if (parsed === "help") {
|
|
16190
|
-
process.stdout.write(verifyFindingsUsage());
|
|
16191
|
-
return;
|
|
16192
|
-
}
|
|
16193
|
-
const findings = readFindingsFile(parsed.findingsPath);
|
|
16194
|
-
const apiKey = process.env[parsed.apiKeyEnv];
|
|
16195
|
-
const run = await verifyFindings(findings, {
|
|
16196
|
-
source: parsed.source,
|
|
16197
|
-
out: parsed.out,
|
|
16198
|
-
fixCommand: parsed.fixCommand,
|
|
16199
|
-
stepTimeoutMs: parsed.stepTimeoutMs,
|
|
16200
|
-
prefixLimit: parsed.prefixLimit,
|
|
16201
|
-
apiKey,
|
|
16202
|
-
baseUrl: parsed.baseUrl,
|
|
16203
|
-
maxLifetimeSeconds: parsed.maxLifetimeSeconds,
|
|
16204
|
-
onProgress: (message) => process.stderr.write(`${message}
|
|
16205
|
-
`)
|
|
14263
|
+
stepTimeoutMs: flags.number("--step-timeout"),
|
|
14264
|
+
prefixLimit: flags.number("--prefix-limit")
|
|
16206
14265
|
});
|
|
16207
14266
|
process.stdout.write(`${JSON.stringify(run, null, 2)}
|
|
16208
14267
|
`);
|
|
16209
|
-
|
|
16210
|
-
`verify-findings: ${run.verifications.length} finding(s) \u2192 ${run.counts.reproduced} reproduced, ${run.counts["fix-flipped"]} fix-flipped, ${run.counts.divergent} divergent, ${run.counts["not-replayable"]} not-replayable (${run.executions} execution(s)) \u2192 ${run.out}
|
|
16211
|
-
`
|
|
14268
|
+
progress(
|
|
14269
|
+
`verify-findings: ${run.verifications.length} finding(s) \u2192 ${run.counts.reproduced} reproduced, ${run.counts["fix-flipped"]} fix-flipped, ${run.counts.divergent} divergent, ${run.counts["not-replayable"]} not-replayable (${run.executions} execution(s)) \u2192 ${run.out}`
|
|
16212
14270
|
);
|
|
16213
14271
|
}
|
|
16214
14272
|
|
|
16215
14273
|
// src/hodoscope.ts
|
|
16216
|
-
import { createHash as
|
|
14274
|
+
import { createHash as createHash12 } from "crypto";
|
|
16217
14275
|
import { mkdir as mkdir5, mkdtemp as mkdtemp5, readFile as readFile11, readdir as readdir17, writeFile as writeFile7 } from "fs/promises";
|
|
16218
|
-
import { endianness, tmpdir as
|
|
16219
|
-
import { join as
|
|
14276
|
+
import { endianness, tmpdir as tmpdir5 } from "os";
|
|
14277
|
+
import { join as join24, resolve as resolve10 } from "path";
|
|
16220
14278
|
var HODOSCOPE_VERSION = "0.2.4";
|
|
16221
14279
|
async function writeHodoscopeInput(otlpPath, options = {}) {
|
|
16222
14280
|
const explicitDirectory = options.directory ? resolve10(options.directory) : void 0;
|
|
16223
|
-
const directory = explicitDirectory ?? await mkdtemp5(
|
|
14281
|
+
const directory = explicitDirectory ?? await mkdtemp5(join24(tmpdir5(), "traces-hodoscope-input-"));
|
|
16224
14282
|
if (explicitDirectory) {
|
|
16225
14283
|
await mkdir5(directory, { recursive: true });
|
|
16226
14284
|
if ((await readdir17(directory)).length > 0) {
|
|
16227
14285
|
throw new Error(`Hodoscope explicit directory must be empty: ${directory}`);
|
|
16228
14286
|
}
|
|
16229
14287
|
}
|
|
16230
|
-
const samples =
|
|
14288
|
+
const samples = join24(directory, "samples");
|
|
16231
14289
|
await mkdir5(samples);
|
|
16232
14290
|
const rows = [];
|
|
16233
14291
|
for await (const value of readJsonl(otlpPath)) rows.push(parseOpenInferenceRow(value));
|
|
@@ -16252,8 +14310,8 @@ async function writeHodoscopeInput(otlpPath, options = {}) {
|
|
|
16252
14310
|
throw new Error(`Hodoscope trace '${traceId}' contains no assistant actions`);
|
|
16253
14311
|
}
|
|
16254
14312
|
actionCount += actions;
|
|
16255
|
-
const file = `${
|
|
16256
|
-
await writeFile7(
|
|
14313
|
+
const file = `${createHash12("sha256").update(traceId).digest("hex").slice(0, 24)}.json`;
|
|
14314
|
+
await writeFile7(join24(samples, file), `${JSON.stringify(trajectory, null, 2)}
|
|
16257
14315
|
`, "utf8");
|
|
16258
14316
|
}
|
|
16259
14317
|
return {
|
|
@@ -16275,12 +14333,12 @@ function hodoscopeAnalyzer(options = {}) {
|
|
|
16275
14333
|
return {
|
|
16276
14334
|
name: "hodoscope",
|
|
16277
14335
|
async analyze(otlpPath, runOptions2 = {}) {
|
|
16278
|
-
const outputRoot = options.outputDir ? resolve10(options.outputDir) :
|
|
14336
|
+
const outputRoot = options.outputDir ? resolve10(options.outputDir) : tmpdir5();
|
|
16279
14337
|
await mkdir5(outputRoot, { recursive: true });
|
|
16280
|
-
const directory = await mkdtemp5(
|
|
16281
|
-
const input =
|
|
16282
|
-
const analysisFile =
|
|
16283
|
-
const samplesFile =
|
|
14338
|
+
const directory = await mkdtemp5(join24(outputRoot, "traces-hodoscope-"));
|
|
14339
|
+
const input = join24(directory, "input");
|
|
14340
|
+
const analysisFile = join24(directory, "analysis.hodoscope.json");
|
|
14341
|
+
const samplesFile = join24(directory, "samples.json");
|
|
16284
14342
|
try {
|
|
16285
14343
|
const prepared = await writeHodoscopeInput(otlpPath, {
|
|
16286
14344
|
directory: input,
|
|
@@ -16900,7 +14958,7 @@ function assertPositiveInteger(name, value) {
|
|
|
16900
14958
|
}
|
|
16901
14959
|
|
|
16902
14960
|
// src/analyst-engine-prime.ts
|
|
16903
|
-
import { createHash as
|
|
14961
|
+
import { createHash as createHash13 } from "crypto";
|
|
16904
14962
|
import { request as httpRequest } from "http";
|
|
16905
14963
|
import { request as httpsRequest } from "https";
|
|
16906
14964
|
var DEFAULT_BRIDGE_URL = "http://localhost:4181";
|
|
@@ -17124,7 +15182,7 @@ function toAnalystFinding(row, index, spanTraces, model, producedAt) {
|
|
|
17124
15182
|
const traces = spanTraces.get(spanId);
|
|
17125
15183
|
return spanEvidenceUri([...traces][0], spanId);
|
|
17126
15184
|
});
|
|
17127
|
-
const findingId = `prime-${
|
|
15185
|
+
const findingId = `prime-${createHash13("sha256").update(`${index}
|
|
17128
15186
|
${row.claim}
|
|
17129
15187
|
${uris.join("\n")}`).digest("hex").slice(0, 12)}`;
|
|
17130
15188
|
return {
|
|
@@ -17408,10 +15466,10 @@ async function applyRedactor(spans, redactor) {
|
|
|
17408
15466
|
}
|
|
17409
15467
|
|
|
17410
15468
|
// src/file-export.ts
|
|
17411
|
-
import { createHash as
|
|
15469
|
+
import { createHash as createHash14 } from "crypto";
|
|
17412
15470
|
import { mkdtemp as mkdtemp6, readFile as readFile12, writeFile as writeFile8 } from "fs/promises";
|
|
17413
|
-
import { tmpdir as
|
|
17414
|
-
import { join as
|
|
15471
|
+
import { tmpdir as tmpdir6 } from "os";
|
|
15472
|
+
import { join as join25 } from "path";
|
|
17415
15473
|
import {
|
|
17416
15474
|
firstNumberAttr as firstNumberAttr2,
|
|
17417
15475
|
LLM_CACHED_TOKEN_ATTR_KEYS,
|
|
@@ -17446,7 +15504,7 @@ function stableJson(value) {
|
|
|
17446
15504
|
});
|
|
17447
15505
|
}
|
|
17448
15506
|
function hashId(value, chars = 16) {
|
|
17449
|
-
return
|
|
15507
|
+
return createHash14("sha256").update(stableJson(value)).digest("hex").slice(0, chars);
|
|
17450
15508
|
}
|
|
17451
15509
|
function isoTime2(value) {
|
|
17452
15510
|
if (typeof value === "string") {
|
|
@@ -17757,6 +15815,10 @@ function llmEventUsage(row) {
|
|
|
17757
15815
|
...costUsd !== void 0 ? { costUsd } : {}
|
|
17758
15816
|
};
|
|
17759
15817
|
}
|
|
15818
|
+
function toolPartId(row) {
|
|
15819
|
+
const part = objectValue2(objectValue2(row.data)?.part) ?? objectValue2(row.part);
|
|
15820
|
+
return part && part.type === "tool" ? stringValue3(part.id) : void 0;
|
|
15821
|
+
}
|
|
17760
15822
|
function sandboxEventsToSpans(rows, wrapper) {
|
|
17761
15823
|
const context = wrapper ?? {};
|
|
17762
15824
|
const sessionId = findStringKey(context, ["session_id", "sessionId"], 3) ?? findStringKey(rows, ["session_id", "sessionId"], 4);
|
|
@@ -17788,12 +15850,32 @@ function sandboxEventsToSpans(rows, wrapper) {
|
|
|
17788
15850
|
}
|
|
17789
15851
|
})
|
|
17790
15852
|
];
|
|
15853
|
+
const lastEventOfPart = /* @__PURE__ */ new Map();
|
|
15854
|
+
const firstTimeOfPart = /* @__PURE__ */ new Map();
|
|
15855
|
+
const errorOfPart = /* @__PURE__ */ new Map();
|
|
15856
|
+
rows.forEach((row, index) => {
|
|
15857
|
+
const part = toolPartId(row);
|
|
15858
|
+
if (part === void 0) return;
|
|
15859
|
+
lastEventOfPart.set(part, index);
|
|
15860
|
+
const time = eventTimestamp(row);
|
|
15861
|
+
const first = firstTimeOfPart.get(part);
|
|
15862
|
+
if (time && (first === void 0 || time < first)) firstTimeOfPart.set(part, time);
|
|
15863
|
+
const status = eventStatus(row, eventType(row) ?? "event");
|
|
15864
|
+
if (status.status === "ERROR") errorOfPart.set(part, status);
|
|
15865
|
+
});
|
|
15866
|
+
const mergedToolEvents = rows.filter((row, index) => {
|
|
15867
|
+
const part = toolPartId(row);
|
|
15868
|
+
return part !== void 0 && lastEventOfPart.get(part) !== index;
|
|
15869
|
+
}).length;
|
|
15870
|
+
if (mergedToolEvents > 0) spans[0].attributes["traces.event.tool_part_updates_merged"] = mergedToolEvents;
|
|
17791
15871
|
let previousTime = rootStart;
|
|
17792
15872
|
rows.forEach((row, index) => {
|
|
15873
|
+
const part = toolPartId(row);
|
|
15874
|
+
if (part !== void 0 && lastEventOfPart.get(part) !== index) return;
|
|
17793
15875
|
const type = eventType(row) ?? "event";
|
|
17794
|
-
const time = eventTimestamp(row) ?? previousTime;
|
|
15876
|
+
const time = (part !== void 0 ? firstTimeOfPart.get(part) : void 0) ?? eventTimestamp(row) ?? previousTime;
|
|
17795
15877
|
previousTime = time;
|
|
17796
|
-
const { status, message } = eventStatus(row, type);
|
|
15878
|
+
const { status, message } = (part !== void 0 ? errorOfPart.get(part) : void 0) ?? eventStatus(row, type);
|
|
17797
15879
|
const { kind, tool } = eventKind(row, type);
|
|
17798
15880
|
const name = kind === "TOOL" && tool ? `tool.${tool}` : `event.${type}`;
|
|
17799
15881
|
const toolInput2 = kind === "TOOL" ? eventValue(row, ["toolInput", "tool_input", "input", "args", "arguments", "full_command"]) : void 0;
|
|
@@ -17808,6 +15890,7 @@ function sandboxEventsToSpans(rows, wrapper) {
|
|
|
17808
15890
|
}) : {}
|
|
17809
15891
|
};
|
|
17810
15892
|
copyDefined(extra, {
|
|
15893
|
+
"traces.event.part_id": part,
|
|
17811
15894
|
"traces.event.id": stringValue3(row.id),
|
|
17812
15895
|
"traces.event.raw_type": stringValue3(row.type),
|
|
17813
15896
|
"traces.event.raw_event": stringValue3(row.event)
|
|
@@ -18057,15 +16140,15 @@ async function exportTraceEvidenceFile(inputPath, opts = {}) {
|
|
|
18057
16140
|
}
|
|
18058
16141
|
async function writeTraceEvidenceExportFile(inputPath, outPath, opts = {}) {
|
|
18059
16142
|
const result = await exportTraceEvidenceFile(inputPath, opts);
|
|
18060
|
-
const path = outPath ??
|
|
16143
|
+
const path = outPath ?? join25(await mkdtemp6(join25(tmpdir6(), "traces-export-")), "spans.openinference.jsonl");
|
|
18061
16144
|
await writeFile8(path, serializeSpans(result.spans), "utf8");
|
|
18062
16145
|
return { ...result, path };
|
|
18063
16146
|
}
|
|
18064
16147
|
|
|
18065
16148
|
// src/inspect.ts
|
|
18066
16149
|
import { mkdtemp as mkdtemp7, readFile as readFile13, writeFile as writeFile9 } from "fs/promises";
|
|
18067
|
-
import { tmpdir as
|
|
18068
|
-
import { basename as basename15, join as
|
|
16150
|
+
import { tmpdir as tmpdir7 } from "os";
|
|
16151
|
+
import { basename as basename15, join as join26 } from "path";
|
|
18069
16152
|
var severityRank2 = {
|
|
18070
16153
|
high: 0,
|
|
18071
16154
|
medium: 1,
|
|
@@ -18081,7 +16164,7 @@ function sessionLabel(row) {
|
|
|
18081
16164
|
const id = row.session.sessionId.length > 12 ? row.session.sessionId.slice(0, 12) : row.session.sessionId;
|
|
18082
16165
|
return `${row.session.harness}:${id} (${repoLabel(row)})`;
|
|
18083
16166
|
}
|
|
18084
|
-
function
|
|
16167
|
+
function pct(value) {
|
|
18085
16168
|
return `${(value * 100).toFixed(value >= 0.1 ? 0 : 1)}%`;
|
|
18086
16169
|
}
|
|
18087
16170
|
function topToolErrors(row) {
|
|
@@ -18160,7 +16243,7 @@ function addSessionFindings(index, findings) {
|
|
|
18160
16243
|
area: "session",
|
|
18161
16244
|
title: `High tool-error count/rate in ${errorSessions.length}/${index.totals.sessions} session(s)`,
|
|
18162
16245
|
impact: errors,
|
|
18163
|
-
evidence: errorSessions.slice(0, 5).map((row) => `${sessionLabel(row)}: ${row.metrics.erroredToolCallCount}/${row.metrics.toolCallCount} tool errors (${
|
|
16246
|
+
evidence: errorSessions.slice(0, 5).map((row) => `${sessionLabel(row)}: ${row.metrics.erroredToolCallCount}/${row.metrics.toolCallCount} tool errors (${pct(row.metrics.toolErrorRate)}); ${topToolErrors(row)}`),
|
|
18164
16247
|
next: "Fix the most common failing command/tool first, then rerun `traces index` to confirm the error count drops.",
|
|
18165
16248
|
refs: errorSessions.slice(0, 5).map((row) => row.session.path)
|
|
18166
16249
|
});
|
|
@@ -18354,13 +16437,13 @@ async function readSessionIndexFile(path) {
|
|
|
18354
16437
|
return parsed;
|
|
18355
16438
|
}
|
|
18356
16439
|
async function writeInspectionReportFile(report, outPath) {
|
|
18357
|
-
const path = outPath ??
|
|
16440
|
+
const path = outPath ?? join26(await mkdtemp7(join26(tmpdir7(), "traces-inspect-")), "inspection-report.md");
|
|
18358
16441
|
await writeFile9(path, renderInspectionReport(report), "utf8");
|
|
18359
16442
|
return path;
|
|
18360
16443
|
}
|
|
18361
16444
|
|
|
18362
16445
|
// src/live.ts
|
|
18363
|
-
import { createHash as
|
|
16446
|
+
import { createHash as createHash15 } from "crypto";
|
|
18364
16447
|
var VERIFY_RE = /\b(test|tests|vitest|jest|pytest|go test|cargo test|tsc|typecheck|lint|biome|eslint|build|check:invariants|preflight)\b/i;
|
|
18365
16448
|
var CHANGE_RE = /\b(apply_patch|edit|write|create|delete|rename|replace|update|git apply|sed -i|perl -pi|tee\s|cat\s+>)/i;
|
|
18366
16449
|
var CLAIM_DONE_RE = /\b(done|fixed|complete|completed|implemented|merged|verified|all green|tests pass|ready|works now)\b/i;
|
|
@@ -18380,7 +16463,7 @@ var sleep = (ms2, signal) => new Promise((resolve13) => {
|
|
|
18380
16463
|
signal?.addEventListener("abort", onAbort, { once: true });
|
|
18381
16464
|
});
|
|
18382
16465
|
function hash(parts) {
|
|
18383
|
-
return
|
|
16466
|
+
return createHash15("sha256").update(parts.join("\0")).digest("hex").slice(0, 16);
|
|
18384
16467
|
}
|
|
18385
16468
|
function asString(value) {
|
|
18386
16469
|
if (typeof value === "string" && value.trim()) return value;
|
|
@@ -18788,7 +16871,7 @@ async function streamSessions(opts) {
|
|
|
18788
16871
|
|
|
18789
16872
|
// src/supervisor-run-context.ts
|
|
18790
16873
|
import { readFile as readFile14, stat as stat16 } from "fs/promises";
|
|
18791
|
-
import { basename as basename16, join as
|
|
16874
|
+
import { basename as basename16, join as join27 } from "path";
|
|
18792
16875
|
import { materializeTreeView } from "@tangle-network/agent-runtime/kernel";
|
|
18793
16876
|
import {
|
|
18794
16877
|
readRuntimeSupervisorRun
|
|
@@ -18950,14 +17033,14 @@ function parseInstant(v) {
|
|
|
18950
17033
|
}
|
|
18951
17034
|
async function readFileRunContext(runDir, opts = {}) {
|
|
18952
17035
|
const sources = await readRuntimeSupervisorRun(runDir);
|
|
18953
|
-
const coordination = readCoordinationLog(await readMaybe(
|
|
17036
|
+
const coordination = readCoordinationLog(await readMaybe(join27(runDir, COORDINATION_LOG_FILE)));
|
|
18954
17037
|
return {
|
|
18955
17038
|
...sources,
|
|
18956
17039
|
instanceId: opts.instanceId !== void 0 ? opts.instanceId : sources.instanceId,
|
|
18957
17040
|
arm: opts.arm !== void 0 ? opts.arm : basename16(runDir),
|
|
18958
17041
|
workers: sources.workers !== null && coordination.present ? workerSources(sources.workers, coordination) : sources.workers,
|
|
18959
17042
|
...sources.journal !== null ? {
|
|
18960
|
-
rootTranscriptRef:
|
|
17043
|
+
rootTranscriptRef: join27(runDir, SPAWN_JOURNAL_FILE),
|
|
18961
17044
|
traceCommand: `traces watch ${runDir}`
|
|
18962
17045
|
} : {}
|
|
18963
17046
|
};
|
|
@@ -18966,7 +17049,7 @@ function fileRunContextSupervisorRunReader(runDir, opts = {}) {
|
|
|
18966
17049
|
return { runRef: runDir, read: () => readFileRunContext(runDir, opts) };
|
|
18967
17050
|
}
|
|
18968
17051
|
async function isFileRunContextDir(runDir) {
|
|
18969
|
-
return stat16(
|
|
17052
|
+
return stat16(join27(runDir, SPAWN_JOURNAL_FILE)).then((s) => s.isFile()).catch(() => false);
|
|
18970
17053
|
}
|
|
18971
17054
|
function parseJsonObject(text2) {
|
|
18972
17055
|
if (text2 === null) return null;
|
|
@@ -19032,9 +17115,9 @@ function mergeRolls(a, b) {
|
|
|
19032
17115
|
};
|
|
19033
17116
|
}
|
|
19034
17117
|
async function readRunContextSnapshot(runDir) {
|
|
19035
|
-
const journal = readRunContextJournal(await readMaybe(
|
|
19036
|
-
const coordination = readCoordinationLog(await readMaybe(
|
|
19037
|
-
const resultText = await readMaybe(
|
|
17118
|
+
const journal = readRunContextJournal(await readMaybe(join27(runDir, SPAWN_JOURNAL_FILE)));
|
|
17119
|
+
const coordination = readCoordinationLog(await readMaybe(join27(runDir, COORDINATION_LOG_FILE)));
|
|
17120
|
+
const resultText = await readMaybe(join27(runDir, RESULT_FILE));
|
|
19038
17121
|
const result = parseResult(resultText);
|
|
19039
17122
|
const view = safeTreeView(journal.events);
|
|
19040
17123
|
const nodes = view === null ? [] : orderNodes(view, journal);
|
|
@@ -19217,7 +17300,7 @@ function wrap(text2, indent) {
|
|
|
19217
17300
|
|
|
19218
17301
|
// src/run-span-tree.ts
|
|
19219
17302
|
import { readdir as readdir18 } from "fs/promises";
|
|
19220
|
-
import { join as
|
|
17303
|
+
import { join as join28 } from "path";
|
|
19221
17304
|
import {
|
|
19222
17305
|
firstNumberAttr as firstNumberAttr3,
|
|
19223
17306
|
LLM_COST_ATTR_KEYS as LLM_COST_ATTR_KEYS2,
|
|
@@ -19492,7 +17575,7 @@ async function findSpanSourceFiles(dir) {
|
|
|
19492
17575
|
const found = [];
|
|
19493
17576
|
const entries = await readdir18(dir, { withFileTypes: true }).catch(() => []);
|
|
19494
17577
|
for (const entry of entries) {
|
|
19495
|
-
const child =
|
|
17578
|
+
const child = join28(dir, entry.name);
|
|
19496
17579
|
if (entry.isFile()) {
|
|
19497
17580
|
if (SPAN_FILE_PATTERN.test(entry.name)) found.push(child);
|
|
19498
17581
|
continue;
|
|
@@ -19500,7 +17583,7 @@ async function findSpanSourceFiles(dir) {
|
|
|
19500
17583
|
if (!entry.isDirectory()) continue;
|
|
19501
17584
|
const nested = await readdir18(child, { withFileTypes: true }).catch(() => []);
|
|
19502
17585
|
for (const inner of nested) {
|
|
19503
|
-
if (inner.isFile() && SPAN_FILE_PATTERN.test(inner.name)) found.push(
|
|
17586
|
+
if (inner.isFile() && SPAN_FILE_PATTERN.test(inner.name)) found.push(join28(child, inner.name));
|
|
19504
17587
|
}
|
|
19505
17588
|
}
|
|
19506
17589
|
return found.sort();
|
|
@@ -19508,7 +17591,7 @@ async function findSpanSourceFiles(dir) {
|
|
|
19508
17591
|
|
|
19509
17592
|
// src/supervisor-run-watch.ts
|
|
19510
17593
|
import { readdir as readdir19 } from "fs/promises";
|
|
19511
|
-
import { join as
|
|
17594
|
+
import { join as join29 } from "path";
|
|
19512
17595
|
function budget(node) {
|
|
19513
17596
|
const parts = [`${int(node.budget.maxIterations)}it`, `${int(node.budget.maxTokens)}tok`];
|
|
19514
17597
|
if (node.budget.maxUsd !== void 0) parts.push(`$${node.budget.maxUsd.toFixed(2)}`);
|
|
@@ -19652,7 +17735,7 @@ async function resolveRunContextDir(dir) {
|
|
|
19652
17735
|
const candidates = [];
|
|
19653
17736
|
for (const entry of entries) {
|
|
19654
17737
|
if (!entry.isDirectory()) continue;
|
|
19655
|
-
const child =
|
|
17738
|
+
const child = join29(dir, entry.name);
|
|
19656
17739
|
if (await isFileRunContextDir(child)) candidates.push(child);
|
|
19657
17740
|
}
|
|
19658
17741
|
if (candidates.length === 1) return candidates[0];
|
|
@@ -19988,7 +18071,7 @@ function parseJsonAnswer(text2) {
|
|
|
19988
18071
|
|
|
19989
18072
|
// src/ask.ts
|
|
19990
18073
|
import { mkdir as mkdir6, readFile as readFile15, writeFile as writeFile10 } from "fs/promises";
|
|
19991
|
-
import { join as
|
|
18074
|
+
import { join as join30, resolve as resolve11 } from "path";
|
|
19992
18075
|
import { performance } from "perf_hooks";
|
|
19993
18076
|
import { CostCeilingReachedError, CostLedger } from "@tangle-network/agent-eval";
|
|
19994
18077
|
import {
|
|
@@ -20543,8 +18626,8 @@ async function writeTraceQuestionsArtifacts(result, outDir) {
|
|
|
20543
18626
|
await mkdir6(directory, { recursive: true });
|
|
20544
18627
|
const paths = {
|
|
20545
18628
|
directory,
|
|
20546
|
-
result:
|
|
20547
|
-
report:
|
|
18629
|
+
result: join30(directory, "answers.json"),
|
|
18630
|
+
report: join30(directory, "report.md"),
|
|
20548
18631
|
traces: result.otlpPath
|
|
20549
18632
|
};
|
|
20550
18633
|
const { report, ...machine } = result;
|
|
@@ -20606,9 +18689,9 @@ async function watchSessions(opts) {
|
|
|
20606
18689
|
}
|
|
20607
18690
|
|
|
20608
18691
|
// src/bundle-view.ts
|
|
20609
|
-
import { createHash as
|
|
18692
|
+
import { createHash as createHash16 } from "crypto";
|
|
20610
18693
|
import { mkdir as mkdir7, readdir as readdir20, readFile as readFile16, writeFile as writeFile11 } from "fs/promises";
|
|
20611
|
-
import { dirname as dirname11, join as
|
|
18694
|
+
import { dirname as dirname11, join as join31, resolve as resolve12 } from "path";
|
|
20612
18695
|
var EVIDENCE_ONLY_ALLOW_LIST = [
|
|
20613
18696
|
{
|
|
20614
18697
|
path: "derived/session-index.json",
|
|
@@ -20732,7 +18815,7 @@ async function findSessionTextLeaks(opts) {
|
|
|
20732
18815
|
const index = /* @__PURE__ */ new Map();
|
|
20733
18816
|
const carried = [];
|
|
20734
18817
|
for (const path of opts.carriedPaths) {
|
|
20735
|
-
const signatures = sessionTextSignatures(await readFile16(
|
|
18818
|
+
const signatures = sessionTextSignatures(await readFile16(join31(opts.viewDir, path), "utf8"));
|
|
20736
18819
|
carried.push({ path, signatures: signatures.size });
|
|
20737
18820
|
for (const signature of signatures) if (!index.has(signature)) index.set(signature, path);
|
|
20738
18821
|
}
|
|
@@ -20740,7 +18823,7 @@ async function findSessionTextLeaks(opts) {
|
|
|
20740
18823
|
const comparedSources = [];
|
|
20741
18824
|
let sourceSignatures = 0;
|
|
20742
18825
|
for (const sourcePath of opts.excludedPaths) {
|
|
20743
|
-
const signatures = sessionTextSignatures(await readFile16(
|
|
18826
|
+
const signatures = sessionTextSignatures(await readFile16(join31(opts.sourceDir, sourcePath), "utf8"));
|
|
20744
18827
|
comparedSources.push(sourcePath);
|
|
20745
18828
|
sourceSignatures += signatures.size;
|
|
20746
18829
|
for (const signature of signatures) {
|
|
@@ -20799,17 +18882,17 @@ async function projectSessionBundle(opts) {
|
|
|
20799
18882
|
const candidates = [];
|
|
20800
18883
|
for (const path of await listSessionBundleFiles(sourceDir)) {
|
|
20801
18884
|
if (path === "manifest.json") continue;
|
|
20802
|
-
const bytes = await readFile16(
|
|
20803
|
-
const sha2562 =
|
|
18885
|
+
const bytes = await readFile16(join31(sourceDir, path));
|
|
18886
|
+
const sha2562 = createHash16("sha256").update(bytes).digest("hex");
|
|
20804
18887
|
const recorded = sourceHashes.get(path);
|
|
20805
18888
|
if (!recorded) {
|
|
20806
18889
|
throw new Error(
|
|
20807
|
-
`${
|
|
18890
|
+
`${join31(sourceDir, path)} is not listed in the bundle manifest \u2014 the bundle changed after assembly and cannot be projected; re-run \`traces bundle\``
|
|
20808
18891
|
);
|
|
20809
18892
|
}
|
|
20810
18893
|
if (recorded.sha256 !== sha2562) {
|
|
20811
18894
|
throw new Error(
|
|
20812
|
-
`${
|
|
18895
|
+
`${join31(sourceDir, path)} does not match its manifest hash (${recorded.sha256} recorded, ${sha2562} on disk) \u2014 the bundle changed after assembly and cannot be projected; re-run \`traces bundle\``
|
|
20813
18896
|
);
|
|
20814
18897
|
}
|
|
20815
18898
|
const verdict = classifyEvidenceOnlyPath(path);
|
|
@@ -20845,7 +18928,7 @@ async function projectSessionBundle(opts) {
|
|
|
20845
18928
|
});
|
|
20846
18929
|
continue;
|
|
20847
18930
|
}
|
|
20848
|
-
const target =
|
|
18931
|
+
const target = join31(outDir, candidate.path);
|
|
20849
18932
|
await mkdir7(dirname11(target), { recursive: true });
|
|
20850
18933
|
await writeFile11(target, candidate.bytes);
|
|
20851
18934
|
carriedPaths.push(candidate.path);
|
|
@@ -20862,7 +18945,7 @@ async function projectSessionBundle(opts) {
|
|
|
20862
18945
|
})),
|
|
20863
18946
|
matches: 0
|
|
20864
18947
|
};
|
|
20865
|
-
const sourceManifestBytes = await readFile16(
|
|
18948
|
+
const sourceManifestBytes = await readFile16(join31(sourceDir, "manifest.json"));
|
|
20866
18949
|
const manifest = {
|
|
20867
18950
|
schemaVersion: 2,
|
|
20868
18951
|
kind: "traces.session_bundle",
|
|
@@ -20876,7 +18959,7 @@ async function projectSessionBundle(opts) {
|
|
|
20876
18959
|
knownLimits: [...source.knownLimits, ...EVIDENCE_ONLY_LIMITS],
|
|
20877
18960
|
projection: {
|
|
20878
18961
|
sourceDirectory: sourceDir,
|
|
20879
|
-
sourceManifestSha256:
|
|
18962
|
+
sourceManifestSha256: createHash16("sha256").update(sourceManifestBytes).digest("hex"),
|
|
20880
18963
|
leakCheck
|
|
20881
18964
|
}
|
|
20882
18965
|
};
|
|
@@ -20887,11 +18970,11 @@ async function projectSessionBundle(opts) {
|
|
|
20887
18970
|
// src/upload-state.ts
|
|
20888
18971
|
import { mkdir as mkdir8, readFile as readFile17, writeFile as writeFile12 } from "fs/promises";
|
|
20889
18972
|
import { homedir as homedir12 } from "os";
|
|
20890
|
-
import { dirname as dirname12, join as
|
|
18973
|
+
import { dirname as dirname12, join as join32 } from "path";
|
|
20891
18974
|
import { contentHash } from "@tangle-network/agent-eval";
|
|
20892
18975
|
function stateFile() {
|
|
20893
|
-
const base = process.env.XDG_STATE_HOME ??
|
|
20894
|
-
return
|
|
18976
|
+
const base = process.env.XDG_STATE_HOME ?? join32(homedir12(), ".local", "state");
|
|
18977
|
+
return join32(base, "tangle-traces", "uploaded.json");
|
|
20895
18978
|
}
|
|
20896
18979
|
function uploadKey(harness, sessionId) {
|
|
20897
18980
|
return `${harness}:${sessionId}`;
|
|
@@ -20923,7 +19006,7 @@ function alreadyUploaded(state, harness, sessionId, hash2) {
|
|
|
20923
19006
|
import { readFile as readFile18 } from "fs/promises";
|
|
20924
19007
|
import { createRequire as createRequire2 } from "module";
|
|
20925
19008
|
import { hostname as hostname2 } from "os";
|
|
20926
|
-
import { basename as basename17, join as
|
|
19009
|
+
import { basename as basename17, join as join33 } from "path";
|
|
20927
19010
|
import { hostedClientFromEnv } from "@tangle-network/agent-eval/hosted";
|
|
20928
19011
|
import { REDACTION_VERSION } from "@tangle-network/agent-eval/traces";
|
|
20929
19012
|
var require2 = createRequire2(import.meta.url);
|
|
@@ -20947,7 +19030,7 @@ async function planUpload(opts = {}) {
|
|
|
20947
19030
|
async function gitBranch(cwd) {
|
|
20948
19031
|
if (!cwd) return void 0;
|
|
20949
19032
|
try {
|
|
20950
|
-
const head = await readFile18(
|
|
19033
|
+
const head = await readFile18(join33(cwd, ".git", "HEAD"), "utf8");
|
|
20951
19034
|
const m = head.match(/^ref:\s*refs\/heads\/(.+)$/m);
|
|
20952
19035
|
return m?.[1]?.trim();
|
|
20953
19036
|
} catch {
|
|
@@ -21275,60 +19358,20 @@ export {
|
|
|
21275
19358
|
hashSessionBundleFiles,
|
|
21276
19359
|
writeSessionBundleManifest,
|
|
21277
19360
|
readSessionBundleManifest,
|
|
19361
|
+
verifySessionBundle,
|
|
21278
19362
|
createBundleSourceReader,
|
|
21279
19363
|
assertOutsideSourceBundle,
|
|
21280
19364
|
isChatTrajectoryMessage,
|
|
21281
19365
|
isChatTrajectory,
|
|
21282
19366
|
chatTrajectoryToSpans,
|
|
21283
19367
|
importCodeTraceBench,
|
|
21284
|
-
|
|
21285
|
-
parseObservationOutput,
|
|
21286
|
-
deriveFailureSignature,
|
|
21287
|
-
ingestCodeTraceBenchSteps,
|
|
19368
|
+
DEFAULT_SANDBOX_BASE_URL,
|
|
21288
19369
|
sandboxReplayBackend,
|
|
21289
|
-
|
|
21290
|
-
SandboxCounterfactualRunner,
|
|
21291
|
-
replayVerify,
|
|
21292
|
-
parseReplayVerifyArgs,
|
|
21293
|
-
replayVerifyUsage,
|
|
21294
|
-
cmdReplayVerify,
|
|
21295
|
-
parseCorpusFlag,
|
|
21296
|
-
SUBMIT_ACTION_SIGNATURE,
|
|
21297
|
-
isSubmitAction,
|
|
21298
|
-
readLabelEntries,
|
|
21299
|
-
goldIncorrectSteps,
|
|
21300
|
-
resolveCaseResources,
|
|
21301
|
-
enumerateReplayableCases,
|
|
19370
|
+
assertSandboxReachable,
|
|
21302
19371
|
zaiChatCaller,
|
|
21303
|
-
|
|
21304
|
-
buildFixPrompt,
|
|
21305
|
-
buildRetryFixPrompt,
|
|
21306
|
-
countScriptCommands,
|
|
21307
|
-
extractFixCommand,
|
|
21308
|
-
generateFixCommand,
|
|
21309
|
-
runFixLoop,
|
|
21310
|
-
derivedImageTag,
|
|
21311
|
-
dockerImagePreparer,
|
|
21312
|
-
seededSample,
|
|
21313
|
-
runReplayBatch,
|
|
21314
|
-
renderBatchReport,
|
|
21315
|
-
parseReplayBatchArgs,
|
|
21316
|
-
replayBatchUsage,
|
|
19372
|
+
cmdReplayVerify,
|
|
21317
19373
|
cmdReplayVerifyBatch,
|
|
21318
|
-
|
|
21319
|
-
resolveFindingInvocation,
|
|
21320
|
-
replayVerifyFinding,
|
|
21321
|
-
findingReplayStep,
|
|
21322
|
-
findingTrajectoryId,
|
|
21323
|
-
resolveFindingReplayability,
|
|
21324
|
-
classifyVerdict,
|
|
21325
|
-
DEFAULT_SANDBOX_BASE_URL,
|
|
21326
|
-
assertSandboxReachable,
|
|
21327
|
-
verifyFindings,
|
|
21328
|
-
renderVerifiedFindingsSection,
|
|
21329
|
-
readFindingsFile,
|
|
21330
|
-
verifyFindingsUsage,
|
|
21331
|
-
parseVerifyFindingsArgs,
|
|
19374
|
+
verifyFindingsInSandbox,
|
|
21332
19375
|
cmdVerifyFindings,
|
|
21333
19376
|
writeHodoscopeInput,
|
|
21334
19377
|
hodoscopeAnalyzer,
|