@kontourai/flow-agents 4.2.0 → 4.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/build/src/builder-flow-runtime.js +10 -5
- package/context/contracts/verification-contract.md +6 -0
- package/dist/base/build/package.json +1 -1
- package/dist/base/build/src/builder-flow-runtime.js +10 -5
- package/dist/base/context/contracts/verification-contract.md +6 -0
- package/dist/base/evals/acceptance/prove-capture-teeth.sh +24 -0
- package/dist/base/evals/integration/test_effective_backlog_settings.sh +118 -15
- package/dist/base/install.sh +1 -1
- package/dist/base/scripts/hooks/lib/codex-exit-code.js +75 -23
- package/dist/claude-code/build/package.json +1 -1
- package/dist/claude-code/build/src/builder-flow-runtime.js +10 -5
- package/dist/claude-code/context/contracts/verification-contract.md +6 -0
- package/dist/claude-code/evals/acceptance/prove-capture-teeth.sh +24 -0
- package/dist/claude-code/evals/integration/test_effective_backlog_settings.sh +118 -15
- package/dist/claude-code/install.sh +1 -1
- package/dist/claude-code/scripts/hooks/lib/codex-exit-code.js +75 -23
- package/dist/codex/build/package.json +1 -1
- package/dist/codex/build/src/builder-flow-runtime.js +10 -5
- package/dist/codex/context/contracts/verification-contract.md +6 -0
- package/dist/codex/evals/acceptance/prove-capture-teeth.sh +24 -0
- package/dist/codex/evals/integration/test_effective_backlog_settings.sh +118 -15
- package/dist/codex/install.sh +1 -1
- package/dist/codex/scripts/hooks/lib/codex-exit-code.js +75 -23
- package/dist/kiro/build/package.json +1 -1
- package/dist/kiro/build/src/builder-flow-runtime.js +10 -5
- package/dist/kiro/context/contracts/verification-contract.md +6 -0
- package/dist/kiro/evals/acceptance/prove-capture-teeth.sh +24 -0
- package/dist/kiro/evals/integration/test_effective_backlog_settings.sh +118 -15
- package/dist/kiro/install.sh +1 -1
- package/dist/kiro/scripts/hooks/lib/codex-exit-code.js +75 -23
- package/dist/opencode/build/package.json +1 -1
- package/dist/opencode/build/src/builder-flow-runtime.js +10 -5
- package/dist/opencode/context/contracts/verification-contract.md +6 -0
- package/dist/opencode/evals/acceptance/prove-capture-teeth.sh +24 -0
- package/dist/opencode/evals/integration/test_effective_backlog_settings.sh +118 -15
- package/dist/opencode/install.sh +1 -1
- package/dist/opencode/scripts/hooks/lib/codex-exit-code.js +75 -23
- package/dist/pi/build/package.json +1 -1
- package/dist/pi/build/src/builder-flow-runtime.js +10 -5
- package/dist/pi/context/contracts/verification-contract.md +6 -0
- package/dist/pi/evals/acceptance/prove-capture-teeth.sh +24 -0
- package/dist/pi/evals/integration/test_effective_backlog_settings.sh +118 -15
- package/dist/pi/install.sh +1 -1
- package/dist/pi/scripts/hooks/lib/codex-exit-code.js +75 -23
- package/evals/acceptance/prove-capture-teeth.sh +24 -0
- package/evals/integration/test_effective_backlog_settings.sh +118 -15
- package/package.json +2 -1
- package/scripts/hooks/lib/codex-exit-code.js +75 -23
- package/src/builder-flow-runtime.ts +10 -5
- package/src/cli/builder-flow-runtime.test.mjs +55 -2
- package/src/cli/codex-exit-code.test.mjs +217 -2
|
@@ -113,10 +113,13 @@ function normalizeCallArguments(argumentsField) {
|
|
|
113
113
|
}
|
|
114
114
|
}
|
|
115
115
|
if (!parsed || typeof parsed !== 'object') return null;
|
|
116
|
-
|
|
116
|
+
// Codex records exec_command arguments under `cmd`; older shell fixtures
|
|
117
|
+
// and some adapters use `command`.
|
|
118
|
+
return normalizeCommandText(parsed.cmd) || normalizeCommandText(parsed.command);
|
|
117
119
|
}
|
|
118
120
|
|
|
119
121
|
const OUTPUT_FIELD_NEEDLE = '"output":"';
|
|
122
|
+
const COMMAND_FUNCTION_NAMES = new Set(['exec_command', 'shell']);
|
|
120
123
|
|
|
121
124
|
/**
|
|
122
125
|
* parseCandidateLine(line, maxLineHeadBytes) → candidate | null
|
|
@@ -149,7 +152,15 @@ function parseCandidateLine(line, maxLineHeadBytes) {
|
|
|
149
152
|
return { type: 'function_call_output', callId, output: payload.output };
|
|
150
153
|
}
|
|
151
154
|
if (payload.type === 'function_call') {
|
|
152
|
-
|
|
155
|
+
const name = typeof payload.name === 'string' && payload.name.trim() ? payload.name.trim() : null;
|
|
156
|
+
const command = normalizeCallArguments(payload.arguments);
|
|
157
|
+
const commandCapable = name === null || COMMAND_FUNCTION_NAMES.has(name) || command !== null;
|
|
158
|
+
return {
|
|
159
|
+
type: 'function_call',
|
|
160
|
+
callId,
|
|
161
|
+
commandCapable,
|
|
162
|
+
command,
|
|
163
|
+
};
|
|
153
164
|
}
|
|
154
165
|
return null;
|
|
155
166
|
}
|
|
@@ -214,13 +225,12 @@ function resolveContainedRealPath(transcriptPath) {
|
|
|
214
225
|
* Correlation policy (Decision B, #470 iteration 2, HIGH finding #4), in
|
|
215
226
|
* priority order:
|
|
216
227
|
* 1. call_id match wins — authoritative.
|
|
217
|
-
* 2. Absent a call_id match:
|
|
218
|
-
*
|
|
219
|
-
*
|
|
220
|
-
*
|
|
221
|
-
*
|
|
222
|
-
*
|
|
223
|
-
* back to the newest `function_call_output` banner.
|
|
228
|
+
* 2. Absent a call_id match: pair outputs with calls by rollout call_id and
|
|
229
|
+
* select the only pair whose normalized arguments match `command`. Zero
|
|
230
|
+
* matches or multiple matches DECLINE (`null`) rather than attribute a
|
|
231
|
+
* neighboring or repeated command's exit code.
|
|
232
|
+
* 3. If no pairing is resolvable at all and the rollout contains exactly one
|
|
233
|
+
* output with no function call, use that genuinely unpaired legacy output.
|
|
224
234
|
*
|
|
225
235
|
* Any failure (missing/unreadable/non-regular file, containment violation,
|
|
226
236
|
* malformed JSON lines, no candidate found) yields `null` — never throws.
|
|
@@ -271,41 +281,83 @@ function readExitCodeFromRollout(transcriptPath, options) {
|
|
|
271
281
|
}
|
|
272
282
|
if (truncated && lines.length > 1) lines.shift();
|
|
273
283
|
|
|
274
|
-
|
|
284
|
+
const outputsByRequestedCallId = [];
|
|
275
285
|
let newestOutputEntry = null;
|
|
276
|
-
|
|
286
|
+
const outputEntries = [];
|
|
287
|
+
const commandByCallId = new Map();
|
|
288
|
+
const seenFunctionCallIds = new Set();
|
|
289
|
+
const ambiguousCallIds = new Set();
|
|
290
|
+
let functionCallCount = 0;
|
|
291
|
+
let sawUnclassifiableLine = false;
|
|
292
|
+
let sawUnresolvableFunctionCall = false;
|
|
277
293
|
|
|
278
294
|
for (let i = lines.length - 1; i >= 0; i--) {
|
|
279
295
|
const line = lines[i].trim();
|
|
280
296
|
if (!line) continue;
|
|
281
297
|
const candidate = parseCandidateLine(line, maxLineHeadBytes);
|
|
282
|
-
if (!candidate)
|
|
298
|
+
if (!candidate) {
|
|
299
|
+
if (Buffer.byteLength(line, 'utf8') > maxLineHeadBytes) {
|
|
300
|
+
sawUnclassifiableLine = true;
|
|
301
|
+
} else {
|
|
302
|
+
try {
|
|
303
|
+
JSON.parse(line); // valid non-candidate events do not poison correlation
|
|
304
|
+
} catch {
|
|
305
|
+
sawUnclassifiableLine = true;
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
continue; // malformed/partial/unrecognized line — skip, keep scanning
|
|
309
|
+
}
|
|
283
310
|
|
|
284
311
|
if (candidate.type === 'function_call_output') {
|
|
312
|
+
outputEntries.push(candidate); // scan order is newest to oldest
|
|
285
313
|
if (newestOutputEntry === null) newestOutputEntry = candidate; // first seen scanning backward = newest
|
|
286
314
|
if (callId && candidate.callId === callId) {
|
|
287
|
-
|
|
288
|
-
break; // call_id match is authoritative — stop scanning
|
|
315
|
+
outputsByRequestedCallId.push(candidate);
|
|
289
316
|
}
|
|
290
317
|
} else if (candidate.type === 'function_call') {
|
|
291
|
-
|
|
292
|
-
|
|
318
|
+
functionCallCount += 1;
|
|
319
|
+
if (candidate.callId) {
|
|
320
|
+
if (seenFunctionCallIds.has(candidate.callId)) ambiguousCallIds.add(candidate.callId);
|
|
321
|
+
else seenFunctionCallIds.add(candidate.callId);
|
|
322
|
+
}
|
|
323
|
+
if (candidate.commandCapable && candidate.callId && candidate.command !== null) {
|
|
324
|
+
if (!commandByCallId.has(candidate.callId)) commandByCallId.set(candidate.callId, candidate.command);
|
|
325
|
+
} else if (candidate.commandCapable) {
|
|
326
|
+
sawUnresolvableFunctionCall = true;
|
|
293
327
|
}
|
|
294
328
|
}
|
|
295
329
|
}
|
|
296
330
|
|
|
297
331
|
let chosenOutput = null;
|
|
298
|
-
if (
|
|
299
|
-
|
|
332
|
+
if ((callId || command) && sawUnclassifiableLine) return null;
|
|
333
|
+
if (callId) {
|
|
334
|
+
if (outputsByRequestedCallId.length === 1 && !ambiguousCallIds.has(callId)) {
|
|
335
|
+
chosenOutput = outputsByRequestedCallId[0].output;
|
|
336
|
+
} else {
|
|
337
|
+
return null; // explicit call ID is unresolved, reused, or has duplicate outputs
|
|
338
|
+
}
|
|
300
339
|
} else if (newestOutputEntry) {
|
|
301
|
-
if (
|
|
302
|
-
if (
|
|
303
|
-
|
|
340
|
+
if (command) {
|
|
341
|
+
if (sawUnresolvableFunctionCall) return null;
|
|
342
|
+
const matchingCallIds = [...commandByCallId.entries()]
|
|
343
|
+
.filter(([, candidateCommand]) => candidateCommand === command)
|
|
344
|
+
.map(([candidateCallId]) => candidateCallId);
|
|
345
|
+
const matches = outputEntries.filter((entry) => commandByCallId.get(entry.callId) === command);
|
|
346
|
+
if (matchingCallIds.length === 1
|
|
347
|
+
&& !ambiguousCallIds.has(matchingCallIds[0])
|
|
348
|
+
&& matches.length === 1) {
|
|
349
|
+
chosenOutput = matches[0].output;
|
|
350
|
+
} else if (matchingCallIds.length > 0 || matches.length > 0 || commandByCallId.size > 0) {
|
|
351
|
+
return null; // no unique command correlation
|
|
352
|
+
} else if (!truncated && outputEntries.length === 1 && functionCallCount === 0) {
|
|
353
|
+
chosenOutput = newestOutputEntry.output; // genuinely unpaired legacy fallback
|
|
304
354
|
} else {
|
|
305
|
-
return null; //
|
|
355
|
+
return null; // multi-call or partially paired tail
|
|
306
356
|
}
|
|
357
|
+
} else if (!truncated && outputEntries.length === 1 && functionCallCount === 0) {
|
|
358
|
+
chosenOutput = newestOutputEntry.output; // genuinely unpaired legacy fallback
|
|
307
359
|
} else {
|
|
308
|
-
|
|
360
|
+
return null; // no correlation signal for a paired or multi-call tail
|
|
309
361
|
}
|
|
310
362
|
}
|
|
311
363
|
|
|
@@ -710,7 +710,7 @@ async function bundleGateEvidence(
|
|
|
710
710
|
throw new BuilderBuildRunInputError("evidence.claims.metadata.gate_claim.route_reason", `is not declared by gate ${String((gate as AnyRecord).id ?? "<unknown>")}`);
|
|
711
711
|
}
|
|
712
712
|
if (String((gate as AnyRecord).id) === "verify-gate" && relevant.some((claim) => claim.claimType === "builder.verify.tests" && claim.value === "pass")) {
|
|
713
|
-
await assertVerifiedTestsTrust(
|
|
713
|
+
await assertVerifiedTestsTrust(relevant, projectRoot);
|
|
714
714
|
}
|
|
715
715
|
return { failed, routeReason, expectationIds, visitEnteredAt: enteredAt };
|
|
716
716
|
}
|
|
@@ -739,15 +739,20 @@ function timestampAtOrAfter(value: unknown, boundary: number): boolean {
|
|
|
739
739
|
return parsed !== null && parsed >= boundary;
|
|
740
740
|
}
|
|
741
741
|
|
|
742
|
-
async function assertVerifiedTestsTrust(
|
|
743
|
-
const testClaims =
|
|
742
|
+
async function assertVerifiedTestsTrust(currentGateClaims: AnyRecord[], projectRoot: string): Promise<void> {
|
|
743
|
+
const testClaims = currentGateClaims.filter((claim): claim is AnyRecord => isRecord(claim)
|
|
744
744
|
&& claim.claimType === "builder.verify.tests"
|
|
745
745
|
&& claim.value === "pass"
|
|
746
746
|
&& isRecord(claim.metadata)
|
|
747
747
|
&& isRecord(claim.metadata.gate_claim)
|
|
748
748
|
&& claim.metadata.gate_claim.expectation_id === "tests-evidence");
|
|
749
749
|
if (testClaims.length === 0) throw new BuilderBuildRunInputError("evidence.tests", "is missing a passing tests-evidence claim");
|
|
750
|
-
|
|
750
|
+
// A route-back starts a new gate visit and therefore a new critique generation. Historical
|
|
751
|
+
// reviewer slices remain in the bundle and manifest for audit, but only critiques acquired
|
|
752
|
+
// during this visit describe the implementation snapshot currently being verified. Within a
|
|
753
|
+
// visit every live reviewer slice still participates, so changing reviewers cannot bury a
|
|
754
|
+
// disputed finding.
|
|
755
|
+
const liveCritiques = currentGateClaims.filter((claim): claim is AnyRecord => isRecord(claim)
|
|
751
756
|
&& isRecord(claim.metadata)
|
|
752
757
|
&& claim.metadata.origin === "critique"
|
|
753
758
|
&& typeof claim.metadata.superseded_by !== "string");
|
|
@@ -763,7 +768,7 @@ async function assertVerifiedTestsTrust(claims: unknown[], projectRoot: string):
|
|
|
763
768
|
await Promise.all(artifacts.map((artifact) => assertReviewedArtifactDigest(artifact, projectRoot)));
|
|
764
769
|
assertReviewedWorkspaceSnapshot(claim, artifacts, projectRoot);
|
|
765
770
|
}));
|
|
766
|
-
const criteria =
|
|
771
|
+
const criteria = currentGateClaims.filter((claim): claim is AnyRecord => isRecord(claim) && isRecord(claim.metadata) && claim.metadata.origin === "acceptance");
|
|
767
772
|
if (criteria.length === 0 || criteria.some((claim) => {
|
|
768
773
|
const criterion = isRecord(claim.metadata.criterion) ? claim.metadata.criterion : null;
|
|
769
774
|
return claim.value !== "pass" || !criterion || !Array.isArray(criterion.evidence_refs) || criterion.evidence_refs.length === 0;
|
|
@@ -1382,6 +1382,8 @@ test("failed verification projects Flow-owned route-back attempt and budget", as
|
|
|
1382
1382
|
assert.equal(verify.run.state.current_step, "verify");
|
|
1383
1383
|
|
|
1384
1384
|
const failureTimestamp = new Date().toISOString();
|
|
1385
|
+
const initialPrerequisites = verifiedTestsPrerequisites(session, failureTimestamp);
|
|
1386
|
+
initialPrerequisites[0].claim.metadata.reviewer = "reviewer-before-route-back";
|
|
1385
1387
|
const routed = await writeAndSync(session, [bundleClaim({
|
|
1386
1388
|
expectation: "tests-evidence",
|
|
1387
1389
|
claimType: "builder.verify.tests",
|
|
@@ -1389,7 +1391,7 @@ test("failed verification projects Flow-owned route-back attempt and budget", as
|
|
|
1389
1391
|
status: "fail",
|
|
1390
1392
|
routeReason: "implementation_defect",
|
|
1391
1393
|
timestamp: failureTimestamp,
|
|
1392
|
-
}), ...
|
|
1394
|
+
}), ...initialPrerequisites]);
|
|
1393
1395
|
|
|
1394
1396
|
assert.equal(routed.run.state.current_step, "execute");
|
|
1395
1397
|
assert.equal(routed.projection.flow_run.route_back_attempt, 1);
|
|
@@ -1415,15 +1417,66 @@ test("failed verification projects Flow-owned route-back attempt and budget", as
|
|
|
1415
1417
|
assert.equal(staleRetry.attached, false);
|
|
1416
1418
|
assert.equal(staleRetry.run.state.current_step, "verify");
|
|
1417
1419
|
assert.equal(staleRetry.run.state.transitions.filter((transition) => transition.type === "route_back").length, 1);
|
|
1420
|
+
fs.writeFileSync(path.join(session.projectRoot, "review-target", "delivery.md"), "corrected delivery after route-back\n");
|
|
1418
1421
|
const correctedAt = new Date(Date.parse(reentered.run.state.transitions.at(-1).at) + 1).toISOString();
|
|
1422
|
+
const correctedPrerequisites = verifiedTestsPrerequisites(session, correctedAt)
|
|
1423
|
+
.map((entry, index) => withIdentitySuffix(entry, `corrected-${index}`));
|
|
1424
|
+
correctedPrerequisites[0].claim.metadata.reviewer = "reviewer-after-route-back";
|
|
1419
1425
|
const corrected = await writeAndSync(session, [
|
|
1420
1426
|
withIdentitySuffix(bundleClaim({ expectation: "tests-evidence", claimType: "builder.verify.tests", subjectType: "flow-step", timestamp: correctedAt }), "corrected"),
|
|
1421
|
-
...
|
|
1427
|
+
...correctedPrerequisites,
|
|
1428
|
+
// Compose-safe writers preserve this older reviewer's still-live PASS slice. It targets the
|
|
1429
|
+
// prior implementation bytes and must remain audit history without deadlocking the new gate
|
|
1430
|
+
// visit or requiring the new reviewer to impersonate the old one.
|
|
1431
|
+
initialPrerequisites[0],
|
|
1422
1432
|
]);
|
|
1423
1433
|
assert.equal(corrected.run.state.current_step, "merge-ready");
|
|
1424
1434
|
const verifyEvidence = corrected.run.manifest.evidence.filter((entry) => entry.gate_id === "verify-gate");
|
|
1425
1435
|
assert.equal(verifyEvidence.length, 2);
|
|
1426
1436
|
assert.equal(verifyEvidence[0].superseded_by, verifyEvidence[1].id);
|
|
1437
|
+
const retainedCritiques = readJson(path.join(session.sessionDir, "trust.bundle")).claims
|
|
1438
|
+
.filter((claim) => claim.metadata?.origin === "critique" && !claim.metadata.superseded_by);
|
|
1439
|
+
assert.deepEqual(retainedCritiques.map((claim) => claim.metadata.reviewer).sort(), [
|
|
1440
|
+
"reviewer-after-route-back",
|
|
1441
|
+
"reviewer-before-route-back",
|
|
1442
|
+
]);
|
|
1443
|
+
});
|
|
1444
|
+
|
|
1445
|
+
test("a different passing reviewer cannot hide a disputed critique in the same gate visit", async () => {
|
|
1446
|
+
const session = makeSession("same-visit-reviewer-handoff");
|
|
1447
|
+
await startBuilderFlowSession({ sessionDir: session.sessionDir });
|
|
1448
|
+
await writeAndSync(session, [bundleClaim({ expectation: "selected-work", claimType: "builder.pull-work.selected", subjectType: "work-item" })]);
|
|
1449
|
+
await writeAndSync(session, [
|
|
1450
|
+
bundleClaim({ expectation: "pickup-probe-readiness", claimType: "builder.design-probe.pickup-readiness", subjectType: "work-item" }),
|
|
1451
|
+
bundleClaim({ expectation: "probe-decisions-or-accepted-gaps", claimType: "builder.design-probe.decisions", subjectType: "decision" }),
|
|
1452
|
+
]);
|
|
1453
|
+
await writeAndSync(session, [bundleClaim({ expectation: "implementation-plan", claimType: "builder.plan.implementation", subjectType: "artifact" })]);
|
|
1454
|
+
await writeAndSync(session, [bundleClaim({ expectation: "implementation-scope", claimType: "builder.execute.scope", subjectType: "change" })]);
|
|
1455
|
+
|
|
1456
|
+
const timestamp = new Date().toISOString();
|
|
1457
|
+
const prerequisites = verifiedTestsPrerequisites(session, timestamp);
|
|
1458
|
+
prerequisites[0].claim.metadata.reviewer = "passing-reviewer";
|
|
1459
|
+
const disputedCritique = withIdentitySuffix(structuredClone(prerequisites[0]), "disputed-reviewer");
|
|
1460
|
+
disputedCritique.claim.value = "fail";
|
|
1461
|
+
disputedCritique.claim.status = "disputed";
|
|
1462
|
+
disputedCritique.claim.metadata.reviewer = "disputed-reviewer";
|
|
1463
|
+
disputedCritique.claim.metadata.lanes = [{ id: "code", status: "fail" }];
|
|
1464
|
+
disputedCritique.claim.metadata.findings = [{ id: "unresolved", status: "open", severity: "high" }];
|
|
1465
|
+
disputedCritique.event.status = "disputed";
|
|
1466
|
+
|
|
1467
|
+
const blocked = await writeAndSync(session, [
|
|
1468
|
+
bundleClaim({ expectation: "tests-evidence", claimType: "builder.verify.tests", subjectType: "flow-step", timestamp }),
|
|
1469
|
+
...prerequisites,
|
|
1470
|
+
disputedCritique,
|
|
1471
|
+
]);
|
|
1472
|
+
assert.equal(blocked.attached, false);
|
|
1473
|
+
assert.equal(blocked.run.state.current_step, "verify");
|
|
1474
|
+
const retainedCritiques = readJson(path.join(session.sessionDir, "trust.bundle")).claims
|
|
1475
|
+
.filter((claim) => claim.metadata?.origin === "critique" && !claim.metadata.superseded_by);
|
|
1476
|
+
assert.deepEqual(retainedCritiques.map((claim) => [claim.metadata.reviewer, claim.value]).sort(), [
|
|
1477
|
+
["disputed-reviewer", "fail"],
|
|
1478
|
+
["passing-reviewer", "pass"],
|
|
1479
|
+
]);
|
|
1427
1480
|
});
|
|
1428
1481
|
|
|
1429
1482
|
test("producer-superseded FAIL is audit history and live PASS drives verify", async () => {
|
|
@@ -55,6 +55,22 @@ function functionCall(callId, command) {
|
|
|
55
55
|
};
|
|
56
56
|
}
|
|
57
57
|
|
|
58
|
+
function execCommandCall(callId, cmd) {
|
|
59
|
+
return {
|
|
60
|
+
timestamp: "2026-07-06T00:00:00Z",
|
|
61
|
+
type: "response_item",
|
|
62
|
+
payload: { type: "function_call", call_id: callId, name: "exec_command", arguments: JSON.stringify({ cmd }) },
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function functionToolCall(callId, name, args) {
|
|
67
|
+
return {
|
|
68
|
+
timestamp: "2026-07-06T00:00:00Z",
|
|
69
|
+
type: "response_item",
|
|
70
|
+
payload: { type: "function_call", call_id: callId, name, arguments: JSON.stringify(args) },
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
|
|
58
74
|
// --- extractExitCodeFromBanner: preamble-anchored (CRITICAL finding #1) ---
|
|
59
75
|
|
|
60
76
|
test("extractExitCodeFromBanner: forgery in post-Output: stdout is ignored (preamble wins)", () => {
|
|
@@ -95,7 +111,7 @@ test("readExitCodeFromRollout: malformed JSONL lines are skipped, valid entry st
|
|
|
95
111
|
|
|
96
112
|
// --- truncation arithmetic (HEAD-anchored bounded scan, MEDIUM finding #5) ---
|
|
97
113
|
|
|
98
|
-
test("readExitCodeFromRollout: target
|
|
114
|
+
test("readExitCodeFromRollout: call-id target near EOF is found when file exceeds the scan window", () => {
|
|
99
115
|
const filler = [];
|
|
100
116
|
for (let i = 0; i < 60; i++) filler.push({ timestamp: "2026-07-06T00:00:00Z", type: "turn_context", payload: {} });
|
|
101
117
|
const target = functionCallOutput("call_1", "Process exited with code 1\nOutput:\n...");
|
|
@@ -106,7 +122,7 @@ test("readExitCodeFromRollout: target line near EOF still found when file exceed
|
|
|
106
122
|
// Window smaller than the whole file (forces truncation) but comfortably
|
|
107
123
|
// larger than the target line itself.
|
|
108
124
|
const maxScanBytes = targetLineBytes + 50;
|
|
109
|
-
assert.equal(readExitCodeFromRollout(file, { maxScanBytes }), 1);
|
|
125
|
+
assert.equal(readExitCodeFromRollout(file, { callId: "call_1", maxScanBytes }), 1);
|
|
110
126
|
});
|
|
111
127
|
|
|
112
128
|
test("readExitCodeFromRollout: target line start beyond maxScanBytes yields null (never mis-reads a fragment)", () => {
|
|
@@ -130,6 +146,15 @@ test("readExitCodeFromRollout: call_id match wins over the newer entry", () => {
|
|
|
130
146
|
assert.equal(readExitCodeFromRollout(file, { callId: "call_b" }), 5);
|
|
131
147
|
});
|
|
132
148
|
|
|
149
|
+
test("readExitCodeFromRollout: an unresolved explicit call_id never downgrades to command matching", () => {
|
|
150
|
+
const file = writeRollout([
|
|
151
|
+
execCommandCall("call_other", "npm test"),
|
|
152
|
+
functionCallOutput("call_other", "Process exited with code 0\nOutput:\n..."),
|
|
153
|
+
]);
|
|
154
|
+
|
|
155
|
+
assert.equal(readExitCodeFromRollout(file, { callId: "call_missing", command: "npm test" }), null);
|
|
156
|
+
});
|
|
157
|
+
|
|
133
158
|
// --- command cross-check correlation (Decision B #2, HIGH finding #4) ---
|
|
134
159
|
|
|
135
160
|
test("readExitCodeFromRollout: command cross-check mismatch declines to null", () => {
|
|
@@ -148,6 +173,196 @@ test("readExitCodeFromRollout: command cross-check match uses the correlated ent
|
|
|
148
173
|
assert.equal(readExitCodeFromRollout(file, { command: "npm run lint" }), 1);
|
|
149
174
|
});
|
|
150
175
|
|
|
176
|
+
test("readExitCodeFromRollout: Codex exec_command cmd argument correlates the result", () => {
|
|
177
|
+
const file = writeRollout([
|
|
178
|
+
execCommandCall("call_1", "npm run lint"),
|
|
179
|
+
functionCallOutput("call_1", "Process exited with code 1\nOutput:\n..."),
|
|
180
|
+
]);
|
|
181
|
+
assert.equal(readExitCodeFromRollout(file, { command: "npm run lint" }), 1);
|
|
182
|
+
assert.equal(readExitCodeFromRollout(file, { command: "npm test" }), null);
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
test("readExitCodeFromRollout: parallel call cannot inherit the previous call result", () => {
|
|
186
|
+
const file = writeRollout([
|
|
187
|
+
execCommandCall("call_pass", "find seed -type f"),
|
|
188
|
+
execCommandCall("call_fail", "cat seed/src/window.js"),
|
|
189
|
+
functionCallOutput("call_pass", "Process exited with code 0\nOutput:\n..."),
|
|
190
|
+
]);
|
|
191
|
+
|
|
192
|
+
assert.equal(readExitCodeFromRollout(file, { command: "find seed -type f" }), 0);
|
|
193
|
+
assert.equal(readExitCodeFromRollout(file, { command: "cat seed/src/window.js" }), null);
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
test("readExitCodeFromRollout: parallel outputs correlate to their exact commands", () => {
|
|
197
|
+
const file = writeRollout([
|
|
198
|
+
execCommandCall("call_pass", "find seed -type f"),
|
|
199
|
+
execCommandCall("call_fail", "cat seed/src/window.js"),
|
|
200
|
+
functionCallOutput("call_pass", "Process exited with code 0\nOutput:\n..."),
|
|
201
|
+
functionCallOutput("call_fail", "Process exited with code 1\nOutput:\n..."),
|
|
202
|
+
]);
|
|
203
|
+
|
|
204
|
+
assert.equal(readExitCodeFromRollout(file, { command: "find seed -type f" }), 0);
|
|
205
|
+
assert.equal(readExitCodeFromRollout(file, { command: "cat seed/src/window.js" }), 1);
|
|
206
|
+
});
|
|
207
|
+
|
|
208
|
+
test("readExitCodeFromRollout: repeated parallel command text is ambiguous without call_id", () => {
|
|
209
|
+
const file = writeRollout([
|
|
210
|
+
execCommandCall("call_a", "npm test"),
|
|
211
|
+
execCommandCall("call_b", "npm test"),
|
|
212
|
+
functionCallOutput("call_a", "Process exited with code 0\nOutput:\n..."),
|
|
213
|
+
functionCallOutput("call_b", "Process exited with code 1\nOutput:\n..."),
|
|
214
|
+
]);
|
|
215
|
+
|
|
216
|
+
assert.equal(readExitCodeFromRollout(file, { command: "npm test" }), null);
|
|
217
|
+
assert.equal(readExitCodeFromRollout(file, { callId: "call_a", command: "npm test" }), 0);
|
|
218
|
+
assert.equal(readExitCodeFromRollout(file, { callId: "call_b", command: "npm test" }), 1);
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
test("readExitCodeFromRollout: a repeated call without its output cannot reuse an older result", () => {
|
|
222
|
+
const file = writeRollout([
|
|
223
|
+
execCommandCall("call_old", "npm test"),
|
|
224
|
+
functionCallOutput("call_old", "Process exited with code 0\nOutput:\n..."),
|
|
225
|
+
execCommandCall("call_current", "npm test"),
|
|
226
|
+
]);
|
|
227
|
+
|
|
228
|
+
assert.equal(readExitCodeFromRollout(file, { command: "npm test" }), null);
|
|
229
|
+
});
|
|
230
|
+
|
|
231
|
+
test("readExitCodeFromRollout: unresolved multi-call tail never falls back to newest output", () => {
|
|
232
|
+
const file = writeRollout([
|
|
233
|
+
{
|
|
234
|
+
timestamp: "2026-07-06T00:00:00Z",
|
|
235
|
+
type: "response_item",
|
|
236
|
+
payload: { type: "function_call", call_id: "call_a", name: "unknown", arguments: "not-json" },
|
|
237
|
+
},
|
|
238
|
+
functionCallOutput("call_a", "Process exited with code 0\nOutput:\n..."),
|
|
239
|
+
functionCallOutput("call_b", "Process exited with code 1\nOutput:\n..."),
|
|
240
|
+
]);
|
|
241
|
+
|
|
242
|
+
assert.equal(readExitCodeFromRollout(file, { command: "npm test" }), null);
|
|
243
|
+
assert.equal(readExitCodeFromRollout(file, {}), null);
|
|
244
|
+
});
|
|
245
|
+
|
|
246
|
+
test("readExitCodeFromRollout: malformed function call cannot make an output look unpaired", () => {
|
|
247
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "codex-exit-code-"));
|
|
248
|
+
const file = path.join(dir, "rollout.jsonl");
|
|
249
|
+
const output = JSON.stringify(functionCallOutput("call_old", "Process exited with code 0\nOutput:\n..."));
|
|
250
|
+
fs.writeFileSync(file, `${output}\n{"payload":{"type":"function_call","call_id":"call_current"\n`);
|
|
251
|
+
|
|
252
|
+
assert.equal(readExitCodeFromRollout(file, { command: "npm test" }), null);
|
|
253
|
+
});
|
|
254
|
+
|
|
255
|
+
test("readExitCodeFromRollout: truncated call context cannot make an output look unpaired", () => {
|
|
256
|
+
const output = functionCallOutput("call_old", "Process exited with code 7\nOutput:\n...");
|
|
257
|
+
const file = writeRollout([functionCall("call_old", "npm run lint"), output]);
|
|
258
|
+
const outputLineBytes = Buffer.byteLength(JSON.stringify(output), "utf8") + 1;
|
|
259
|
+
|
|
260
|
+
assert.equal(readExitCodeFromRollout(file, {
|
|
261
|
+
command: "npm test",
|
|
262
|
+
maxScanBytes: outputLineBytes + 10,
|
|
263
|
+
}), null);
|
|
264
|
+
});
|
|
265
|
+
|
|
266
|
+
test("readExitCodeFromRollout: reused call IDs are ambiguous", () => {
|
|
267
|
+
const file = writeRollout([
|
|
268
|
+
execCommandCall("same", "npm test"),
|
|
269
|
+
execCommandCall("same", "npm run lint"),
|
|
270
|
+
functionCallOutput("same", "Process exited with code 9\nOutput:\n..."),
|
|
271
|
+
]);
|
|
272
|
+
|
|
273
|
+
assert.equal(readExitCodeFromRollout(file, { command: "npm test" }), null);
|
|
274
|
+
assert.equal(readExitCodeFromRollout(file, { command: "npm run lint" }), null);
|
|
275
|
+
});
|
|
276
|
+
|
|
277
|
+
test("readExitCodeFromRollout: an unresolvable current call blocks historical command reuse", () => {
|
|
278
|
+
const file = writeRollout([
|
|
279
|
+
execCommandCall("call_old", "npm test"),
|
|
280
|
+
functionCallOutput("call_old", "Process exited with code 0\nOutput:\n..."),
|
|
281
|
+
{
|
|
282
|
+
timestamp: "2026-07-06T00:00:01Z",
|
|
283
|
+
type: "response_item",
|
|
284
|
+
payload: { type: "function_call", call_id: "call_current", name: "exec_command", arguments: "not-json" },
|
|
285
|
+
},
|
|
286
|
+
]);
|
|
287
|
+
|
|
288
|
+
assert.equal(readExitCodeFromRollout(file, { command: "npm test" }), null);
|
|
289
|
+
});
|
|
290
|
+
|
|
291
|
+
test("readExitCodeFromRollout: malformed duplicate call IDs remain ambiguous", () => {
|
|
292
|
+
const file = writeRollout([
|
|
293
|
+
execCommandCall("same", "npm test"),
|
|
294
|
+
{
|
|
295
|
+
timestamp: "2026-07-06T00:00:01Z",
|
|
296
|
+
type: "response_item",
|
|
297
|
+
payload: { type: "function_call", call_id: "same", name: "exec_command", arguments: "not-json" },
|
|
298
|
+
},
|
|
299
|
+
functionCallOutput("same", "Process exited with code 9\nOutput:\n..."),
|
|
300
|
+
]);
|
|
301
|
+
|
|
302
|
+
assert.equal(readExitCodeFromRollout(file, { callId: "same", command: "npm test" }), null);
|
|
303
|
+
assert.equal(readExitCodeFromRollout(file, { command: "npm test" }), null);
|
|
304
|
+
});
|
|
305
|
+
|
|
306
|
+
test("readExitCodeFromRollout: malformed JSON blocks historical and explicit-id correlation", () => {
|
|
307
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "codex-exit-code-"));
|
|
308
|
+
const file = path.join(dir, "rollout.jsonl");
|
|
309
|
+
const call = JSON.stringify(execCommandCall("same", "npm test"));
|
|
310
|
+
const output = JSON.stringify(functionCallOutput("same", "Process exited with code 0\nOutput:\n..."));
|
|
311
|
+
fs.writeFileSync(file, `${call}\n${output}\n{"payload":{"type":"function_call","call_id":"same"\n`);
|
|
312
|
+
|
|
313
|
+
assert.equal(readExitCodeFromRollout(file, { command: "npm test" }), null);
|
|
314
|
+
assert.equal(readExitCodeFromRollout(file, { callId: "same", command: "npm test" }), null);
|
|
315
|
+
});
|
|
316
|
+
|
|
317
|
+
test("readExitCodeFromRollout: a partial function-call token blocks historical correlation", () => {
|
|
318
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "codex-exit-code-"));
|
|
319
|
+
const file = path.join(dir, "rollout.jsonl");
|
|
320
|
+
const call = JSON.stringify(execCommandCall("old", "npm test"));
|
|
321
|
+
const output = JSON.stringify(functionCallOutput("old", "Process exited with code 0\nOutput:\n..."));
|
|
322
|
+
fs.writeFileSync(file, `${call}\n${output}\n{"payload":{"type":"function_cal\n`);
|
|
323
|
+
|
|
324
|
+
assert.equal(readExitCodeFromRollout(file, { command: "npm test" }), null);
|
|
325
|
+
});
|
|
326
|
+
|
|
327
|
+
test("readExitCodeFromRollout: unrelated function tools do not poison exact command correlation", () => {
|
|
328
|
+
const file = writeRollout([
|
|
329
|
+
functionToolCall("patch_call", "apply_patch", { patch: "*** Begin Patch" }),
|
|
330
|
+
execCommandCall("command_call", "npm test"),
|
|
331
|
+
functionCallOutput("command_call", "Process exited with code 0\nOutput:\n..."),
|
|
332
|
+
]);
|
|
333
|
+
|
|
334
|
+
assert.equal(readExitCodeFromRollout(file, { command: "npm test" }), 0);
|
|
335
|
+
});
|
|
336
|
+
|
|
337
|
+
test("readExitCodeFromRollout: nameless command-shaped calls participate in ambiguity checks", () => {
|
|
338
|
+
const file = writeRollout([
|
|
339
|
+
execCommandCall("old", "npm test"),
|
|
340
|
+
functionCallOutput("old", "Process exited with code 0\nOutput:\n..."),
|
|
341
|
+
functionToolCall("current", null, { cmd: "npm test" }),
|
|
342
|
+
functionCallOutput("current", "Process exited with code 1\nOutput:\n..."),
|
|
343
|
+
]);
|
|
344
|
+
|
|
345
|
+
assert.equal(readExitCodeFromRollout(file, { command: "npm test" }), null);
|
|
346
|
+
assert.equal(readExitCodeFromRollout(file, { callId: "current", command: "npm test" }), 1);
|
|
347
|
+
});
|
|
348
|
+
|
|
349
|
+
test("readExitCodeFromRollout: semantically nameless malformed calls block historical command reuse", () => {
|
|
350
|
+
for (const name of [null, "", " "]) {
|
|
351
|
+
const file = writeRollout([
|
|
352
|
+
execCommandCall("old", "npm test"),
|
|
353
|
+
functionCallOutput("old", "Process exited with code 0\nOutput:\n..."),
|
|
354
|
+
{
|
|
355
|
+
timestamp: "2026-07-06T00:00:01Z",
|
|
356
|
+
type: "response_item",
|
|
357
|
+
payload: { type: "function_call", call_id: "current", name, arguments: "not-json" },
|
|
358
|
+
},
|
|
359
|
+
functionCallOutput("current", "Process exited with code 1\nOutput:\n..."),
|
|
360
|
+
]);
|
|
361
|
+
|
|
362
|
+
assert.equal(readExitCodeFromRollout(file, { command: "npm test" }), null);
|
|
363
|
+
}
|
|
364
|
+
});
|
|
365
|
+
|
|
151
366
|
test("readExitCodeFromRollout: no resolvable pairing falls back to the newest banner (single-call case)", () => {
|
|
152
367
|
const file = writeRollout([
|
|
153
368
|
functionCallOutput("call_1", "Process exited with code 1\nOutput:\n..."),
|