harnery 0.45.0 → 0.45.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commander.js +0 -1
- package/dist/commands/files.d.ts +2 -2
- package/dist/commands/files.d.ts.map +1 -1
- package/dist/commands/files.js +4 -7
- package/dist/commands/servers.d.ts.map +1 -1
- package/dist/commands/servers.js +17 -11
- package/dist/commands/tunnel.d.ts +0 -15
- package/dist/commands/tunnel.d.ts.map +1 -1
- package/dist/commands/tunnel.js +153 -453
- package/dist/core/agents/live-authority-v3.d.ts +0 -1
- package/dist/core/agents/live-authority-v3.d.ts.map +1 -1
- package/dist/core/agents/live-authority-v3.js +0 -11
- package/dist/core/agents/live-lifecycle-v3.d.ts +2 -4
- package/dist/core/agents/live-lifecycle-v3.d.ts.map +1 -1
- package/dist/core/agents/live-lifecycle-v3.js +7 -28
- package/dist/core/agents/state/heartbeat-writer.d.ts.map +1 -1
- package/dist/core/agents/state/heartbeat-writer.js +14 -25
- package/dist/core/agents/state/live-coordination-writer.d.ts.map +1 -1
- package/dist/core/agents/state/live-coordination-writer.js +0 -5
- package/dist/core/agents/state/pidmap.d.ts.map +1 -1
- package/dist/core/agents/state/pidmap.js +0 -5
- package/dist/core/agents/state/stale-sweep.d.ts.map +1 -1
- package/dist/core/agents/state/stale-sweep.js +70 -180
- package/dist/core/artifacts/activity.d.ts +4 -1
- package/dist/core/artifacts/activity.d.ts.map +1 -1
- package/dist/core/artifacts/activity.js +8 -2
- package/dist/core/artifacts/delivery-card.d.ts.map +1 -1
- package/dist/core/artifacts/delivery-card.js +1 -2
- package/dist/core/artifacts/index.d.ts.map +1 -1
- package/dist/core/artifacts/index.js +38 -12
- package/dist/core/events/v3/authority-outbox.d.ts.map +1 -1
- package/dist/core/events/v3/authority-outbox.js +0 -29
- package/dist/core/events/v3/producers/coordination-recorder.d.ts +0 -1
- package/dist/core/events/v3/producers/coordination-recorder.d.ts.map +1 -1
- package/dist/core/events/v3/producers/coordination-recorder.js +0 -7
- package/dist/core/events/v3/producers/recorder.d.ts.map +1 -1
- package/dist/core/events/v3/producers/recorder.js +64 -77
- package/dist/core/hooks/cli.js +78 -48
- package/dist/core/servers/index.d.ts +13 -10
- package/dist/core/servers/index.d.ts.map +1 -1
- package/dist/core/servers/index.js +97 -192
- package/dist/core/servers/tunnels.d.ts +3 -5
- package/dist/core/servers/tunnels.d.ts.map +1 -1
- package/dist/core/servers/tunnels.js +2 -45
- package/dist/core/storage/durable-history.js +56 -22
- package/dist/core/storage/segments.d.ts.map +1 -1
- package/dist/core/storage/segments.js +73 -17
- package/dist/core/work/state.js +7 -7
- package/dist/core/workflow/engine.d.ts.map +1 -1
- package/dist/core/workflow/engine.js +21 -65
- package/dist/core/workflow/live-session-v3.d.ts +0 -6
- package/dist/core/workflow/live-session-v3.d.ts.map +1 -1
- package/dist/core/workflow/live-session-v3.js +5 -126
- package/dist/core/workflow/proof.d.ts +4 -4
- package/dist/core/workflow/proof.d.ts.map +1 -1
- package/dist/core/workflow/proof.js +4 -12
- package/dist/core/workflow/spawn-claude.d.ts.map +1 -1
- package/dist/core/workflow/spawn-claude.js +0 -19
- package/dist/core/workflow/spawn-codex.d.ts.map +1 -1
- package/dist/core/workflow/spawn-codex.js +0 -13
- package/dist/core/workflow/spawn-cursor.d.ts.map +1 -1
- package/dist/core/workflow/spawn-cursor.js +0 -19
- package/dist/core/workflow/spawn-opencode.d.ts.map +1 -1
- package/dist/core/workflow/spawn-opencode.js +0 -14
- package/dist/core/workflow/types.d.ts +11 -54
- package/dist/core/workflow/types.d.ts.map +1 -1
- package/dist/lib/identities/assume.d.ts.map +1 -1
- package/dist/lib/identities/assume.js +0 -4
- package/dist/lib/tunnel/state.d.ts +6 -13
- package/dist/lib/tunnel/state.d.ts.map +1 -1
- package/dist/lib/tunnel/state.js +12 -58
- package/package.json +1 -6
- package/src/commander.ts +0 -10
- package/src/commands/files.ts +5 -13
- package/src/commands/servers.ts +23 -12
- package/src/commands/tunnel.ts +166 -533
- package/src/core/agents/live-authority-v3.ts +0 -17
- package/src/core/agents/live-lifecycle-v3.ts +14 -52
- package/src/core/agents/state/heartbeat-writer.ts +12 -29
- package/src/core/agents/state/live-coordination-writer.ts +0 -29
- package/src/core/agents/state/pidmap.ts +0 -13
- package/src/core/agents/state/stale-sweep.ts +82 -247
- package/src/core/artifacts/activity.ts +9 -2
- package/src/core/artifacts/delivery-card.ts +1 -7
- package/src/core/artifacts/index.ts +45 -12
- package/src/core/events/v3/authority-outbox.ts +0 -56
- package/src/core/events/v3/producers/coordination-recorder.ts +0 -15
- package/src/core/events/v3/producers/recorder.ts +68 -94
- package/src/core/hooks/cli.ts +85 -47
- package/src/core/servers/index.ts +106 -228
- package/src/core/servers/tunnels.ts +3 -55
- package/src/core/storage/durable-history.ts +52 -20
- package/src/core/storage/segments.ts +79 -17
- package/src/core/work/state.ts +7 -7
- package/src/core/workflow/engine.ts +21 -73
- package/src/core/workflow/live-session-v3.ts +5 -146
- package/src/core/workflow/proof.ts +5 -21
- package/src/core/workflow/spawn-claude.ts +0 -18
- package/src/core/workflow/spawn-codex.ts +0 -13
- package/src/core/workflow/spawn-cursor.ts +0 -18
- package/src/core/workflow/spawn-opencode.ts +0 -14
- package/src/core/workflow/types.ts +11 -41
- package/src/lib/identities/assume.ts +0 -4
- package/src/lib/tunnel/state.ts +15 -70
- package/dist/commands/turn-rollback.d.ts +0 -5
- package/dist/commands/turn-rollback.d.ts.map +0 -1
- package/dist/commands/turn-rollback.js +0 -76
- package/dist/core/agents/state/unchanged-file.d.ts +0 -44
- package/dist/core/agents/state/unchanged-file.d.ts.map +0 -1
- package/dist/core/agents/state/unchanged-file.js +0 -390
- package/dist/core/events/v3/generation-custody.d.ts +0 -5
- package/dist/core/events/v3/generation-custody.d.ts.map +0 -1
- package/dist/core/events/v3/generation-custody.js +0 -50
- package/dist/core/turn-rollback/content.d.ts +0 -53
- package/dist/core/turn-rollback/content.d.ts.map +0 -1
- package/dist/core/turn-rollback/content.js +0 -396
- package/dist/core/turn-rollback/index.d.ts +0 -72
- package/dist/core/turn-rollback/index.d.ts.map +0 -1
- package/dist/core/turn-rollback/index.js +0 -443
- package/dist/core/workflow/failure-evidence.d.ts +0 -16
- package/dist/core/workflow/failure-evidence.d.ts.map +0 -1
- package/dist/core/workflow/failure-evidence.js +0 -114
- package/src/commands/turn-rollback.ts +0 -111
- package/src/core/agents/state/unchanged-file.ts +0 -456
- package/src/core/events/v3/generation-custody.ts +0 -59
- package/src/core/turn-rollback/content.ts +0 -499
- package/src/core/turn-rollback/index.ts +0 -580
- package/src/core/workflow/failure-evidence.ts +0 -141
|
@@ -338,23 +338,20 @@ export async function withFamilyLease<T>(
|
|
|
338
338
|
let owner: LeaseOwner | undefined;
|
|
339
339
|
while (true) {
|
|
340
340
|
try {
|
|
341
|
-
|
|
342
|
-
const leaseIdentity = captureDirectoryIdentity(lease);
|
|
343
|
-
owner = {
|
|
344
|
-
owner_id: randomUUID(),
|
|
345
|
-
pid: process.pid,
|
|
346
|
-
acquired_at: new Date().toISOString(),
|
|
347
|
-
};
|
|
348
|
-
writeNewFile(join(lease, "owner.json"), Buffer.from(`${JSON.stringify(owner)}\n`));
|
|
349
|
-
assertDirectoryIdentity(leaseIdentity);
|
|
341
|
+
owner = claimLease(lease);
|
|
350
342
|
break;
|
|
351
343
|
} catch (error) {
|
|
352
|
-
if (
|
|
344
|
+
if (leaseHasWrongType(lease)) {
|
|
353
345
|
throw new HarneryLogLeaseError("wrong_type", `log lease is not a directory: ${lease}`);
|
|
354
346
|
}
|
|
355
347
|
if (recoverStaleLease(lease, policy.lease_stale_ms)) continue;
|
|
356
348
|
if (Date.now() - started >= policy.lease_timeout_ms) {
|
|
357
|
-
throw new HarneryLogLeaseError(
|
|
349
|
+
throw new HarneryLogLeaseError(
|
|
350
|
+
"timeout",
|
|
351
|
+
leaseHasValidOwner(lease)
|
|
352
|
+
? `timed out acquiring log lease: ${lease}`
|
|
353
|
+
: `timed out acquiring log lease with no valid owner; an interrupted writer left it, and it is never recovered automatically: ${lease}`,
|
|
354
|
+
);
|
|
358
355
|
}
|
|
359
356
|
await new Promise((resolve) => setTimeout(resolve, policy.lease_retry_ms));
|
|
360
357
|
void error;
|
|
@@ -367,6 +364,41 @@ export async function withFamilyLease<T>(
|
|
|
367
364
|
}
|
|
368
365
|
}
|
|
369
366
|
|
|
367
|
+
/**
|
|
368
|
+
* Publish a complete lease in one rename. The owner record is written and synced
|
|
369
|
+
* in a private claim directory first, so a writer stopped at any point leaves
|
|
370
|
+
* either no lease or one with an owner. A held lease always has an owner record,
|
|
371
|
+
* here and in the earlier create-then-write protocol, so a rename that replaces
|
|
372
|
+
* an empty lease directory never takes a held lease. A lease with any entry makes
|
|
373
|
+
* the rename fail, and the caller waits or recovers as before.
|
|
374
|
+
*/
|
|
375
|
+
function claimLease(lease: string): LeaseOwner {
|
|
376
|
+
const owner: LeaseOwner = {
|
|
377
|
+
owner_id: randomUUID(),
|
|
378
|
+
pid: process.pid,
|
|
379
|
+
acquired_at: new Date().toISOString(),
|
|
380
|
+
};
|
|
381
|
+
const content = `${JSON.stringify(owner)}\n`;
|
|
382
|
+
const claim = `${lease}.claim-${process.pid}-${owner.owner_id}`;
|
|
383
|
+
try {
|
|
384
|
+
mkdirSync(claim, { mode: stateDirMode() });
|
|
385
|
+
writeNewFile(join(claim, "owner.json"), Buffer.from(content));
|
|
386
|
+
renameSync(claim, lease);
|
|
387
|
+
} finally {
|
|
388
|
+
rmSync(claim, { recursive: true, force: true });
|
|
389
|
+
}
|
|
390
|
+
try {
|
|
391
|
+
captureDirectoryIdentity(lease);
|
|
392
|
+
if (readRegularFile(join(lease, "owner.json"), 4_096).toString("utf8") !== content) {
|
|
393
|
+
throw new Error(`log lease owner changed during claim: ${lease}`);
|
|
394
|
+
}
|
|
395
|
+
} catch (error) {
|
|
396
|
+
releaseOwnedLease(lease, owner);
|
|
397
|
+
throw error;
|
|
398
|
+
}
|
|
399
|
+
return owner;
|
|
400
|
+
}
|
|
401
|
+
|
|
370
402
|
function shouldRotate(
|
|
371
403
|
active: string,
|
|
372
404
|
incoming: readonly Buffer[],
|
|
@@ -476,6 +508,22 @@ interface LeaseOwner {
|
|
|
476
508
|
acquired_at: string;
|
|
477
509
|
}
|
|
478
510
|
|
|
511
|
+
function leaseHasValidOwner(lease: string): boolean {
|
|
512
|
+
try {
|
|
513
|
+
const owner = JSON.parse(
|
|
514
|
+
readRegularFile(join(lease, "owner.json"), 4_096).toString("utf8"),
|
|
515
|
+
) as LeaseOwner;
|
|
516
|
+
return (
|
|
517
|
+
typeof owner?.owner_id === "string" &&
|
|
518
|
+
owner.owner_id.length > 0 &&
|
|
519
|
+
Number.isSafeInteger(owner.pid) &&
|
|
520
|
+
Number.isFinite(Date.parse(owner.acquired_at))
|
|
521
|
+
);
|
|
522
|
+
} catch {
|
|
523
|
+
return false;
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
|
|
479
527
|
function recoverStaleLease(lease: string, maximumAgeMs: number): boolean {
|
|
480
528
|
let identity: DirectoryIdentity;
|
|
481
529
|
let raw: string;
|
|
@@ -504,7 +552,7 @@ function recoverStaleLease(lease: string, maximumAgeMs: number): boolean {
|
|
|
504
552
|
try {
|
|
505
553
|
assertDirectoryIdentity(identity);
|
|
506
554
|
if (readRegularFile(join(lease, "owner.json"), 4_096).toString("utf8") !== raw) return false;
|
|
507
|
-
|
|
555
|
+
retireLease(lease, randomUUID());
|
|
508
556
|
return true;
|
|
509
557
|
} catch {
|
|
510
558
|
return false;
|
|
@@ -517,12 +565,23 @@ function releaseOwnedLease(lease: string, owner: LeaseOwner): void {
|
|
|
517
565
|
readRegularFile(join(lease, "owner.json"), 4_096).toString("utf8"),
|
|
518
566
|
) as LeaseOwner;
|
|
519
567
|
if (current.owner_id !== owner.owner_id) return;
|
|
520
|
-
|
|
568
|
+
retireLease(lease, owner.owner_id);
|
|
521
569
|
} catch {
|
|
522
570
|
// A missing or replaced lease does not belong to this operation.
|
|
523
571
|
}
|
|
524
572
|
}
|
|
525
573
|
|
|
574
|
+
/**
|
|
575
|
+
* Move a lease aside before deleting it. Its path goes from held to absent in
|
|
576
|
+
* one rename and is never an empty directory that a concurrent claim could
|
|
577
|
+
* replace while a recursive removal is still running.
|
|
578
|
+
*/
|
|
579
|
+
function retireLease(lease: string, tag: string): void {
|
|
580
|
+
const retired = `${lease}.retired-${process.pid}-${tag}`;
|
|
581
|
+
renameSync(lease, retired);
|
|
582
|
+
rmSync(retired, { recursive: true, force: true });
|
|
583
|
+
}
|
|
584
|
+
|
|
526
585
|
function processIsAlive(pid: number): boolean {
|
|
527
586
|
try {
|
|
528
587
|
process.kill(pid, 0);
|
|
@@ -699,11 +758,14 @@ function nativeWriteSync(fd: number, buffer: Buffer, offset: number, length: num
|
|
|
699
758
|
return writeSync(fd, buffer, offset, length);
|
|
700
759
|
}
|
|
701
760
|
|
|
702
|
-
|
|
761
|
+
/** One lstat, so a lease released between checks reads as absent, not as a wrong type. */
|
|
762
|
+
function leaseHasWrongType(path: string): boolean {
|
|
703
763
|
try {
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
764
|
+
const stat = lstatSync(path);
|
|
765
|
+
return stat.isSymbolicLink() || !stat.isDirectory();
|
|
766
|
+
} catch (error) {
|
|
767
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return false;
|
|
768
|
+
throw error;
|
|
707
769
|
}
|
|
708
770
|
}
|
|
709
771
|
|
package/src/core/work/state.ts
CHANGED
|
@@ -20,7 +20,7 @@ import {
|
|
|
20
20
|
import { stateDirMode, stateFileMode } from "../storage/modes.ts";
|
|
21
21
|
import { readWorkflowApproval } from "../workflow/approvals.ts";
|
|
22
22
|
import { stableDigest } from "../workflow/durable-record.ts";
|
|
23
|
-
import {
|
|
23
|
+
import { readWorkflowProof } from "../workflow/proof.ts";
|
|
24
24
|
import { readWorkflowRunManifest, workflowScriptDigest } from "../workflow/run-state.ts";
|
|
25
25
|
import { WORKFLOW_TRANSCRIPT_EVENT_BYTES, workflowTranscriptPath } from "../workflow/transcript.ts";
|
|
26
26
|
import { isWorkspaceBoundExecutionEvidence } from "../workflow/workspaces/validate.ts";
|
|
@@ -718,14 +718,14 @@ function inspectAttempt(
|
|
|
718
718
|
proof.acceptance.summary.unknown === 0
|
|
719
719
|
? "succeeded"
|
|
720
720
|
: "failed";
|
|
721
|
-
//
|
|
722
|
-
//
|
|
723
|
-
//
|
|
721
|
+
// A failed attempt the run classified as uninformative about the work is
|
|
722
|
+
// uncharged (ADR 0046). Absent class ⇒ charged, as before. Read only for a
|
|
723
|
+
// failed attempt: a succeeded run never carries a class.
|
|
724
724
|
if (
|
|
725
725
|
attempt.status === "failed" &&
|
|
726
|
-
(proof.run.class === "
|
|
727
|
-
|
|
728
|
-
|
|
726
|
+
(proof.run.class === "environment" ||
|
|
727
|
+
proof.run.class === "upstream" ||
|
|
728
|
+
proof.run.class === "decision")
|
|
729
729
|
) {
|
|
730
730
|
attempt.uncharged = proof.run.class;
|
|
731
731
|
if (proof.run.class === "decision" && typeof proof.run.decision_id === "string") {
|
|
@@ -45,16 +45,7 @@ import { freezeWorkflowAttemptContext } from "./attempt-context.ts";
|
|
|
45
45
|
import { type BillingProbe, probeBilling } from "./billing.ts";
|
|
46
46
|
import { stableDigest } from "./durable-record.ts";
|
|
47
47
|
import { evidencePreflightError } from "./evidence-preflight.ts";
|
|
48
|
-
import {
|
|
49
|
-
canRetryFailedSpawn,
|
|
50
|
-
observeSpawnFailure,
|
|
51
|
-
resolveFailureEvidence,
|
|
52
|
-
} from "./failure-evidence.ts";
|
|
53
|
-
import {
|
|
54
|
-
endWorkflowChildSessionV3,
|
|
55
|
-
recordWorkflowChildTerminalFailureV3,
|
|
56
|
-
startWorkflowChildSessionV3,
|
|
57
|
-
} from "./live-session-v3.ts";
|
|
48
|
+
import { endWorkflowChildSessionV3, startWorkflowChildSessionV3 } from "./live-session-v3.ts";
|
|
58
49
|
import {
|
|
59
50
|
buildWorkflowProof,
|
|
60
51
|
createEvidenceRecord,
|
|
@@ -1038,38 +1029,22 @@ async function executeWorkflow(
|
|
|
1038
1029
|
});
|
|
1039
1030
|
agentProof.access = access;
|
|
1040
1031
|
|
|
1041
|
-
let childGenerationId: string | undefined;
|
|
1042
1032
|
let recordAgentEnd: (() => void) | undefined;
|
|
1043
1033
|
const finalizeChild = (): void => {
|
|
1044
1034
|
try {
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
// Result publication is the resume-cache boundary, after canonical custody.
|
|
1035
|
+
// Record the canonical terminal first. agent.end is the resume-cache
|
|
1036
|
+
// boundary, so a child whose end was not recorded never reports success.
|
|
1037
|
+
endWorkflowChildSessionV3({
|
|
1038
|
+
coordRoot: opts.coordRoot,
|
|
1039
|
+
instanceId: childInstanceId(runId, id),
|
|
1040
|
+
runId,
|
|
1041
|
+
agentId: id,
|
|
1042
|
+
adapter,
|
|
1043
|
+
cleanExit: agentProof.status === "succeeded",
|
|
1044
|
+
});
|
|
1056
1045
|
recordAgentEnd?.();
|
|
1057
1046
|
} catch (error) {
|
|
1058
|
-
|
|
1059
|
-
try {
|
|
1060
|
-
if (childGenerationId)
|
|
1061
|
-
recordWorkflowChildTerminalFailureV3({
|
|
1062
|
-
coordRoot: opts.coordRoot,
|
|
1063
|
-
instanceId: childInstanceId(runId, id),
|
|
1064
|
-
runId,
|
|
1065
|
-
agentId: id,
|
|
1066
|
-
adapter,
|
|
1067
|
-
expectedGenerationId: childGenerationId,
|
|
1068
|
-
reason,
|
|
1069
|
-
});
|
|
1070
|
-
} catch (visibilityError) {
|
|
1071
|
-
reason += `; blocked lifecycle recording failed: ${(visibilityError as Error).message}`;
|
|
1072
|
-
}
|
|
1047
|
+
const reason = `V3 terminal recording failed: ${(error as Error).message}`;
|
|
1073
1048
|
const failure = new Error(agentProof.error ? `${agentProof.error}; ${reason}` : reason, {
|
|
1074
1049
|
cause: error,
|
|
1075
1050
|
});
|
|
@@ -1102,7 +1077,7 @@ async function executeWorkflow(
|
|
|
1102
1077
|
// Headless children do not reliably fire adapter hooks. The engine owns
|
|
1103
1078
|
// their canonical V3 start so every dispatched child has an auditable
|
|
1104
1079
|
// generation before it begins work.
|
|
1105
|
-
|
|
1080
|
+
startWorkflowChildSessionV3({
|
|
1106
1081
|
coordRoot: opts.coordRoot,
|
|
1107
1082
|
instanceId: childInstanceId(runId, id),
|
|
1108
1083
|
runId,
|
|
@@ -1111,13 +1086,10 @@ async function executeWorkflow(
|
|
|
1111
1086
|
label,
|
|
1112
1087
|
model: agentOpts.model,
|
|
1113
1088
|
});
|
|
1114
|
-
if (!child.v3_generation_id) throw new Error("workflow_child_v3_generation_missing");
|
|
1115
|
-
childGenerationId = child.v3_generation_id;
|
|
1116
1089
|
|
|
1117
1090
|
let attemptPrompt = dispatchPrompt;
|
|
1118
1091
|
let last: SpawnResult | null = null;
|
|
1119
1092
|
let agentCostUsd = 0;
|
|
1120
|
-
let informativeAttempt: ReturnType<typeof observeSpawnFailure> | undefined;
|
|
1121
1093
|
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
1122
1094
|
last = await spawner({
|
|
1123
1095
|
prompt: attemptPrompt,
|
|
@@ -1140,30 +1112,14 @@ async function executeWorkflow(
|
|
|
1140
1112
|
costUsd += last.costUsd ?? 0;
|
|
1141
1113
|
|
|
1142
1114
|
if (!last.ok) {
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
effect: { state: "unknown" },
|
|
1151
|
-
partial_output: informativeAttempt.partial_output,
|
|
1152
|
-
}
|
|
1153
|
-
: observed;
|
|
1154
|
-
transcript("agent.attempt_failed", {
|
|
1155
|
-
id,
|
|
1156
|
-
attempt,
|
|
1157
|
-
error: last.error,
|
|
1158
|
-
failure_evidence: observed,
|
|
1159
|
-
result_digest: digestResult(last.text, "text"),
|
|
1160
|
-
text: last.text,
|
|
1161
|
-
});
|
|
1162
|
-
if (!canRetryFailedSpawn({ ...last, failureEvidence: agentProof.failure_evidence }))
|
|
1163
|
-
break;
|
|
1164
|
-
continue;
|
|
1115
|
+
transcript("agent.attempt_failed", { id, attempt, error: last.error });
|
|
1116
|
+
// ADR 0046: an environment failure (the binary was absent) cannot be
|
|
1117
|
+
// helped by retrying an unchanged environment, so stop the in-agent
|
|
1118
|
+
// retry too — not just the outer attempt/replan budget. An upstream
|
|
1119
|
+
// refusal keeps retrying here: the vendor may recover mid-loop.
|
|
1120
|
+
if (last.class === "environment") break;
|
|
1121
|
+
continue; // spawn-level failure: retry with the original prompt
|
|
1165
1122
|
}
|
|
1166
|
-
informativeAttempt = observeSpawnFailure({}, last.text, "final_message");
|
|
1167
1123
|
if (!agentOpts.schema) {
|
|
1168
1124
|
agentProof.status = "succeeded";
|
|
1169
1125
|
agentProof.cost_usd =
|
|
@@ -1217,14 +1173,7 @@ async function executeWorkflow(
|
|
|
1217
1173
|
return parsed.value;
|
|
1218
1174
|
}
|
|
1219
1175
|
|
|
1220
|
-
|
|
1221
|
-
transcript("agent.schema_retry", {
|
|
1222
|
-
id,
|
|
1223
|
-
attempt,
|
|
1224
|
-
problems,
|
|
1225
|
-
rejected_text: last.text,
|
|
1226
|
-
result_digest: digestResult(last.text, "text"),
|
|
1227
|
-
});
|
|
1176
|
+
transcript("agent.schema_retry", { id, attempt, problems });
|
|
1228
1177
|
// Keep the original cache identity while making the actual output
|
|
1229
1178
|
// contract explicit. A full bounded copy of the rejected value lets the
|
|
1230
1179
|
// child repair omissions instead of reconstructing an unseen object.
|
|
@@ -1260,7 +1209,6 @@ async function executeWorkflow(
|
|
|
1260
1209
|
try {
|
|
1261
1210
|
finalizeChild();
|
|
1262
1211
|
} finally {
|
|
1263
|
-
// A slot is disposable; unresolved custody remains visible and failed.
|
|
1264
1212
|
release();
|
|
1265
1213
|
}
|
|
1266
1214
|
}
|
|
@@ -1,16 +1,9 @@
|
|
|
1
|
-
import { randomUUID } from "node:crypto";
|
|
2
|
-
import { existsSync, linkSync, readdirSync, readFileSync, renameSync, unlinkSync } from "node:fs";
|
|
3
|
-
import { basename, dirname } from "node:path";
|
|
4
1
|
import { type Adapter, adapterFromPlatform } from "../adapter.ts";
|
|
5
|
-
import {
|
|
6
|
-
recordLiveLifecycleChangeV3,
|
|
7
|
-
recordLiveTaskChangeV3,
|
|
8
|
-
} from "../agents/live-authority-v3.ts";
|
|
2
|
+
import { recordLiveTaskChangeV3 } from "../agents/live-authority-v3.ts";
|
|
9
3
|
import { endSessionExplicitV3 } from "../agents/session-finalizer-v3.ts";
|
|
10
|
-
import { heartbeatPath } from "../agents/state/heartbeat-reader.ts";
|
|
11
4
|
import {
|
|
5
|
+
clearCoordinationCache,
|
|
12
6
|
type Heartbeat,
|
|
13
|
-
readHeartbeat,
|
|
14
7
|
setIdentityCache,
|
|
15
8
|
} from "../agents/state/heartbeat-writer.ts";
|
|
16
9
|
import { readLiveCoordinationRow } from "../agents/state/live-coordination-view.ts";
|
|
@@ -19,17 +12,9 @@ import { assignName } from "../agents/state/names.ts";
|
|
|
19
12
|
import { coordFreshnessSeconds } from "../config.ts";
|
|
20
13
|
import { ensureEventLedgerV3 } from "../events/v3/bootstrap.ts";
|
|
21
14
|
import {
|
|
22
|
-
readCoordinationViewV3,
|
|
23
|
-
requireAuthoritySafeCoordinationViewV3,
|
|
24
|
-
} from "../events/v3/coordination-view.ts";
|
|
25
|
-
import { withGenerationCustodyV3 } from "../events/v3/generation-custody.ts";
|
|
26
|
-
import {
|
|
27
|
-
liveInstanceIdV3,
|
|
28
15
|
recordLiveHookSignalV3,
|
|
29
16
|
resolveLiveEventLedgerRouteV3,
|
|
30
17
|
} from "../events/v3/live-routing.ts";
|
|
31
|
-
import { readHookProducerStateV3 } from "../events/v3/producers/recorder.ts";
|
|
32
|
-
import { fsyncParentDirectory } from "./durable-record.ts";
|
|
33
18
|
import { stableScopeId } from "./scope-id.ts";
|
|
34
19
|
|
|
35
20
|
export interface WorkflowChildSessionV3Input {
|
|
@@ -92,45 +77,16 @@ export function startWorkflowChildSessionV3(input: WorkflowChildSessionV3Input):
|
|
|
92
77
|
}
|
|
93
78
|
const projected = readLiveCoordinationRow(input.coordRoot, input.instanceId);
|
|
94
79
|
if (!projected) throw new Error("workflow_child_v3_projection_missing");
|
|
95
|
-
const own = readHookProducerStateV3(input.coordRoot, adapter, sessionId);
|
|
96
|
-
if (!own || own.terminal || own.generation_id !== projected.v3_generation_id) {
|
|
97
|
-
throw new Error("workflow_child_v3_start_generation_changed");
|
|
98
|
-
}
|
|
99
80
|
return projected;
|
|
100
81
|
}
|
|
101
82
|
|
|
102
83
|
/** Record the authoritative terminal before removing the disposable coordination cache. */
|
|
103
84
|
export function endWorkflowChildSessionV3(
|
|
104
|
-
input: WorkflowChildSessionV3Input & { cleanExit: boolean
|
|
105
|
-
): void {
|
|
106
|
-
withGenerationCustodyV3(input.coordRoot, [liveInstanceIdV3(input.instanceId)], () =>
|
|
107
|
-
endExpectedChild(input),
|
|
108
|
-
);
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
function endExpectedChild(
|
|
112
|
-
input: WorkflowChildSessionV3Input & { cleanExit: boolean; expectedGenerationId: string },
|
|
85
|
+
input: WorkflowChildSessionV3Input & { cleanExit: boolean },
|
|
113
86
|
): void {
|
|
114
87
|
requireV3Route(input.coordRoot);
|
|
115
|
-
const view = requireAuthoritySafeCoordinationViewV3(readCoordinationViewV3(input.coordRoot));
|
|
116
|
-
const instanceId = liveInstanceIdV3(input.instanceId);
|
|
117
|
-
const current = view.instances[instanceId];
|
|
118
|
-
const terminal = view.terminal_generations[input.expectedGenerationId];
|
|
119
|
-
if (terminal?.instance_id === instanceId) recoverHeldCaches(input);
|
|
120
|
-
if (current && current.generation_id !== input.expectedGenerationId) {
|
|
121
|
-
throw new Error("workflow_child_v3_generation_changed");
|
|
122
|
-
}
|
|
123
|
-
if (!current) {
|
|
124
|
-
if (!terminal || terminal.instance_id !== instanceId)
|
|
125
|
-
throw new Error("workflow_child_v3_generation_missing");
|
|
126
|
-
if (terminal.terminal?.outcome !== (input.cleanExit ? "succeeded" : "failed")) {
|
|
127
|
-
throw new Error("workflow_child_v3_terminal_outcome_changed");
|
|
128
|
-
}
|
|
129
|
-
clearExpectedCache(input);
|
|
130
|
-
return;
|
|
131
|
-
}
|
|
132
88
|
const row = readLiveCoordinationRow(input.coordRoot, input.instanceId);
|
|
133
|
-
if (!row?.v3_instance_id || row.v3_generation_id
|
|
89
|
+
if (!row?.v3_instance_id || !row.v3_generation_id) {
|
|
134
90
|
throw new Error("workflow_child_v3_generation_missing");
|
|
135
91
|
}
|
|
136
92
|
const result = endSessionExplicitV3({
|
|
@@ -145,104 +101,7 @@ function endExpectedChild(
|
|
|
145
101
|
`workflow_child_v3_end_failed:${result.state}${"reason" in result ? `:${result.reason}` : ""}`,
|
|
146
102
|
);
|
|
147
103
|
}
|
|
148
|
-
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
/** Persist unresolved terminal custody only for the generation this dispatch owns. */
|
|
152
|
-
export function recordWorkflowChildTerminalFailureV3(
|
|
153
|
-
input: WorkflowChildSessionV3Input & { expectedGenerationId: string; reason: string },
|
|
154
|
-
): void {
|
|
155
|
-
withGenerationCustodyV3(input.coordRoot, [liveInstanceIdV3(input.instanceId)], () => {
|
|
156
|
-
const row = readLiveCoordinationRow(input.coordRoot, input.instanceId);
|
|
157
|
-
if (!row || row.v3_generation_id !== input.expectedGenerationId) return;
|
|
158
|
-
recordLiveLifecycleChangeV3({
|
|
159
|
-
coordRoot: input.coordRoot,
|
|
160
|
-
owner: input.instanceId,
|
|
161
|
-
nativeSessionId: input.sessionId ?? input.instanceId,
|
|
162
|
-
adapter: workflowAdapter(input.adapter),
|
|
163
|
-
expectedGenerationId: input.expectedGenerationId as `gen_${string}`,
|
|
164
|
-
state: "blocked",
|
|
165
|
-
reason: input.reason,
|
|
166
|
-
});
|
|
167
|
-
});
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
function heldPrefix(input: WorkflowChildSessionV3Input & { expectedGenerationId: string }): string {
|
|
171
|
-
if (!/^gen_[a-zA-Z0-9._-]{1,128}$/.test(input.expectedGenerationId))
|
|
172
|
-
throw new Error("workflow_child_v3_generation_invalid");
|
|
173
|
-
return `${basename(heartbeatPath(input.coordRoot, input.instanceId))}.terminal-${input.expectedGenerationId}-`;
|
|
174
|
-
}
|
|
175
|
-
|
|
176
|
-
function recoverHeldCaches(
|
|
177
|
-
input: WorkflowChildSessionV3Input & { expectedGenerationId: string },
|
|
178
|
-
): void {
|
|
179
|
-
const path = heartbeatPath(input.coordRoot, input.instanceId);
|
|
180
|
-
if (!existsSync(dirname(path))) return;
|
|
181
|
-
const prefix = heldPrefix(input);
|
|
182
|
-
for (const name of readdirSync(dirname(path)).filter((value) => value.startsWith(prefix))) {
|
|
183
|
-
finishHeldCache(`${dirname(path)}/${name}`, path, input.expectedGenerationId);
|
|
184
|
-
}
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
function clearExpectedCache(
|
|
188
|
-
input: WorkflowChildSessionV3Input & { expectedGenerationId: string },
|
|
189
|
-
): void {
|
|
190
|
-
recoverHeldCaches(input);
|
|
191
|
-
if (
|
|
192
|
-
readHeartbeat(input.coordRoot, input.instanceId)?.v3_generation_id !==
|
|
193
|
-
input.expectedGenerationId
|
|
194
|
-
)
|
|
195
|
-
return;
|
|
196
|
-
const path = heartbeatPath(input.coordRoot, input.instanceId);
|
|
197
|
-
const held = `${dirname(path)}/${heldPrefix(input)}${randomUUID()}`;
|
|
198
|
-
try {
|
|
199
|
-
// A unique held name pins the actual bytes atomically. Checking before
|
|
200
|
-
// unlink alone could consume a replacement published at this boundary.
|
|
201
|
-
renameSync(path, held);
|
|
202
|
-
fsyncParentDirectory(path);
|
|
203
|
-
} catch (error) {
|
|
204
|
-
if ((error as NodeJS.ErrnoException).code === "ENOENT") return;
|
|
205
|
-
throw error;
|
|
206
|
-
}
|
|
207
|
-
finishHeldCache(held, path, input.expectedGenerationId);
|
|
208
|
-
}
|
|
209
|
-
|
|
210
|
-
function finishHeldCache(held: string, path: string, expectedGenerationId: string): void {
|
|
211
|
-
let removed = false;
|
|
212
|
-
let cleanupFailed = false;
|
|
213
|
-
let cleanupError: unknown;
|
|
214
|
-
try {
|
|
215
|
-
const cache = JSON.parse(readFileSync(held, "utf8")) as Heartbeat;
|
|
216
|
-
if (cache.v3_generation_id === expectedGenerationId) {
|
|
217
|
-
unlinkSync(held);
|
|
218
|
-
fsyncParentDirectory(held);
|
|
219
|
-
removed = true;
|
|
220
|
-
}
|
|
221
|
-
} catch (error) {
|
|
222
|
-
cleanupFailed = true;
|
|
223
|
-
cleanupError = error;
|
|
224
|
-
}
|
|
225
|
-
if (!removed) {
|
|
226
|
-
try {
|
|
227
|
-
// Never overwrite a later publication. Keep the held evidence when
|
|
228
|
-
// another cache already owns the live name.
|
|
229
|
-
linkSync(held, path);
|
|
230
|
-
fsyncParentDirectory(path);
|
|
231
|
-
unlinkSync(held);
|
|
232
|
-
fsyncParentDirectory(held);
|
|
233
|
-
} catch (error) {
|
|
234
|
-
if ((error as NodeJS.ErrnoException).code !== "EEXIST") {
|
|
235
|
-
if (cleanupFailed) {
|
|
236
|
-
throw new AggregateError(
|
|
237
|
-
[cleanupError, error],
|
|
238
|
-
`workflow_child_v3_cache_cleanup_and_restore_failed: ${String(cleanupError)}; restoration: ${String(error)}`,
|
|
239
|
-
);
|
|
240
|
-
}
|
|
241
|
-
throw error;
|
|
242
|
-
}
|
|
243
|
-
}
|
|
244
|
-
}
|
|
245
|
-
if (cleanupFailed) throw cleanupError;
|
|
104
|
+
clearCoordinationCache(input.coordRoot, input.instanceId);
|
|
246
105
|
}
|
|
247
106
|
|
|
248
107
|
function requireV3Route(coordRoot: string) {
|
|
@@ -12,7 +12,6 @@ import type {
|
|
|
12
12
|
import { policyDigest } from "../policy/index.ts";
|
|
13
13
|
import { isCanonicalWorkflowAttemptContext } from "./attempt-context.ts";
|
|
14
14
|
import { stableDigest, writeImmutableJson } from "./durable-record.ts";
|
|
15
|
-
import { failureEvidenceAllowsUncharged, isSpawnFailureEvidence } from "./failure-evidence.ts";
|
|
16
15
|
import { readWorkflowRunManifest } from "./run-state.ts";
|
|
17
16
|
import type {
|
|
18
17
|
AcceptanceCriterion,
|
|
@@ -243,25 +242,17 @@ export function digestResult(value: unknown, kind?: "text" | "json"): ResultDige
|
|
|
243
242
|
* attempt was informative about the work — charge it. This also keeps a
|
|
244
243
|
* resumed run whose earlier segment did real work from being written off by
|
|
245
244
|
* a later environment failure.
|
|
246
|
-
* 3.
|
|
247
|
-
*
|
|
248
|
-
*
|
|
245
|
+
* 3. Otherwise, among the failed agents, environment wins over upstream: a
|
|
246
|
+
* missing binary means nothing ran at all, and it is the operator-chosen
|
|
247
|
+
* hard stop.
|
|
249
248
|
* 4. Anything else is undefined ⇒ a charged work failure, exactly as today.
|
|
250
249
|
*/
|
|
251
250
|
export function deriveRunFailureClass(
|
|
252
251
|
status: "succeeded" | "failed",
|
|
253
|
-
agents: readonly Pick<WorkflowAgentProof, "status" | "class"
|
|
252
|
+
agents: readonly Pick<WorkflowAgentProof, "status" | "class">[],
|
|
254
253
|
): SpawnFailureClass | undefined {
|
|
255
254
|
if (status !== "failed") return undefined;
|
|
256
|
-
if (
|
|
257
|
-
agents.length === 0 ||
|
|
258
|
-
agents.some(
|
|
259
|
-
(agent) =>
|
|
260
|
-
agent.status !== "failed" ||
|
|
261
|
-
(agent.class !== "environment" && agent.class !== "upstream") ||
|
|
262
|
-
!failureEvidenceAllowsUncharged(agent.failure_evidence),
|
|
263
|
-
)
|
|
264
|
-
) {
|
|
255
|
+
if (agents.some((agent) => agent.status === "succeeded" || agent.status === "cached")) {
|
|
265
256
|
return undefined;
|
|
266
257
|
}
|
|
267
258
|
const failed = agents.filter((agent) => agent.status === "failed");
|
|
@@ -394,13 +385,6 @@ export function readWorkflowProof(coordRoot: string, runId: string): WorkflowPro
|
|
|
394
385
|
proof.run.class !== "environment" &&
|
|
395
386
|
proof.run.class !== "upstream" &&
|
|
396
387
|
proof.run.class !== "decision") ||
|
|
397
|
-
!Array.isArray(proof.agents) ||
|
|
398
|
-
proof.agents.some(
|
|
399
|
-
(agent) =>
|
|
400
|
-
!agent ||
|
|
401
|
-
typeof agent !== "object" ||
|
|
402
|
-
(agent.failure_evidence !== undefined && !isSpawnFailureEvidence(agent.failure_evidence)),
|
|
403
|
-
) ||
|
|
404
388
|
(proof.run.work_context !== undefined &&
|
|
405
389
|
(!proof.run.work_item_id ||
|
|
406
390
|
proof.run.work_context.id !== proof.run.work_item_id ||
|
|
@@ -23,7 +23,6 @@ import { builtinAdapterProfile, validateAdapterEffort } from "../adapters/profil
|
|
|
23
23
|
import type { AdapterInvocation, AdapterRawResult } from "../adapters/types.ts";
|
|
24
24
|
import { notFoundError } from "./adapters.ts";
|
|
25
25
|
import { buildChildEnv } from "./child-env.ts";
|
|
26
|
-
import { observeSpawnFailure } from "./failure-evidence.ts";
|
|
27
26
|
import { resolveFullAccessArgv, resolveSandboxProjection } from "./sandbox-projection.ts";
|
|
28
27
|
import { isUpstreamFailureText, vendorFailureText } from "./spawn-failure.ts";
|
|
29
28
|
import type { Spawner, SpawnRequest, SpawnResult } from "./types.ts";
|
|
@@ -67,23 +66,6 @@ export function buildClaudeInvocation(req: SpawnRequest): AdapterInvocation {
|
|
|
67
66
|
}
|
|
68
67
|
|
|
69
68
|
export function normalizeClaudeResult(raw: AdapterRawResult): SpawnResult {
|
|
70
|
-
const result = normalizeClaudeTransportResult(raw);
|
|
71
|
-
if (result.ok) return result;
|
|
72
|
-
let text = raw.stdout.trim();
|
|
73
|
-
let source: "stdout" | "vendor_envelope" = "stdout";
|
|
74
|
-
try {
|
|
75
|
-
const envelope = JSON.parse(raw.stdout) as ClaudeEnvelope;
|
|
76
|
-
if (typeof envelope.result === "string" && envelope.result.length > 0) {
|
|
77
|
-
text = envelope.result;
|
|
78
|
-
source = "vendor_envelope";
|
|
79
|
-
}
|
|
80
|
-
} catch {
|
|
81
|
-
// Retain unstructured output without treating it as native authority.
|
|
82
|
-
}
|
|
83
|
-
return { ...result, text, failureEvidence: observeSpawnFailure(raw, text, source) };
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
function normalizeClaudeTransportResult(raw: AdapterRawResult): SpawnResult {
|
|
87
69
|
if (raw.timedOut) {
|
|
88
70
|
return {
|
|
89
71
|
ok: false,
|
|
@@ -26,7 +26,6 @@ import { builtinAdapterProfile, validateAdapterEffort } from "../adapters/profil
|
|
|
26
26
|
import type { AdapterInvocation, AdapterRawResult } from "../adapters/types.ts";
|
|
27
27
|
import { notFoundError } from "./adapters.ts";
|
|
28
28
|
import { buildChildEnv } from "./child-env.ts";
|
|
29
|
-
import { observeSpawnFailure } from "./failure-evidence.ts";
|
|
30
29
|
import { resolveFullAccessArgv, resolveSandboxProjection } from "./sandbox-projection.ts";
|
|
31
30
|
import { isUpstreamFailureText, vendorFailureText } from "./spawn-failure.ts";
|
|
32
31
|
import type { Spawner, SpawnRequest, SpawnResult } from "./types.ts";
|
|
@@ -79,18 +78,6 @@ export function buildCodexInvocation(req: SpawnRequest, resultFile?: string): Ad
|
|
|
79
78
|
}
|
|
80
79
|
|
|
81
80
|
export function normalizeCodexResult(raw: AdapterRawResult): SpawnResult {
|
|
82
|
-
const result = normalizeCodexTransportResult(raw);
|
|
83
|
-
if (result.ok) return result;
|
|
84
|
-
const finalText = raw.resultFileText?.trim();
|
|
85
|
-
const text = finalText || raw.stdout.trim();
|
|
86
|
-
return {
|
|
87
|
-
...result,
|
|
88
|
-
text,
|
|
89
|
-
failureEvidence: observeSpawnFailure(raw, text, finalText ? "final_message" : "stdout"),
|
|
90
|
-
};
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
function normalizeCodexTransportResult(raw: AdapterRawResult): SpawnResult {
|
|
94
81
|
if (raw.timedOut) {
|
|
95
82
|
return {
|
|
96
83
|
ok: false,
|
|
@@ -24,7 +24,6 @@ import { builtinAdapterProfile, validateAdapterEffort } from "../adapters/profil
|
|
|
24
24
|
import type { AdapterInvocation, AdapterRawResult } from "../adapters/types.ts";
|
|
25
25
|
import { notFoundError } from "./adapters.ts";
|
|
26
26
|
import { buildChildEnv } from "./child-env.ts";
|
|
27
|
-
import { observeSpawnFailure } from "./failure-evidence.ts";
|
|
28
27
|
import { resolveFullAccessArgv, resolveSandboxProjection } from "./sandbox-projection.ts";
|
|
29
28
|
import { isUpstreamFailureText, vendorFailureText } from "./spawn-failure.ts";
|
|
30
29
|
import type { Spawner, SpawnRequest, SpawnResult } from "./types.ts";
|
|
@@ -79,23 +78,6 @@ export function buildCursorInvocation(req: SpawnRequest): AdapterInvocation {
|
|
|
79
78
|
}
|
|
80
79
|
|
|
81
80
|
export function normalizeCursorResult(raw: AdapterRawResult): SpawnResult {
|
|
82
|
-
const result = normalizeCursorTransportResult(raw);
|
|
83
|
-
if (result.ok) return result;
|
|
84
|
-
let text = raw.stdout;
|
|
85
|
-
let source: "stdout" | "vendor_envelope" = "stdout";
|
|
86
|
-
try {
|
|
87
|
-
const envelope = JSON.parse(raw.stdout) as CursorEnvelope;
|
|
88
|
-
if (typeof envelope.result === "string" && envelope.result.length > 0) {
|
|
89
|
-
text = envelope.result;
|
|
90
|
-
source = "vendor_envelope";
|
|
91
|
-
}
|
|
92
|
-
} catch {
|
|
93
|
-
// Raw output remains evidence, not an inferred refusal or effect result.
|
|
94
|
-
}
|
|
95
|
-
return { ...result, text, failureEvidence: observeSpawnFailure(raw, text, source) };
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
function normalizeCursorTransportResult(raw: AdapterRawResult): SpawnResult {
|
|
99
81
|
if (raw.timedOut) {
|
|
100
82
|
return {
|
|
101
83
|
ok: false,
|