harnery 0.44.0 → 0.45.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/workflow.d.ts.map +1 -1
- package/dist/commands/workflow.js +44 -0
- package/dist/core/agents/coord-client.d.ts.map +1 -1
- package/dist/core/agents/coord-client.js +18 -0
- package/dist/core/agents/live-authority-v3.d.ts +0 -1
- package/dist/core/agents/live-authority-v3.d.ts.map +1 -1
- package/dist/core/agents/live-authority-v3.js +0 -11
- package/dist/core/agents/live-lifecycle-v3.d.ts +2 -4
- package/dist/core/agents/live-lifecycle-v3.d.ts.map +1 -1
- package/dist/core/agents/live-lifecycle-v3.js +7 -28
- package/dist/core/agents/state/heartbeat-writer.d.ts.map +1 -1
- package/dist/core/agents/state/heartbeat-writer.js +14 -25
- package/dist/core/agents/state/live-coordination-writer.d.ts.map +1 -1
- package/dist/core/agents/state/live-coordination-writer.js +0 -5
- package/dist/core/agents/state/pidmap.d.ts.map +1 -1
- package/dist/core/agents/state/pidmap.js +0 -5
- package/dist/core/agents/state/proc-start.d.ts +5 -6
- package/dist/core/agents/state/proc-start.d.ts.map +1 -1
- package/dist/core/agents/state/proc-start.js +13 -9
- package/dist/core/agents/state/process-handle.d.ts +26 -0
- package/dist/core/agents/state/process-handle.d.ts.map +1 -0
- package/dist/core/agents/state/process-handle.js +121 -0
- package/dist/core/agents/state/stale-sweep.d.ts.map +1 -1
- package/dist/core/agents/state/stale-sweep.js +70 -180
- package/dist/core/artifacts/activity.d.ts +4 -1
- package/dist/core/artifacts/activity.d.ts.map +1 -1
- package/dist/core/artifacts/activity.js +8 -2
- package/dist/core/artifacts/index.d.ts.map +1 -1
- package/dist/core/artifacts/index.js +38 -12
- package/dist/core/events/v3/authority-outbox.d.ts.map +1 -1
- package/dist/core/events/v3/authority-outbox.js +0 -29
- package/dist/core/events/v3/producers/coordination-recorder.d.ts +0 -1
- package/dist/core/events/v3/producers/coordination-recorder.d.ts.map +1 -1
- package/dist/core/events/v3/producers/coordination-recorder.js +0 -7
- package/dist/core/events/v3/producers/recorder.d.ts.map +1 -1
- package/dist/core/events/v3/producers/recorder.js +64 -77
- package/dist/core/hooks/cli.js +78 -48
- package/dist/core/resources/service.d.ts +2 -16
- package/dist/core/resources/service.d.ts.map +1 -1
- package/dist/core/resources/service.js +1 -78
- package/dist/core/semantic/service.d.ts.map +1 -1
- package/dist/core/semantic/service.js +2 -9
- package/dist/core/servers/index.d.ts +4 -0
- package/dist/core/servers/index.d.ts.map +1 -1
- package/dist/core/servers/index.js +21 -3
- package/dist/core/storage/durable-history.js +56 -22
- package/dist/core/storage/segments.d.ts.map +1 -1
- package/dist/core/storage/segments.js +73 -17
- package/dist/core/supervisor/service.d.ts +2 -4
- package/dist/core/supervisor/service.d.ts.map +1 -1
- package/dist/core/work/state.js +7 -7
- package/dist/core/workflow/engine.d.ts.map +1 -1
- package/dist/core/workflow/engine.js +21 -65
- package/dist/core/workflow/index.d.ts +4 -2
- package/dist/core/workflow/index.d.ts.map +1 -1
- package/dist/core/workflow/index.js +2 -1
- package/dist/core/workflow/live-session-v3.d.ts +0 -6
- package/dist/core/workflow/live-session-v3.d.ts.map +1 -1
- package/dist/core/workflow/live-session-v3.js +5 -126
- package/dist/core/workflow/proof.d.ts +4 -4
- package/dist/core/workflow/proof.d.ts.map +1 -1
- package/dist/core/workflow/proof.js +4 -12
- package/dist/core/workflow/recovery-audit.d.ts +30 -0
- package/dist/core/workflow/recovery-audit.d.ts.map +1 -0
- package/dist/core/workflow/recovery-audit.js +116 -0
- package/dist/core/workflow/run-state.d.ts +19 -2
- package/dist/core/workflow/run-state.d.ts.map +1 -1
- package/dist/core/workflow/run-state.js +113 -8
- package/dist/core/workflow/spawn-claude.d.ts.map +1 -1
- package/dist/core/workflow/spawn-claude.js +0 -19
- package/dist/core/workflow/spawn-codex.d.ts.map +1 -1
- package/dist/core/workflow/spawn-codex.js +0 -13
- package/dist/core/workflow/spawn-cursor.d.ts.map +1 -1
- package/dist/core/workflow/spawn-cursor.js +0 -19
- package/dist/core/workflow/spawn-opencode.d.ts.map +1 -1
- package/dist/core/workflow/spawn-opencode.js +0 -14
- package/dist/core/workflow/types.d.ts +11 -54
- package/dist/core/workflow/types.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/local-git.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/local-git.js +70 -14
- package/dist/core/workflow/workspaces/reclaim.d.ts +6 -1
- package/dist/core/workflow/workspaces/reclaim.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/recovery.d.ts +2 -0
- package/dist/core/workflow/workspaces/recovery.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/recovery.js +24 -0
- package/dist/lib/identities/assume.d.ts.map +1 -1
- package/dist/lib/identities/assume.js +0 -4
- package/package.json +1 -1
- package/src/commands/workflow.ts +59 -0
- package/src/core/agents/coord-client.ts +20 -0
- package/src/core/agents/live-authority-v3.ts +0 -17
- package/src/core/agents/live-lifecycle-v3.ts +14 -52
- package/src/core/agents/state/heartbeat-writer.ts +12 -29
- package/src/core/agents/state/live-coordination-writer.ts +0 -29
- package/src/core/agents/state/pidmap.ts +0 -13
- package/src/core/agents/state/proc-start.ts +14 -11
- package/src/core/agents/state/process-handle.ts +165 -0
- package/src/core/agents/state/stale-sweep.ts +82 -247
- package/src/core/artifacts/activity.ts +9 -2
- package/src/core/artifacts/index.ts +45 -12
- package/src/core/events/v3/authority-outbox.ts +0 -56
- package/src/core/events/v3/producers/coordination-recorder.ts +0 -15
- package/src/core/events/v3/producers/recorder.ts +68 -94
- package/src/core/hooks/cli.ts +85 -47
- package/src/core/resources/service.ts +3 -88
- package/src/core/semantic/service.ts +2 -8
- package/src/core/servers/index.ts +30 -3
- package/src/core/storage/durable-history.ts +52 -20
- package/src/core/storage/segments.ts +79 -17
- package/src/core/supervisor/service.ts +2 -3
- package/src/core/work/state.ts +7 -7
- package/src/core/workflow/engine.ts +21 -73
- package/src/core/workflow/index.ts +12 -1
- package/src/core/workflow/live-session-v3.ts +5 -146
- package/src/core/workflow/proof.ts +5 -21
- package/src/core/workflow/recovery-audit.ts +181 -0
- package/src/core/workflow/run-state.ts +137 -8
- package/src/core/workflow/spawn-claude.ts +0 -18
- package/src/core/workflow/spawn-codex.ts +0 -13
- package/src/core/workflow/spawn-cursor.ts +0 -18
- package/src/core/workflow/spawn-opencode.ts +0 -14
- package/src/core/workflow/types.ts +11 -41
- package/src/core/workflow/workspaces/local-git.ts +86 -15
- package/src/core/workflow/workspaces/reclaim.ts +4 -0
- package/src/core/workflow/workspaces/recovery.ts +23 -0
- package/src/lib/identities/assume.ts +0 -4
- package/dist/core/agents/state/unchanged-file.d.ts +0 -44
- package/dist/core/agents/state/unchanged-file.d.ts.map +0 -1
- package/dist/core/agents/state/unchanged-file.js +0 -390
- package/dist/core/events/v3/generation-custody.d.ts +0 -5
- package/dist/core/events/v3/generation-custody.d.ts.map +0 -1
- package/dist/core/events/v3/generation-custody.js +0 -50
- package/dist/core/workflow/failure-evidence.d.ts +0 -16
- package/dist/core/workflow/failure-evidence.d.ts.map +0 -1
- package/dist/core/workflow/failure-evidence.js +0 -114
- package/src/core/agents/state/unchanged-file.ts +0 -456
- package/src/core/events/v3/generation-custody.ts +0 -59
- package/src/core/workflow/failure-evidence.ts +0 -141
|
@@ -521,28 +521,49 @@ function acquireLease(lease: string): LeaseOwner {
|
|
|
521
521
|
pid: process.pid,
|
|
522
522
|
acquired_at: new Date().toISOString(),
|
|
523
523
|
};
|
|
524
|
-
|
|
524
|
+
if (claimLease(lease, parent, owner)) return owner;
|
|
525
|
+
if (recoverStaleLease(lease, DURABLE_HISTORY_LEASE_STALE_MS)) continue;
|
|
526
|
+
throw new Error(`durable history lease busy: ${lease}`);
|
|
527
|
+
}
|
|
528
|
+
throw new Error(`durable history lease busy: ${lease}`);
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
/**
|
|
532
|
+
* Publish a complete lease in one rename. The owner record is written and synced
|
|
533
|
+
* in a private claim directory first, so a writer stopped at any point leaves
|
|
534
|
+
* either no lease or one with an owner. A held lease always has an owner record,
|
|
535
|
+
* here and in the earlier create-then-write protocol, so a rename that replaces
|
|
536
|
+
* an empty lease directory never takes a held lease. A lease with any entry makes
|
|
537
|
+
* the rename fail and the caller reports it busy.
|
|
538
|
+
*/
|
|
539
|
+
function claimLease(lease: string, parent: DirectoryIdentity, owner: LeaseOwner): boolean {
|
|
540
|
+
const claim = `${lease}.claim-${process.pid}-${owner.owner_id}`;
|
|
541
|
+
try {
|
|
542
|
+
mkdirSync(claim, { mode: stateDirMode() });
|
|
543
|
+
assertDirectoryWithinParent(claim, parent);
|
|
544
|
+
const claimed = captureDirectoryIdentity(claim);
|
|
545
|
+
writeOwner(join(claim, "owner.json"), owner);
|
|
546
|
+
syncDirectory(claimed);
|
|
547
|
+
renameSync(claim, lease);
|
|
548
|
+
} catch {
|
|
549
|
+
return false;
|
|
550
|
+
} finally {
|
|
525
551
|
try {
|
|
526
|
-
|
|
527
|
-
assertDirectoryWithinParent(lease, parent);
|
|
528
|
-
claimed = captureDirectoryIdentity(lease);
|
|
529
|
-
writeOwner(join(lease, "owner.json"), owner);
|
|
530
|
-
syncDirectory(claimed);
|
|
531
|
-
return owner;
|
|
552
|
+
rmSync(claim, { recursive: true, force: true });
|
|
532
553
|
} catch {
|
|
533
|
-
|
|
534
|
-
try {
|
|
535
|
-
assertDirectoryIdentity(claimed);
|
|
536
|
-
rmSync(lease, { recursive: true, force: false });
|
|
537
|
-
} catch {
|
|
538
|
-
// Never remove a lease after its directory identity changes.
|
|
539
|
-
}
|
|
540
|
-
}
|
|
541
|
-
if (recoverStaleLease(lease, DURABLE_HISTORY_LEASE_STALE_MS)) continue;
|
|
542
|
-
throw new Error(`durable history lease busy: ${lease}`);
|
|
554
|
+
// A leftover claim directory never blocks a lease.
|
|
543
555
|
}
|
|
544
556
|
}
|
|
545
|
-
|
|
557
|
+
try {
|
|
558
|
+
assertDirectoryWithinParent(lease, parent);
|
|
559
|
+
const current = JSON.parse(completeJsonl(join(lease, "owner.json"))) as LeaseOwner;
|
|
560
|
+
if (current.owner_id !== owner.owner_id) return false;
|
|
561
|
+
syncDirectory(parent);
|
|
562
|
+
return true;
|
|
563
|
+
} catch {
|
|
564
|
+
releaseOwnedLease(lease, owner);
|
|
565
|
+
return false;
|
|
566
|
+
}
|
|
546
567
|
}
|
|
547
568
|
|
|
548
569
|
function writeOwner(path: string, owner: LeaseOwner): void {
|
|
@@ -595,7 +616,7 @@ function recoverStaleLease(lease: string, staleMs: number): boolean {
|
|
|
595
616
|
try {
|
|
596
617
|
assertDirectoryIdentity(leaseIdentity);
|
|
597
618
|
if (completeJsonl(ownerPath) !== raw) return false;
|
|
598
|
-
|
|
619
|
+
retireLease(lease, randomUUID());
|
|
599
620
|
return true;
|
|
600
621
|
} catch {
|
|
601
622
|
return false;
|
|
@@ -606,12 +627,23 @@ function releaseOwnedLease(lease: string, owner: LeaseOwner): void {
|
|
|
606
627
|
try {
|
|
607
628
|
const current = JSON.parse(completeJsonl(join(lease, "owner.json"))) as LeaseOwner;
|
|
608
629
|
if (current.owner_id !== owner.owner_id) return;
|
|
609
|
-
|
|
630
|
+
retireLease(lease, owner.owner_id);
|
|
610
631
|
} catch {
|
|
611
632
|
// A missing or replaced lease does not belong to this operation.
|
|
612
633
|
}
|
|
613
634
|
}
|
|
614
635
|
|
|
636
|
+
/**
|
|
637
|
+
* Move a lease aside before deleting it. Its path goes from held to absent in
|
|
638
|
+
* one rename and is never an empty directory that a concurrent claim could
|
|
639
|
+
* replace while a recursive removal is still running.
|
|
640
|
+
*/
|
|
641
|
+
function retireLease(lease: string, tag: string): void {
|
|
642
|
+
const retired = `${lease}.retired-${process.pid}-${tag}`;
|
|
643
|
+
renameSync(lease, retired);
|
|
644
|
+
rmSync(retired, { recursive: true, force: true });
|
|
645
|
+
}
|
|
646
|
+
|
|
615
647
|
function processIsAlive(pid: number): boolean {
|
|
616
648
|
try {
|
|
617
649
|
process.kill(pid, 0);
|
|
@@ -338,23 +338,20 @@ export async function withFamilyLease<T>(
|
|
|
338
338
|
let owner: LeaseOwner | undefined;
|
|
339
339
|
while (true) {
|
|
340
340
|
try {
|
|
341
|
-
|
|
342
|
-
const leaseIdentity = captureDirectoryIdentity(lease);
|
|
343
|
-
owner = {
|
|
344
|
-
owner_id: randomUUID(),
|
|
345
|
-
pid: process.pid,
|
|
346
|
-
acquired_at: new Date().toISOString(),
|
|
347
|
-
};
|
|
348
|
-
writeNewFile(join(lease, "owner.json"), Buffer.from(`${JSON.stringify(owner)}\n`));
|
|
349
|
-
assertDirectoryIdentity(leaseIdentity);
|
|
341
|
+
owner = claimLease(lease);
|
|
350
342
|
break;
|
|
351
343
|
} catch (error) {
|
|
352
|
-
if (
|
|
344
|
+
if (leaseHasWrongType(lease)) {
|
|
353
345
|
throw new HarneryLogLeaseError("wrong_type", `log lease is not a directory: ${lease}`);
|
|
354
346
|
}
|
|
355
347
|
if (recoverStaleLease(lease, policy.lease_stale_ms)) continue;
|
|
356
348
|
if (Date.now() - started >= policy.lease_timeout_ms) {
|
|
357
|
-
throw new HarneryLogLeaseError(
|
|
349
|
+
throw new HarneryLogLeaseError(
|
|
350
|
+
"timeout",
|
|
351
|
+
leaseHasValidOwner(lease)
|
|
352
|
+
? `timed out acquiring log lease: ${lease}`
|
|
353
|
+
: `timed out acquiring log lease with no valid owner; an interrupted writer left it, and it is never recovered automatically: ${lease}`,
|
|
354
|
+
);
|
|
358
355
|
}
|
|
359
356
|
await new Promise((resolve) => setTimeout(resolve, policy.lease_retry_ms));
|
|
360
357
|
void error;
|
|
@@ -367,6 +364,41 @@ export async function withFamilyLease<T>(
|
|
|
367
364
|
}
|
|
368
365
|
}
|
|
369
366
|
|
|
367
|
+
/**
|
|
368
|
+
* Publish a complete lease in one rename. The owner record is written and synced
|
|
369
|
+
* in a private claim directory first, so a writer stopped at any point leaves
|
|
370
|
+
* either no lease or one with an owner. A held lease always has an owner record,
|
|
371
|
+
* here and in the earlier create-then-write protocol, so a rename that replaces
|
|
372
|
+
* an empty lease directory never takes a held lease. A lease with any entry makes
|
|
373
|
+
* the rename fail, and the caller waits or recovers as before.
|
|
374
|
+
*/
|
|
375
|
+
function claimLease(lease: string): LeaseOwner {
|
|
376
|
+
const owner: LeaseOwner = {
|
|
377
|
+
owner_id: randomUUID(),
|
|
378
|
+
pid: process.pid,
|
|
379
|
+
acquired_at: new Date().toISOString(),
|
|
380
|
+
};
|
|
381
|
+
const content = `${JSON.stringify(owner)}\n`;
|
|
382
|
+
const claim = `${lease}.claim-${process.pid}-${owner.owner_id}`;
|
|
383
|
+
try {
|
|
384
|
+
mkdirSync(claim, { mode: stateDirMode() });
|
|
385
|
+
writeNewFile(join(claim, "owner.json"), Buffer.from(content));
|
|
386
|
+
renameSync(claim, lease);
|
|
387
|
+
} finally {
|
|
388
|
+
rmSync(claim, { recursive: true, force: true });
|
|
389
|
+
}
|
|
390
|
+
try {
|
|
391
|
+
captureDirectoryIdentity(lease);
|
|
392
|
+
if (readRegularFile(join(lease, "owner.json"), 4_096).toString("utf8") !== content) {
|
|
393
|
+
throw new Error(`log lease owner changed during claim: ${lease}`);
|
|
394
|
+
}
|
|
395
|
+
} catch (error) {
|
|
396
|
+
releaseOwnedLease(lease, owner);
|
|
397
|
+
throw error;
|
|
398
|
+
}
|
|
399
|
+
return owner;
|
|
400
|
+
}
|
|
401
|
+
|
|
370
402
|
function shouldRotate(
|
|
371
403
|
active: string,
|
|
372
404
|
incoming: readonly Buffer[],
|
|
@@ -476,6 +508,22 @@ interface LeaseOwner {
|
|
|
476
508
|
acquired_at: string;
|
|
477
509
|
}
|
|
478
510
|
|
|
511
|
+
function leaseHasValidOwner(lease: string): boolean {
|
|
512
|
+
try {
|
|
513
|
+
const owner = JSON.parse(
|
|
514
|
+
readRegularFile(join(lease, "owner.json"), 4_096).toString("utf8"),
|
|
515
|
+
) as LeaseOwner;
|
|
516
|
+
return (
|
|
517
|
+
typeof owner?.owner_id === "string" &&
|
|
518
|
+
owner.owner_id.length > 0 &&
|
|
519
|
+
Number.isSafeInteger(owner.pid) &&
|
|
520
|
+
Number.isFinite(Date.parse(owner.acquired_at))
|
|
521
|
+
);
|
|
522
|
+
} catch {
|
|
523
|
+
return false;
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
|
|
479
527
|
function recoverStaleLease(lease: string, maximumAgeMs: number): boolean {
|
|
480
528
|
let identity: DirectoryIdentity;
|
|
481
529
|
let raw: string;
|
|
@@ -504,7 +552,7 @@ function recoverStaleLease(lease: string, maximumAgeMs: number): boolean {
|
|
|
504
552
|
try {
|
|
505
553
|
assertDirectoryIdentity(identity);
|
|
506
554
|
if (readRegularFile(join(lease, "owner.json"), 4_096).toString("utf8") !== raw) return false;
|
|
507
|
-
|
|
555
|
+
retireLease(lease, randomUUID());
|
|
508
556
|
return true;
|
|
509
557
|
} catch {
|
|
510
558
|
return false;
|
|
@@ -517,12 +565,23 @@ function releaseOwnedLease(lease: string, owner: LeaseOwner): void {
|
|
|
517
565
|
readRegularFile(join(lease, "owner.json"), 4_096).toString("utf8"),
|
|
518
566
|
) as LeaseOwner;
|
|
519
567
|
if (current.owner_id !== owner.owner_id) return;
|
|
520
|
-
|
|
568
|
+
retireLease(lease, owner.owner_id);
|
|
521
569
|
} catch {
|
|
522
570
|
// A missing or replaced lease does not belong to this operation.
|
|
523
571
|
}
|
|
524
572
|
}
|
|
525
573
|
|
|
574
|
+
/**
|
|
575
|
+
* Move a lease aside before deleting it. Its path goes from held to absent in
|
|
576
|
+
* one rename and is never an empty directory that a concurrent claim could
|
|
577
|
+
* replace while a recursive removal is still running.
|
|
578
|
+
*/
|
|
579
|
+
function retireLease(lease: string, tag: string): void {
|
|
580
|
+
const retired = `${lease}.retired-${process.pid}-${tag}`;
|
|
581
|
+
renameSync(lease, retired);
|
|
582
|
+
rmSync(retired, { recursive: true, force: true });
|
|
583
|
+
}
|
|
584
|
+
|
|
526
585
|
function processIsAlive(pid: number): boolean {
|
|
527
586
|
try {
|
|
528
587
|
process.kill(pid, 0);
|
|
@@ -699,11 +758,14 @@ function nativeWriteSync(fd: number, buffer: Buffer, offset: number, length: num
|
|
|
699
758
|
return writeSync(fd, buffer, offset, length);
|
|
700
759
|
}
|
|
701
760
|
|
|
702
|
-
|
|
761
|
+
/** One lstat, so a lease released between checks reads as absent, not as a wrong type. */
|
|
762
|
+
function leaseHasWrongType(path: string): boolean {
|
|
703
763
|
try {
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
764
|
+
const stat = lstatSync(path);
|
|
765
|
+
return stat.isSymbolicLink() || !stat.isDirectory();
|
|
766
|
+
} catch (error) {
|
|
767
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return false;
|
|
768
|
+
throw error;
|
|
707
769
|
}
|
|
708
770
|
}
|
|
709
771
|
|
|
@@ -17,6 +17,7 @@ import { performance } from "node:perf_hooks";
|
|
|
17
17
|
import { fileURLToPath } from "node:url";
|
|
18
18
|
import { collectCoordinationHealthSnapshot } from "../agents/health.ts";
|
|
19
19
|
import {
|
|
20
|
+
type ProcessGenerationSignalDependencies,
|
|
20
21
|
type ProcessGenerationVerdict,
|
|
21
22
|
processStartToken,
|
|
22
23
|
signalIfSameGeneration,
|
|
@@ -185,9 +186,7 @@ export interface SupervisorStopResult extends SupervisorStatus {
|
|
|
185
186
|
|
|
186
187
|
export function requestSupervisorStop(
|
|
187
188
|
coordRootRaw: string,
|
|
188
|
-
dependencies: SupervisorProcessDependencies & {
|
|
189
|
-
kill?: (pid: number, signal: NodeJS.Signals | number) => unknown;
|
|
190
|
-
} = {},
|
|
189
|
+
dependencies: SupervisorProcessDependencies & ProcessGenerationSignalDependencies = {},
|
|
191
190
|
): SupervisorStopResult {
|
|
192
191
|
const coordRoot = resolve(coordRootRaw);
|
|
193
192
|
const paths = supervisorPaths(coordRoot);
|
package/src/core/work/state.ts
CHANGED
|
@@ -20,7 +20,7 @@ import {
|
|
|
20
20
|
import { stateDirMode, stateFileMode } from "../storage/modes.ts";
|
|
21
21
|
import { readWorkflowApproval } from "../workflow/approvals.ts";
|
|
22
22
|
import { stableDigest } from "../workflow/durable-record.ts";
|
|
23
|
-
import {
|
|
23
|
+
import { readWorkflowProof } from "../workflow/proof.ts";
|
|
24
24
|
import { readWorkflowRunManifest, workflowScriptDigest } from "../workflow/run-state.ts";
|
|
25
25
|
import { WORKFLOW_TRANSCRIPT_EVENT_BYTES, workflowTranscriptPath } from "../workflow/transcript.ts";
|
|
26
26
|
import { isWorkspaceBoundExecutionEvidence } from "../workflow/workspaces/validate.ts";
|
|
@@ -718,14 +718,14 @@ function inspectAttempt(
|
|
|
718
718
|
proof.acceptance.summary.unknown === 0
|
|
719
719
|
? "succeeded"
|
|
720
720
|
: "failed";
|
|
721
|
-
//
|
|
722
|
-
//
|
|
723
|
-
//
|
|
721
|
+
// A failed attempt the run classified as uninformative about the work is
|
|
722
|
+
// uncharged (ADR 0046). Absent class ⇒ charged, as before. Read only for a
|
|
723
|
+
// failed attempt: a succeeded run never carries a class.
|
|
724
724
|
if (
|
|
725
725
|
attempt.status === "failed" &&
|
|
726
|
-
(proof.run.class === "
|
|
727
|
-
|
|
728
|
-
|
|
726
|
+
(proof.run.class === "environment" ||
|
|
727
|
+
proof.run.class === "upstream" ||
|
|
728
|
+
proof.run.class === "decision")
|
|
729
729
|
) {
|
|
730
730
|
attempt.uncharged = proof.run.class;
|
|
731
731
|
if (proof.run.class === "decision" && typeof proof.run.decision_id === "string") {
|
|
@@ -45,16 +45,7 @@ import { freezeWorkflowAttemptContext } from "./attempt-context.ts";
|
|
|
45
45
|
import { type BillingProbe, probeBilling } from "./billing.ts";
|
|
46
46
|
import { stableDigest } from "./durable-record.ts";
|
|
47
47
|
import { evidencePreflightError } from "./evidence-preflight.ts";
|
|
48
|
-
import {
|
|
49
|
-
canRetryFailedSpawn,
|
|
50
|
-
observeSpawnFailure,
|
|
51
|
-
resolveFailureEvidence,
|
|
52
|
-
} from "./failure-evidence.ts";
|
|
53
|
-
import {
|
|
54
|
-
endWorkflowChildSessionV3,
|
|
55
|
-
recordWorkflowChildTerminalFailureV3,
|
|
56
|
-
startWorkflowChildSessionV3,
|
|
57
|
-
} from "./live-session-v3.ts";
|
|
48
|
+
import { endWorkflowChildSessionV3, startWorkflowChildSessionV3 } from "./live-session-v3.ts";
|
|
58
49
|
import {
|
|
59
50
|
buildWorkflowProof,
|
|
60
51
|
createEvidenceRecord,
|
|
@@ -1038,38 +1029,22 @@ async function executeWorkflow(
|
|
|
1038
1029
|
});
|
|
1039
1030
|
agentProof.access = access;
|
|
1040
1031
|
|
|
1041
|
-
let childGenerationId: string | undefined;
|
|
1042
1032
|
let recordAgentEnd: (() => void) | undefined;
|
|
1043
1033
|
const finalizeChild = (): void => {
|
|
1044
1034
|
try {
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
// Result publication is the resume-cache boundary, after canonical custody.
|
|
1035
|
+
// Record the canonical terminal first. agent.end is the resume-cache
|
|
1036
|
+
// boundary, so a child whose end was not recorded never reports success.
|
|
1037
|
+
endWorkflowChildSessionV3({
|
|
1038
|
+
coordRoot: opts.coordRoot,
|
|
1039
|
+
instanceId: childInstanceId(runId, id),
|
|
1040
|
+
runId,
|
|
1041
|
+
agentId: id,
|
|
1042
|
+
adapter,
|
|
1043
|
+
cleanExit: agentProof.status === "succeeded",
|
|
1044
|
+
});
|
|
1056
1045
|
recordAgentEnd?.();
|
|
1057
1046
|
} catch (error) {
|
|
1058
|
-
|
|
1059
|
-
try {
|
|
1060
|
-
if (childGenerationId)
|
|
1061
|
-
recordWorkflowChildTerminalFailureV3({
|
|
1062
|
-
coordRoot: opts.coordRoot,
|
|
1063
|
-
instanceId: childInstanceId(runId, id),
|
|
1064
|
-
runId,
|
|
1065
|
-
agentId: id,
|
|
1066
|
-
adapter,
|
|
1067
|
-
expectedGenerationId: childGenerationId,
|
|
1068
|
-
reason,
|
|
1069
|
-
});
|
|
1070
|
-
} catch (visibilityError) {
|
|
1071
|
-
reason += `; blocked lifecycle recording failed: ${(visibilityError as Error).message}`;
|
|
1072
|
-
}
|
|
1047
|
+
const reason = `V3 terminal recording failed: ${(error as Error).message}`;
|
|
1073
1048
|
const failure = new Error(agentProof.error ? `${agentProof.error}; ${reason}` : reason, {
|
|
1074
1049
|
cause: error,
|
|
1075
1050
|
});
|
|
@@ -1102,7 +1077,7 @@ async function executeWorkflow(
|
|
|
1102
1077
|
// Headless children do not reliably fire adapter hooks. The engine owns
|
|
1103
1078
|
// their canonical V3 start so every dispatched child has an auditable
|
|
1104
1079
|
// generation before it begins work.
|
|
1105
|
-
|
|
1080
|
+
startWorkflowChildSessionV3({
|
|
1106
1081
|
coordRoot: opts.coordRoot,
|
|
1107
1082
|
instanceId: childInstanceId(runId, id),
|
|
1108
1083
|
runId,
|
|
@@ -1111,13 +1086,10 @@ async function executeWorkflow(
|
|
|
1111
1086
|
label,
|
|
1112
1087
|
model: agentOpts.model,
|
|
1113
1088
|
});
|
|
1114
|
-
if (!child.v3_generation_id) throw new Error("workflow_child_v3_generation_missing");
|
|
1115
|
-
childGenerationId = child.v3_generation_id;
|
|
1116
1089
|
|
|
1117
1090
|
let attemptPrompt = dispatchPrompt;
|
|
1118
1091
|
let last: SpawnResult | null = null;
|
|
1119
1092
|
let agentCostUsd = 0;
|
|
1120
|
-
let informativeAttempt: ReturnType<typeof observeSpawnFailure> | undefined;
|
|
1121
1093
|
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
1122
1094
|
last = await spawner({
|
|
1123
1095
|
prompt: attemptPrompt,
|
|
@@ -1140,30 +1112,14 @@ async function executeWorkflow(
|
|
|
1140
1112
|
costUsd += last.costUsd ?? 0;
|
|
1141
1113
|
|
|
1142
1114
|
if (!last.ok) {
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
effect: { state: "unknown" },
|
|
1151
|
-
partial_output: informativeAttempt.partial_output,
|
|
1152
|
-
}
|
|
1153
|
-
: observed;
|
|
1154
|
-
transcript("agent.attempt_failed", {
|
|
1155
|
-
id,
|
|
1156
|
-
attempt,
|
|
1157
|
-
error: last.error,
|
|
1158
|
-
failure_evidence: observed,
|
|
1159
|
-
result_digest: digestResult(last.text, "text"),
|
|
1160
|
-
text: last.text,
|
|
1161
|
-
});
|
|
1162
|
-
if (!canRetryFailedSpawn({ ...last, failureEvidence: agentProof.failure_evidence }))
|
|
1163
|
-
break;
|
|
1164
|
-
continue;
|
|
1115
|
+
transcript("agent.attempt_failed", { id, attempt, error: last.error });
|
|
1116
|
+
// ADR 0046: an environment failure (the binary was absent) cannot be
|
|
1117
|
+
// helped by retrying an unchanged environment, so stop the in-agent
|
|
1118
|
+
// retry too — not just the outer attempt/replan budget. An upstream
|
|
1119
|
+
// refusal keeps retrying here: the vendor may recover mid-loop.
|
|
1120
|
+
if (last.class === "environment") break;
|
|
1121
|
+
continue; // spawn-level failure: retry with the original prompt
|
|
1165
1122
|
}
|
|
1166
|
-
informativeAttempt = observeSpawnFailure({}, last.text, "final_message");
|
|
1167
1123
|
if (!agentOpts.schema) {
|
|
1168
1124
|
agentProof.status = "succeeded";
|
|
1169
1125
|
agentProof.cost_usd =
|
|
@@ -1217,14 +1173,7 @@ async function executeWorkflow(
|
|
|
1217
1173
|
return parsed.value;
|
|
1218
1174
|
}
|
|
1219
1175
|
|
|
1220
|
-
|
|
1221
|
-
transcript("agent.schema_retry", {
|
|
1222
|
-
id,
|
|
1223
|
-
attempt,
|
|
1224
|
-
problems,
|
|
1225
|
-
rejected_text: last.text,
|
|
1226
|
-
result_digest: digestResult(last.text, "text"),
|
|
1227
|
-
});
|
|
1176
|
+
transcript("agent.schema_retry", { id, attempt, problems });
|
|
1228
1177
|
// Keep the original cache identity while making the actual output
|
|
1229
1178
|
// contract explicit. A full bounded copy of the rejected value lets the
|
|
1230
1179
|
// child repair omissions instead of reconstructing an unseen object.
|
|
@@ -1260,7 +1209,6 @@ async function executeWorkflow(
|
|
|
1260
1209
|
try {
|
|
1261
1210
|
finalizeChild();
|
|
1262
1211
|
} finally {
|
|
1263
|
-
// A slot is disposable; unresolved custody remains visible and failed.
|
|
1264
1212
|
release();
|
|
1265
1213
|
}
|
|
1266
1214
|
}
|
|
@@ -39,12 +39,23 @@ export {
|
|
|
39
39
|
rollupAcceptance,
|
|
40
40
|
writeWorkflowProof,
|
|
41
41
|
} from "./proof.ts";
|
|
42
|
-
export type {
|
|
42
|
+
export type {
|
|
43
|
+
WorkflowRecoveryAuthorization,
|
|
44
|
+
WorkflowRecoveryIntent,
|
|
45
|
+
WorkflowRecoveryRecord,
|
|
46
|
+
} from "./recovery-audit.ts";
|
|
47
|
+
export { beginWorkflowRecovery, readWorkflowRecoveryLedger } from "./recovery-audit.ts";
|
|
48
|
+
export type {
|
|
49
|
+
WorkflowResumeRecoveryOptions,
|
|
50
|
+
WorkflowResumeRecoveryResult,
|
|
51
|
+
WorkflowRunManifest,
|
|
52
|
+
} from "./run-state.ts";
|
|
43
53
|
export {
|
|
44
54
|
acquireWorkflowResumeLease,
|
|
45
55
|
assertWorkflowRunResumable,
|
|
46
56
|
assertWorkflowScriptUnchanged,
|
|
47
57
|
readWorkflowRunManifest,
|
|
58
|
+
recoverWorkflowResumeLease,
|
|
48
59
|
WORKFLOW_RUN_MANIFEST_SCHEMA_VERSION,
|
|
49
60
|
workflowRunManifestPath,
|
|
50
61
|
workflowScriptDigest,
|
|
@@ -1,16 +1,9 @@
|
|
|
1
|
-
import { randomUUID } from "node:crypto";
|
|
2
|
-
import { existsSync, linkSync, readdirSync, readFileSync, renameSync, unlinkSync } from "node:fs";
|
|
3
|
-
import { basename, dirname } from "node:path";
|
|
4
1
|
import { type Adapter, adapterFromPlatform } from "../adapter.ts";
|
|
5
|
-
import {
|
|
6
|
-
recordLiveLifecycleChangeV3,
|
|
7
|
-
recordLiveTaskChangeV3,
|
|
8
|
-
} from "../agents/live-authority-v3.ts";
|
|
2
|
+
import { recordLiveTaskChangeV3 } from "../agents/live-authority-v3.ts";
|
|
9
3
|
import { endSessionExplicitV3 } from "../agents/session-finalizer-v3.ts";
|
|
10
|
-
import { heartbeatPath } from "../agents/state/heartbeat-reader.ts";
|
|
11
4
|
import {
|
|
5
|
+
clearCoordinationCache,
|
|
12
6
|
type Heartbeat,
|
|
13
|
-
readHeartbeat,
|
|
14
7
|
setIdentityCache,
|
|
15
8
|
} from "../agents/state/heartbeat-writer.ts";
|
|
16
9
|
import { readLiveCoordinationRow } from "../agents/state/live-coordination-view.ts";
|
|
@@ -19,17 +12,9 @@ import { assignName } from "../agents/state/names.ts";
|
|
|
19
12
|
import { coordFreshnessSeconds } from "../config.ts";
|
|
20
13
|
import { ensureEventLedgerV3 } from "../events/v3/bootstrap.ts";
|
|
21
14
|
import {
|
|
22
|
-
readCoordinationViewV3,
|
|
23
|
-
requireAuthoritySafeCoordinationViewV3,
|
|
24
|
-
} from "../events/v3/coordination-view.ts";
|
|
25
|
-
import { withGenerationCustodyV3 } from "../events/v3/generation-custody.ts";
|
|
26
|
-
import {
|
|
27
|
-
liveInstanceIdV3,
|
|
28
15
|
recordLiveHookSignalV3,
|
|
29
16
|
resolveLiveEventLedgerRouteV3,
|
|
30
17
|
} from "../events/v3/live-routing.ts";
|
|
31
|
-
import { readHookProducerStateV3 } from "../events/v3/producers/recorder.ts";
|
|
32
|
-
import { fsyncParentDirectory } from "./durable-record.ts";
|
|
33
18
|
import { stableScopeId } from "./scope-id.ts";
|
|
34
19
|
|
|
35
20
|
export interface WorkflowChildSessionV3Input {
|
|
@@ -92,45 +77,16 @@ export function startWorkflowChildSessionV3(input: WorkflowChildSessionV3Input):
|
|
|
92
77
|
}
|
|
93
78
|
const projected = readLiveCoordinationRow(input.coordRoot, input.instanceId);
|
|
94
79
|
if (!projected) throw new Error("workflow_child_v3_projection_missing");
|
|
95
|
-
const own = readHookProducerStateV3(input.coordRoot, adapter, sessionId);
|
|
96
|
-
if (!own || own.terminal || own.generation_id !== projected.v3_generation_id) {
|
|
97
|
-
throw new Error("workflow_child_v3_start_generation_changed");
|
|
98
|
-
}
|
|
99
80
|
return projected;
|
|
100
81
|
}
|
|
101
82
|
|
|
102
83
|
/** Record the authoritative terminal before removing the disposable coordination cache. */
|
|
103
84
|
export function endWorkflowChildSessionV3(
|
|
104
|
-
input: WorkflowChildSessionV3Input & { cleanExit: boolean
|
|
105
|
-
): void {
|
|
106
|
-
withGenerationCustodyV3(input.coordRoot, [liveInstanceIdV3(input.instanceId)], () =>
|
|
107
|
-
endExpectedChild(input),
|
|
108
|
-
);
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
function endExpectedChild(
|
|
112
|
-
input: WorkflowChildSessionV3Input & { cleanExit: boolean; expectedGenerationId: string },
|
|
85
|
+
input: WorkflowChildSessionV3Input & { cleanExit: boolean },
|
|
113
86
|
): void {
|
|
114
87
|
requireV3Route(input.coordRoot);
|
|
115
|
-
const view = requireAuthoritySafeCoordinationViewV3(readCoordinationViewV3(input.coordRoot));
|
|
116
|
-
const instanceId = liveInstanceIdV3(input.instanceId);
|
|
117
|
-
const current = view.instances[instanceId];
|
|
118
|
-
const terminal = view.terminal_generations[input.expectedGenerationId];
|
|
119
|
-
if (terminal?.instance_id === instanceId) recoverHeldCaches(input);
|
|
120
|
-
if (current && current.generation_id !== input.expectedGenerationId) {
|
|
121
|
-
throw new Error("workflow_child_v3_generation_changed");
|
|
122
|
-
}
|
|
123
|
-
if (!current) {
|
|
124
|
-
if (!terminal || terminal.instance_id !== instanceId)
|
|
125
|
-
throw new Error("workflow_child_v3_generation_missing");
|
|
126
|
-
if (terminal.terminal?.outcome !== (input.cleanExit ? "succeeded" : "failed")) {
|
|
127
|
-
throw new Error("workflow_child_v3_terminal_outcome_changed");
|
|
128
|
-
}
|
|
129
|
-
clearExpectedCache(input);
|
|
130
|
-
return;
|
|
131
|
-
}
|
|
132
88
|
const row = readLiveCoordinationRow(input.coordRoot, input.instanceId);
|
|
133
|
-
if (!row?.v3_instance_id || row.v3_generation_id
|
|
89
|
+
if (!row?.v3_instance_id || !row.v3_generation_id) {
|
|
134
90
|
throw new Error("workflow_child_v3_generation_missing");
|
|
135
91
|
}
|
|
136
92
|
const result = endSessionExplicitV3({
|
|
@@ -145,104 +101,7 @@ function endExpectedChild(
|
|
|
145
101
|
`workflow_child_v3_end_failed:${result.state}${"reason" in result ? `:${result.reason}` : ""}`,
|
|
146
102
|
);
|
|
147
103
|
}
|
|
148
|
-
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
/** Persist unresolved terminal custody only for the generation this dispatch owns. */
|
|
152
|
-
export function recordWorkflowChildTerminalFailureV3(
|
|
153
|
-
input: WorkflowChildSessionV3Input & { expectedGenerationId: string; reason: string },
|
|
154
|
-
): void {
|
|
155
|
-
withGenerationCustodyV3(input.coordRoot, [liveInstanceIdV3(input.instanceId)], () => {
|
|
156
|
-
const row = readLiveCoordinationRow(input.coordRoot, input.instanceId);
|
|
157
|
-
if (!row || row.v3_generation_id !== input.expectedGenerationId) return;
|
|
158
|
-
recordLiveLifecycleChangeV3({
|
|
159
|
-
coordRoot: input.coordRoot,
|
|
160
|
-
owner: input.instanceId,
|
|
161
|
-
nativeSessionId: input.sessionId ?? input.instanceId,
|
|
162
|
-
adapter: workflowAdapter(input.adapter),
|
|
163
|
-
expectedGenerationId: input.expectedGenerationId as `gen_${string}`,
|
|
164
|
-
state: "blocked",
|
|
165
|
-
reason: input.reason,
|
|
166
|
-
});
|
|
167
|
-
});
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
function heldPrefix(input: WorkflowChildSessionV3Input & { expectedGenerationId: string }): string {
|
|
171
|
-
if (!/^gen_[a-zA-Z0-9._-]{1,128}$/.test(input.expectedGenerationId))
|
|
172
|
-
throw new Error("workflow_child_v3_generation_invalid");
|
|
173
|
-
return `${basename(heartbeatPath(input.coordRoot, input.instanceId))}.terminal-${input.expectedGenerationId}-`;
|
|
174
|
-
}
|
|
175
|
-
|
|
176
|
-
function recoverHeldCaches(
|
|
177
|
-
input: WorkflowChildSessionV3Input & { expectedGenerationId: string },
|
|
178
|
-
): void {
|
|
179
|
-
const path = heartbeatPath(input.coordRoot, input.instanceId);
|
|
180
|
-
if (!existsSync(dirname(path))) return;
|
|
181
|
-
const prefix = heldPrefix(input);
|
|
182
|
-
for (const name of readdirSync(dirname(path)).filter((value) => value.startsWith(prefix))) {
|
|
183
|
-
finishHeldCache(`${dirname(path)}/${name}`, path, input.expectedGenerationId);
|
|
184
|
-
}
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
function clearExpectedCache(
|
|
188
|
-
input: WorkflowChildSessionV3Input & { expectedGenerationId: string },
|
|
189
|
-
): void {
|
|
190
|
-
recoverHeldCaches(input);
|
|
191
|
-
if (
|
|
192
|
-
readHeartbeat(input.coordRoot, input.instanceId)?.v3_generation_id !==
|
|
193
|
-
input.expectedGenerationId
|
|
194
|
-
)
|
|
195
|
-
return;
|
|
196
|
-
const path = heartbeatPath(input.coordRoot, input.instanceId);
|
|
197
|
-
const held = `${dirname(path)}/${heldPrefix(input)}${randomUUID()}`;
|
|
198
|
-
try {
|
|
199
|
-
// A unique held name pins the actual bytes atomically. Checking before
|
|
200
|
-
// unlink alone could consume a replacement published at this boundary.
|
|
201
|
-
renameSync(path, held);
|
|
202
|
-
fsyncParentDirectory(path);
|
|
203
|
-
} catch (error) {
|
|
204
|
-
if ((error as NodeJS.ErrnoException).code === "ENOENT") return;
|
|
205
|
-
throw error;
|
|
206
|
-
}
|
|
207
|
-
finishHeldCache(held, path, input.expectedGenerationId);
|
|
208
|
-
}
|
|
209
|
-
|
|
210
|
-
function finishHeldCache(held: string, path: string, expectedGenerationId: string): void {
|
|
211
|
-
let removed = false;
|
|
212
|
-
let cleanupFailed = false;
|
|
213
|
-
let cleanupError: unknown;
|
|
214
|
-
try {
|
|
215
|
-
const cache = JSON.parse(readFileSync(held, "utf8")) as Heartbeat;
|
|
216
|
-
if (cache.v3_generation_id === expectedGenerationId) {
|
|
217
|
-
unlinkSync(held);
|
|
218
|
-
fsyncParentDirectory(held);
|
|
219
|
-
removed = true;
|
|
220
|
-
}
|
|
221
|
-
} catch (error) {
|
|
222
|
-
cleanupFailed = true;
|
|
223
|
-
cleanupError = error;
|
|
224
|
-
}
|
|
225
|
-
if (!removed) {
|
|
226
|
-
try {
|
|
227
|
-
// Never overwrite a later publication. Keep the held evidence when
|
|
228
|
-
// another cache already owns the live name.
|
|
229
|
-
linkSync(held, path);
|
|
230
|
-
fsyncParentDirectory(path);
|
|
231
|
-
unlinkSync(held);
|
|
232
|
-
fsyncParentDirectory(held);
|
|
233
|
-
} catch (error) {
|
|
234
|
-
if ((error as NodeJS.ErrnoException).code !== "EEXIST") {
|
|
235
|
-
if (cleanupFailed) {
|
|
236
|
-
throw new AggregateError(
|
|
237
|
-
[cleanupError, error],
|
|
238
|
-
`workflow_child_v3_cache_cleanup_and_restore_failed: ${String(cleanupError)}; restoration: ${String(error)}`,
|
|
239
|
-
);
|
|
240
|
-
}
|
|
241
|
-
throw error;
|
|
242
|
-
}
|
|
243
|
-
}
|
|
244
|
-
}
|
|
245
|
-
if (cleanupFailed) throw cleanupError;
|
|
104
|
+
clearCoordinationCache(input.coordRoot, input.instanceId);
|
|
246
105
|
}
|
|
247
106
|
|
|
248
107
|
function requireV3Route(coordRoot: string) {
|