harnery 0.44.0 → 0.45.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. package/dist/commands/workflow.d.ts.map +1 -1
  2. package/dist/commands/workflow.js +44 -0
  3. package/dist/core/agents/coord-client.d.ts.map +1 -1
  4. package/dist/core/agents/coord-client.js +18 -0
  5. package/dist/core/agents/live-authority-v3.d.ts +0 -1
  6. package/dist/core/agents/live-authority-v3.d.ts.map +1 -1
  7. package/dist/core/agents/live-authority-v3.js +0 -11
  8. package/dist/core/agents/live-lifecycle-v3.d.ts +2 -4
  9. package/dist/core/agents/live-lifecycle-v3.d.ts.map +1 -1
  10. package/dist/core/agents/live-lifecycle-v3.js +7 -28
  11. package/dist/core/agents/state/heartbeat-writer.d.ts.map +1 -1
  12. package/dist/core/agents/state/heartbeat-writer.js +14 -25
  13. package/dist/core/agents/state/live-coordination-writer.d.ts.map +1 -1
  14. package/dist/core/agents/state/live-coordination-writer.js +0 -5
  15. package/dist/core/agents/state/pidmap.d.ts.map +1 -1
  16. package/dist/core/agents/state/pidmap.js +0 -5
  17. package/dist/core/agents/state/proc-start.d.ts +5 -6
  18. package/dist/core/agents/state/proc-start.d.ts.map +1 -1
  19. package/dist/core/agents/state/proc-start.js +13 -9
  20. package/dist/core/agents/state/process-handle.d.ts +26 -0
  21. package/dist/core/agents/state/process-handle.d.ts.map +1 -0
  22. package/dist/core/agents/state/process-handle.js +121 -0
  23. package/dist/core/agents/state/stale-sweep.d.ts.map +1 -1
  24. package/dist/core/agents/state/stale-sweep.js +70 -180
  25. package/dist/core/artifacts/activity.d.ts +4 -1
  26. package/dist/core/artifacts/activity.d.ts.map +1 -1
  27. package/dist/core/artifacts/activity.js +8 -2
  28. package/dist/core/artifacts/index.d.ts.map +1 -1
  29. package/dist/core/artifacts/index.js +38 -12
  30. package/dist/core/events/v3/authority-outbox.d.ts.map +1 -1
  31. package/dist/core/events/v3/authority-outbox.js +0 -29
  32. package/dist/core/events/v3/producers/coordination-recorder.d.ts +0 -1
  33. package/dist/core/events/v3/producers/coordination-recorder.d.ts.map +1 -1
  34. package/dist/core/events/v3/producers/coordination-recorder.js +0 -7
  35. package/dist/core/events/v3/producers/recorder.d.ts.map +1 -1
  36. package/dist/core/events/v3/producers/recorder.js +64 -77
  37. package/dist/core/hooks/cli.js +78 -48
  38. package/dist/core/resources/service.d.ts +2 -16
  39. package/dist/core/resources/service.d.ts.map +1 -1
  40. package/dist/core/resources/service.js +1 -78
  41. package/dist/core/semantic/service.d.ts.map +1 -1
  42. package/dist/core/semantic/service.js +2 -9
  43. package/dist/core/servers/index.d.ts +4 -0
  44. package/dist/core/servers/index.d.ts.map +1 -1
  45. package/dist/core/servers/index.js +21 -3
  46. package/dist/core/storage/durable-history.js +56 -22
  47. package/dist/core/storage/segments.d.ts.map +1 -1
  48. package/dist/core/storage/segments.js +73 -17
  49. package/dist/core/supervisor/service.d.ts +2 -4
  50. package/dist/core/supervisor/service.d.ts.map +1 -1
  51. package/dist/core/work/state.js +7 -7
  52. package/dist/core/workflow/engine.d.ts.map +1 -1
  53. package/dist/core/workflow/engine.js +21 -65
  54. package/dist/core/workflow/index.d.ts +4 -2
  55. package/dist/core/workflow/index.d.ts.map +1 -1
  56. package/dist/core/workflow/index.js +2 -1
  57. package/dist/core/workflow/live-session-v3.d.ts +0 -6
  58. package/dist/core/workflow/live-session-v3.d.ts.map +1 -1
  59. package/dist/core/workflow/live-session-v3.js +5 -126
  60. package/dist/core/workflow/proof.d.ts +4 -4
  61. package/dist/core/workflow/proof.d.ts.map +1 -1
  62. package/dist/core/workflow/proof.js +4 -12
  63. package/dist/core/workflow/recovery-audit.d.ts +30 -0
  64. package/dist/core/workflow/recovery-audit.d.ts.map +1 -0
  65. package/dist/core/workflow/recovery-audit.js +116 -0
  66. package/dist/core/workflow/run-state.d.ts +19 -2
  67. package/dist/core/workflow/run-state.d.ts.map +1 -1
  68. package/dist/core/workflow/run-state.js +113 -8
  69. package/dist/core/workflow/spawn-claude.d.ts.map +1 -1
  70. package/dist/core/workflow/spawn-claude.js +0 -19
  71. package/dist/core/workflow/spawn-codex.d.ts.map +1 -1
  72. package/dist/core/workflow/spawn-codex.js +0 -13
  73. package/dist/core/workflow/spawn-cursor.d.ts.map +1 -1
  74. package/dist/core/workflow/spawn-cursor.js +0 -19
  75. package/dist/core/workflow/spawn-opencode.d.ts.map +1 -1
  76. package/dist/core/workflow/spawn-opencode.js +0 -14
  77. package/dist/core/workflow/types.d.ts +11 -54
  78. package/dist/core/workflow/types.d.ts.map +1 -1
  79. package/dist/core/workflow/workspaces/local-git.d.ts.map +1 -1
  80. package/dist/core/workflow/workspaces/local-git.js +70 -14
  81. package/dist/core/workflow/workspaces/reclaim.d.ts +6 -1
  82. package/dist/core/workflow/workspaces/reclaim.d.ts.map +1 -1
  83. package/dist/core/workflow/workspaces/recovery.d.ts +2 -0
  84. package/dist/core/workflow/workspaces/recovery.d.ts.map +1 -1
  85. package/dist/core/workflow/workspaces/recovery.js +24 -0
  86. package/dist/lib/identities/assume.d.ts.map +1 -1
  87. package/dist/lib/identities/assume.js +0 -4
  88. package/package.json +1 -1
  89. package/src/commands/workflow.ts +59 -0
  90. package/src/core/agents/coord-client.ts +20 -0
  91. package/src/core/agents/live-authority-v3.ts +0 -17
  92. package/src/core/agents/live-lifecycle-v3.ts +14 -52
  93. package/src/core/agents/state/heartbeat-writer.ts +12 -29
  94. package/src/core/agents/state/live-coordination-writer.ts +0 -29
  95. package/src/core/agents/state/pidmap.ts +0 -13
  96. package/src/core/agents/state/proc-start.ts +14 -11
  97. package/src/core/agents/state/process-handle.ts +165 -0
  98. package/src/core/agents/state/stale-sweep.ts +82 -247
  99. package/src/core/artifacts/activity.ts +9 -2
  100. package/src/core/artifacts/index.ts +45 -12
  101. package/src/core/events/v3/authority-outbox.ts +0 -56
  102. package/src/core/events/v3/producers/coordination-recorder.ts +0 -15
  103. package/src/core/events/v3/producers/recorder.ts +68 -94
  104. package/src/core/hooks/cli.ts +85 -47
  105. package/src/core/resources/service.ts +3 -88
  106. package/src/core/semantic/service.ts +2 -8
  107. package/src/core/servers/index.ts +30 -3
  108. package/src/core/storage/durable-history.ts +52 -20
  109. package/src/core/storage/segments.ts +79 -17
  110. package/src/core/supervisor/service.ts +2 -3
  111. package/src/core/work/state.ts +7 -7
  112. package/src/core/workflow/engine.ts +21 -73
  113. package/src/core/workflow/index.ts +12 -1
  114. package/src/core/workflow/live-session-v3.ts +5 -146
  115. package/src/core/workflow/proof.ts +5 -21
  116. package/src/core/workflow/recovery-audit.ts +181 -0
  117. package/src/core/workflow/run-state.ts +137 -8
  118. package/src/core/workflow/spawn-claude.ts +0 -18
  119. package/src/core/workflow/spawn-codex.ts +0 -13
  120. package/src/core/workflow/spawn-cursor.ts +0 -18
  121. package/src/core/workflow/spawn-opencode.ts +0 -14
  122. package/src/core/workflow/types.ts +11 -41
  123. package/src/core/workflow/workspaces/local-git.ts +86 -15
  124. package/src/core/workflow/workspaces/reclaim.ts +4 -0
  125. package/src/core/workflow/workspaces/recovery.ts +23 -0
  126. package/src/lib/identities/assume.ts +0 -4
  127. package/dist/core/agents/state/unchanged-file.d.ts +0 -44
  128. package/dist/core/agents/state/unchanged-file.d.ts.map +0 -1
  129. package/dist/core/agents/state/unchanged-file.js +0 -390
  130. package/dist/core/events/v3/generation-custody.d.ts +0 -5
  131. package/dist/core/events/v3/generation-custody.d.ts.map +0 -1
  132. package/dist/core/events/v3/generation-custody.js +0 -50
  133. package/dist/core/workflow/failure-evidence.d.ts +0 -16
  134. package/dist/core/workflow/failure-evidence.d.ts.map +0 -1
  135. package/dist/core/workflow/failure-evidence.js +0 -114
  136. package/src/core/agents/state/unchanged-file.ts +0 -456
  137. package/src/core/events/v3/generation-custody.ts +0 -59
  138. package/src/core/workflow/failure-evidence.ts +0 -141
@@ -521,28 +521,49 @@ function acquireLease(lease: string): LeaseOwner {
521
521
  pid: process.pid,
522
522
  acquired_at: new Date().toISOString(),
523
523
  };
524
- let claimed: DirectoryIdentity | undefined;
524
+ if (claimLease(lease, parent, owner)) return owner;
525
+ if (recoverStaleLease(lease, DURABLE_HISTORY_LEASE_STALE_MS)) continue;
526
+ throw new Error(`durable history lease busy: ${lease}`);
527
+ }
528
+ throw new Error(`durable history lease busy: ${lease}`);
529
+ }
530
+
531
+ /**
532
+ * Publish a complete lease in one rename. The owner record is written and synced
533
+ * in a private claim directory first, so a writer stopped at any point leaves
534
+ * either no lease or one with an owner. A held lease always has an owner record,
535
+ * here and in the earlier create-then-write protocol, so a rename that replaces
536
+ * an empty lease directory never takes a held lease. A lease with any entry makes
537
+ * the rename fail and the caller reports it busy.
538
+ */
539
+ function claimLease(lease: string, parent: DirectoryIdentity, owner: LeaseOwner): boolean {
540
+ const claim = `${lease}.claim-${process.pid}-${owner.owner_id}`;
541
+ try {
542
+ mkdirSync(claim, { mode: stateDirMode() });
543
+ assertDirectoryWithinParent(claim, parent);
544
+ const claimed = captureDirectoryIdentity(claim);
545
+ writeOwner(join(claim, "owner.json"), owner);
546
+ syncDirectory(claimed);
547
+ renameSync(claim, lease);
548
+ } catch {
549
+ return false;
550
+ } finally {
525
551
  try {
526
- mkdirSync(lease, { mode: stateDirMode() });
527
- assertDirectoryWithinParent(lease, parent);
528
- claimed = captureDirectoryIdentity(lease);
529
- writeOwner(join(lease, "owner.json"), owner);
530
- syncDirectory(claimed);
531
- return owner;
552
+ rmSync(claim, { recursive: true, force: true });
532
553
  } catch {
533
- if (claimed) {
534
- try {
535
- assertDirectoryIdentity(claimed);
536
- rmSync(lease, { recursive: true, force: false });
537
- } catch {
538
- // Never remove a lease after its directory identity changes.
539
- }
540
- }
541
- if (recoverStaleLease(lease, DURABLE_HISTORY_LEASE_STALE_MS)) continue;
542
- throw new Error(`durable history lease busy: ${lease}`);
554
+ // A leftover claim directory never blocks a lease.
543
555
  }
544
556
  }
545
- throw new Error(`durable history lease busy: ${lease}`);
557
+ try {
558
+ assertDirectoryWithinParent(lease, parent);
559
+ const current = JSON.parse(completeJsonl(join(lease, "owner.json"))) as LeaseOwner;
560
+ if (current.owner_id !== owner.owner_id) return false;
561
+ syncDirectory(parent);
562
+ return true;
563
+ } catch {
564
+ releaseOwnedLease(lease, owner);
565
+ return false;
566
+ }
546
567
  }
547
568
 
548
569
  function writeOwner(path: string, owner: LeaseOwner): void {
@@ -595,7 +616,7 @@ function recoverStaleLease(lease: string, staleMs: number): boolean {
595
616
  try {
596
617
  assertDirectoryIdentity(leaseIdentity);
597
618
  if (completeJsonl(ownerPath) !== raw) return false;
598
- rmSync(lease, { recursive: true, force: false });
619
+ retireLease(lease, randomUUID());
599
620
  return true;
600
621
  } catch {
601
622
  return false;
@@ -606,12 +627,23 @@ function releaseOwnedLease(lease: string, owner: LeaseOwner): void {
606
627
  try {
607
628
  const current = JSON.parse(completeJsonl(join(lease, "owner.json"))) as LeaseOwner;
608
629
  if (current.owner_id !== owner.owner_id) return;
609
- rmSync(lease, { recursive: true, force: false });
630
+ retireLease(lease, owner.owner_id);
610
631
  } catch {
611
632
  // A missing or replaced lease does not belong to this operation.
612
633
  }
613
634
  }
614
635
 
636
+ /**
637
+ * Move a lease aside before deleting it. Its path goes from held to absent in
638
+ * one rename and is never an empty directory that a concurrent claim could
639
+ * replace while a recursive removal is still running.
640
+ */
641
+ function retireLease(lease: string, tag: string): void {
642
+ const retired = `${lease}.retired-${process.pid}-${tag}`;
643
+ renameSync(lease, retired);
644
+ rmSync(retired, { recursive: true, force: true });
645
+ }
646
+
615
647
  function processIsAlive(pid: number): boolean {
616
648
  try {
617
649
  process.kill(pid, 0);
@@ -338,23 +338,20 @@ export async function withFamilyLease<T>(
338
338
  let owner: LeaseOwner | undefined;
339
339
  while (true) {
340
340
  try {
341
- mkdirSync(lease, { mode: stateDirMode() });
342
- const leaseIdentity = captureDirectoryIdentity(lease);
343
- owner = {
344
- owner_id: randomUUID(),
345
- pid: process.pid,
346
- acquired_at: new Date().toISOString(),
347
- };
348
- writeNewFile(join(lease, "owner.json"), Buffer.from(`${JSON.stringify(owner)}\n`));
349
- assertDirectoryIdentity(leaseIdentity);
341
+ owner = claimLease(lease);
350
342
  break;
351
343
  } catch (error) {
352
- if (pathExists(lease) && !safeLeaseDirectory(lease)) {
344
+ if (leaseHasWrongType(lease)) {
353
345
  throw new HarneryLogLeaseError("wrong_type", `log lease is not a directory: ${lease}`);
354
346
  }
355
347
  if (recoverStaleLease(lease, policy.lease_stale_ms)) continue;
356
348
  if (Date.now() - started >= policy.lease_timeout_ms) {
357
- throw new HarneryLogLeaseError("timeout", `timed out acquiring log lease: ${lease}`);
349
+ throw new HarneryLogLeaseError(
350
+ "timeout",
351
+ leaseHasValidOwner(lease)
352
+ ? `timed out acquiring log lease: ${lease}`
353
+ : `timed out acquiring log lease with no valid owner; an interrupted writer left it, and it is never recovered automatically: ${lease}`,
354
+ );
358
355
  }
359
356
  await new Promise((resolve) => setTimeout(resolve, policy.lease_retry_ms));
360
357
  void error;
@@ -367,6 +364,41 @@ export async function withFamilyLease<T>(
367
364
  }
368
365
  }
369
366
 
367
+ /**
368
+ * Publish a complete lease in one rename. The owner record is written and synced
369
+ * in a private claim directory first, so a writer stopped at any point leaves
370
+ * either no lease or one with an owner. A held lease always has an owner record,
371
+ * here and in the earlier create-then-write protocol, so a rename that replaces
372
+ * an empty lease directory never takes a held lease. A lease with any entry makes
373
+ * the rename fail, and the caller waits or recovers as before.
374
+ */
375
+ function claimLease(lease: string): LeaseOwner {
376
+ const owner: LeaseOwner = {
377
+ owner_id: randomUUID(),
378
+ pid: process.pid,
379
+ acquired_at: new Date().toISOString(),
380
+ };
381
+ const content = `${JSON.stringify(owner)}\n`;
382
+ const claim = `${lease}.claim-${process.pid}-${owner.owner_id}`;
383
+ try {
384
+ mkdirSync(claim, { mode: stateDirMode() });
385
+ writeNewFile(join(claim, "owner.json"), Buffer.from(content));
386
+ renameSync(claim, lease);
387
+ } finally {
388
+ rmSync(claim, { recursive: true, force: true });
389
+ }
390
+ try {
391
+ captureDirectoryIdentity(lease);
392
+ if (readRegularFile(join(lease, "owner.json"), 4_096).toString("utf8") !== content) {
393
+ throw new Error(`log lease owner changed during claim: ${lease}`);
394
+ }
395
+ } catch (error) {
396
+ releaseOwnedLease(lease, owner);
397
+ throw error;
398
+ }
399
+ return owner;
400
+ }
401
+
370
402
  function shouldRotate(
371
403
  active: string,
372
404
  incoming: readonly Buffer[],
@@ -476,6 +508,22 @@ interface LeaseOwner {
476
508
  acquired_at: string;
477
509
  }
478
510
 
511
+ function leaseHasValidOwner(lease: string): boolean {
512
+ try {
513
+ const owner = JSON.parse(
514
+ readRegularFile(join(lease, "owner.json"), 4_096).toString("utf8"),
515
+ ) as LeaseOwner;
516
+ return (
517
+ typeof owner?.owner_id === "string" &&
518
+ owner.owner_id.length > 0 &&
519
+ Number.isSafeInteger(owner.pid) &&
520
+ Number.isFinite(Date.parse(owner.acquired_at))
521
+ );
522
+ } catch {
523
+ return false;
524
+ }
525
+ }
526
+
479
527
  function recoverStaleLease(lease: string, maximumAgeMs: number): boolean {
480
528
  let identity: DirectoryIdentity;
481
529
  let raw: string;
@@ -504,7 +552,7 @@ function recoverStaleLease(lease: string, maximumAgeMs: number): boolean {
504
552
  try {
505
553
  assertDirectoryIdentity(identity);
506
554
  if (readRegularFile(join(lease, "owner.json"), 4_096).toString("utf8") !== raw) return false;
507
- rmSync(lease, { recursive: true, force: false });
555
+ retireLease(lease, randomUUID());
508
556
  return true;
509
557
  } catch {
510
558
  return false;
@@ -517,12 +565,23 @@ function releaseOwnedLease(lease: string, owner: LeaseOwner): void {
517
565
  readRegularFile(join(lease, "owner.json"), 4_096).toString("utf8"),
518
566
  ) as LeaseOwner;
519
567
  if (current.owner_id !== owner.owner_id) return;
520
- rmSync(lease, { recursive: true, force: false });
568
+ retireLease(lease, owner.owner_id);
521
569
  } catch {
522
570
  // A missing or replaced lease does not belong to this operation.
523
571
  }
524
572
  }
525
573
 
574
+ /**
575
+ * Move a lease aside before deleting it. Its path goes from held to absent in
576
+ * one rename and is never an empty directory that a concurrent claim could
577
+ * replace while a recursive removal is still running.
578
+ */
579
+ function retireLease(lease: string, tag: string): void {
580
+ const retired = `${lease}.retired-${process.pid}-${tag}`;
581
+ renameSync(lease, retired);
582
+ rmSync(retired, { recursive: true, force: true });
583
+ }
584
+
526
585
  function processIsAlive(pid: number): boolean {
527
586
  try {
528
587
  process.kill(pid, 0);
@@ -699,11 +758,14 @@ function nativeWriteSync(fd: number, buffer: Buffer, offset: number, length: num
699
758
  return writeSync(fd, buffer, offset, length);
700
759
  }
701
760
 
702
- function safeLeaseDirectory(path: string): boolean {
761
+ /** One lstat, so a lease released between checks reads as absent, not as a wrong type. */
762
+ function leaseHasWrongType(path: string): boolean {
703
763
  try {
704
- return !lstatSync(path).isSymbolicLink() && lstatSync(path).isDirectory();
705
- } catch {
706
- return false;
764
+ const stat = lstatSync(path);
765
+ return stat.isSymbolicLink() || !stat.isDirectory();
766
+ } catch (error) {
767
+ if ((error as NodeJS.ErrnoException).code === "ENOENT") return false;
768
+ throw error;
707
769
  }
708
770
  }
709
771
 
@@ -17,6 +17,7 @@ import { performance } from "node:perf_hooks";
17
17
  import { fileURLToPath } from "node:url";
18
18
  import { collectCoordinationHealthSnapshot } from "../agents/health.ts";
19
19
  import {
20
+ type ProcessGenerationSignalDependencies,
20
21
  type ProcessGenerationVerdict,
21
22
  processStartToken,
22
23
  signalIfSameGeneration,
@@ -185,9 +186,7 @@ export interface SupervisorStopResult extends SupervisorStatus {
185
186
 
186
187
  export function requestSupervisorStop(
187
188
  coordRootRaw: string,
188
- dependencies: SupervisorProcessDependencies & {
189
- kill?: (pid: number, signal: NodeJS.Signals | number) => unknown;
190
- } = {},
189
+ dependencies: SupervisorProcessDependencies & ProcessGenerationSignalDependencies = {},
191
190
  ): SupervisorStopResult {
192
191
  const coordRoot = resolve(coordRootRaw);
193
192
  const paths = supervisorPaths(coordRoot);
@@ -20,7 +20,7 @@ import {
20
20
  import { stateDirMode, stateFileMode } from "../storage/modes.ts";
21
21
  import { readWorkflowApproval } from "../workflow/approvals.ts";
22
22
  import { stableDigest } from "../workflow/durable-record.ts";
23
- import { deriveRunFailureClass, readWorkflowProof } from "../workflow/proof.ts";
23
+ import { readWorkflowProof } from "../workflow/proof.ts";
24
24
  import { readWorkflowRunManifest, workflowScriptDigest } from "../workflow/run-state.ts";
25
25
  import { WORKFLOW_TRANSCRIPT_EVENT_BYTES, workflowTranscriptPath } from "../workflow/transcript.ts";
26
26
  import { isWorkspaceBoundExecutionEvidence } from "../workflow/workspaces/validate.ts";
@@ -718,14 +718,14 @@ function inspectAttempt(
718
718
  proof.acceptance.summary.unknown === 0
719
719
  ? "succeeded"
720
720
  : "failed";
721
- // Re-derive no-effect eligibility from every participant rather than
722
- // trusting a persisted transport class. Missing evidence stays charged.
723
- // The script's explicit decision authority remains a separate path.
721
+ // A failed attempt the run classified as uninformative about the work is
722
+ // uncharged (ADR 0046). Absent class ⇒ charged, as before. Read only for a
723
+ // failed attempt: a succeeded run never carries a class.
724
724
  if (
725
725
  attempt.status === "failed" &&
726
- (proof.run.class === "decision" ||
727
- ((proof.run.class === "environment" || proof.run.class === "upstream") &&
728
- deriveRunFailureClass(proof.run.status, proof.agents) === proof.run.class))
726
+ (proof.run.class === "environment" ||
727
+ proof.run.class === "upstream" ||
728
+ proof.run.class === "decision")
729
729
  ) {
730
730
  attempt.uncharged = proof.run.class;
731
731
  if (proof.run.class === "decision" && typeof proof.run.decision_id === "string") {
@@ -45,16 +45,7 @@ import { freezeWorkflowAttemptContext } from "./attempt-context.ts";
45
45
  import { type BillingProbe, probeBilling } from "./billing.ts";
46
46
  import { stableDigest } from "./durable-record.ts";
47
47
  import { evidencePreflightError } from "./evidence-preflight.ts";
48
- import {
49
- canRetryFailedSpawn,
50
- observeSpawnFailure,
51
- resolveFailureEvidence,
52
- } from "./failure-evidence.ts";
53
- import {
54
- endWorkflowChildSessionV3,
55
- recordWorkflowChildTerminalFailureV3,
56
- startWorkflowChildSessionV3,
57
- } from "./live-session-v3.ts";
48
+ import { endWorkflowChildSessionV3, startWorkflowChildSessionV3 } from "./live-session-v3.ts";
58
49
  import {
59
50
  buildWorkflowProof,
60
51
  createEvidenceRecord,
@@ -1038,38 +1029,22 @@ async function executeWorkflow(
1038
1029
  });
1039
1030
  agentProof.access = access;
1040
1031
 
1041
- let childGenerationId: string | undefined;
1042
1032
  let recordAgentEnd: (() => void) | undefined;
1043
1033
  const finalizeChild = (): void => {
1044
1034
  try {
1045
- if (childGenerationId)
1046
- endWorkflowChildSessionV3({
1047
- coordRoot: opts.coordRoot,
1048
- instanceId: childInstanceId(runId, id),
1049
- runId,
1050
- agentId: id,
1051
- adapter,
1052
- cleanExit: agentProof.status === "succeeded",
1053
- expectedGenerationId: childGenerationId,
1054
- });
1055
- // Result publication is the resume-cache boundary, after canonical custody.
1035
+ // Record the canonical terminal first. agent.end is the resume-cache
1036
+ // boundary, so a child whose end was not recorded never reports success.
1037
+ endWorkflowChildSessionV3({
1038
+ coordRoot: opts.coordRoot,
1039
+ instanceId: childInstanceId(runId, id),
1040
+ runId,
1041
+ agentId: id,
1042
+ adapter,
1043
+ cleanExit: agentProof.status === "succeeded",
1044
+ });
1056
1045
  recordAgentEnd?.();
1057
1046
  } catch (error) {
1058
- let reason = `V3 terminal recording failed: ${(error as Error).message}`;
1059
- try {
1060
- if (childGenerationId)
1061
- recordWorkflowChildTerminalFailureV3({
1062
- coordRoot: opts.coordRoot,
1063
- instanceId: childInstanceId(runId, id),
1064
- runId,
1065
- agentId: id,
1066
- adapter,
1067
- expectedGenerationId: childGenerationId,
1068
- reason,
1069
- });
1070
- } catch (visibilityError) {
1071
- reason += `; blocked lifecycle recording failed: ${(visibilityError as Error).message}`;
1072
- }
1047
+ const reason = `V3 terminal recording failed: ${(error as Error).message}`;
1073
1048
  const failure = new Error(agentProof.error ? `${agentProof.error}; ${reason}` : reason, {
1074
1049
  cause: error,
1075
1050
  });
@@ -1102,7 +1077,7 @@ async function executeWorkflow(
1102
1077
  // Headless children do not reliably fire adapter hooks. The engine owns
1103
1078
  // their canonical V3 start so every dispatched child has an auditable
1104
1079
  // generation before it begins work.
1105
- const child = startWorkflowChildSessionV3({
1080
+ startWorkflowChildSessionV3({
1106
1081
  coordRoot: opts.coordRoot,
1107
1082
  instanceId: childInstanceId(runId, id),
1108
1083
  runId,
@@ -1111,13 +1086,10 @@ async function executeWorkflow(
1111
1086
  label,
1112
1087
  model: agentOpts.model,
1113
1088
  });
1114
- if (!child.v3_generation_id) throw new Error("workflow_child_v3_generation_missing");
1115
- childGenerationId = child.v3_generation_id;
1116
1089
 
1117
1090
  let attemptPrompt = dispatchPrompt;
1118
1091
  let last: SpawnResult | null = null;
1119
1092
  let agentCostUsd = 0;
1120
- let informativeAttempt: ReturnType<typeof observeSpawnFailure> | undefined;
1121
1093
  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
1122
1094
  last = await spawner({
1123
1095
  prompt: attemptPrompt,
@@ -1140,30 +1112,14 @@ async function executeWorkflow(
1140
1112
  costUsd += last.costUsd ?? 0;
1141
1113
 
1142
1114
  if (!last.ok) {
1143
- const observed =
1144
- last.failureEvidence === undefined
1145
- ? observeSpawnFailure({}, last.text, "final_message")
1146
- : resolveFailureEvidence(last.failureEvidence);
1147
- agentProof.failure_evidence = informativeAttempt
1148
- ? {
1149
- ...observed,
1150
- effect: { state: "unknown" },
1151
- partial_output: informativeAttempt.partial_output,
1152
- }
1153
- : observed;
1154
- transcript("agent.attempt_failed", {
1155
- id,
1156
- attempt,
1157
- error: last.error,
1158
- failure_evidence: observed,
1159
- result_digest: digestResult(last.text, "text"),
1160
- text: last.text,
1161
- });
1162
- if (!canRetryFailedSpawn({ ...last, failureEvidence: agentProof.failure_evidence }))
1163
- break;
1164
- continue;
1115
+ transcript("agent.attempt_failed", { id, attempt, error: last.error });
1116
+ // ADR 0046: an environment failure (the binary was absent) cannot be
1117
+ // helped by retrying an unchanged environment, so stop the in-agent
1118
+ // retry too — not just the outer attempt/replan budget. An upstream
1119
+ // refusal keeps retrying here: the vendor may recover mid-loop.
1120
+ if (last.class === "environment") break;
1121
+ continue; // spawn-level failure: retry with the original prompt
1165
1122
  }
1166
- informativeAttempt = observeSpawnFailure({}, last.text, "final_message");
1167
1123
  if (!agentOpts.schema) {
1168
1124
  agentProof.status = "succeeded";
1169
1125
  agentProof.cost_usd =
@@ -1217,14 +1173,7 @@ async function executeWorkflow(
1217
1173
  return parsed.value;
1218
1174
  }
1219
1175
 
1220
- agentProof.failure_evidence = informativeAttempt;
1221
- transcript("agent.schema_retry", {
1222
- id,
1223
- attempt,
1224
- problems,
1225
- rejected_text: last.text,
1226
- result_digest: digestResult(last.text, "text"),
1227
- });
1176
+ transcript("agent.schema_retry", { id, attempt, problems });
1228
1177
  // Keep the original cache identity while making the actual output
1229
1178
  // contract explicit. A full bounded copy of the rejected value lets the
1230
1179
  // child repair omissions instead of reconstructing an unseen object.
@@ -1260,7 +1209,6 @@ async function executeWorkflow(
1260
1209
  try {
1261
1210
  finalizeChild();
1262
1211
  } finally {
1263
- // A slot is disposable; unresolved custody remains visible and failed.
1264
1212
  release();
1265
1213
  }
1266
1214
  }
@@ -39,12 +39,23 @@ export {
39
39
  rollupAcceptance,
40
40
  writeWorkflowProof,
41
41
  } from "./proof.ts";
42
- export type { WorkflowRunManifest } from "./run-state.ts";
42
+ export type {
43
+ WorkflowRecoveryAuthorization,
44
+ WorkflowRecoveryIntent,
45
+ WorkflowRecoveryRecord,
46
+ } from "./recovery-audit.ts";
47
+ export { beginWorkflowRecovery, readWorkflowRecoveryLedger } from "./recovery-audit.ts";
48
+ export type {
49
+ WorkflowResumeRecoveryOptions,
50
+ WorkflowResumeRecoveryResult,
51
+ WorkflowRunManifest,
52
+ } from "./run-state.ts";
43
53
  export {
44
54
  acquireWorkflowResumeLease,
45
55
  assertWorkflowRunResumable,
46
56
  assertWorkflowScriptUnchanged,
47
57
  readWorkflowRunManifest,
58
+ recoverWorkflowResumeLease,
48
59
  WORKFLOW_RUN_MANIFEST_SCHEMA_VERSION,
49
60
  workflowRunManifestPath,
50
61
  workflowScriptDigest,
@@ -1,16 +1,9 @@
1
- import { randomUUID } from "node:crypto";
2
- import { existsSync, linkSync, readdirSync, readFileSync, renameSync, unlinkSync } from "node:fs";
3
- import { basename, dirname } from "node:path";
4
1
  import { type Adapter, adapterFromPlatform } from "../adapter.ts";
5
- import {
6
- recordLiveLifecycleChangeV3,
7
- recordLiveTaskChangeV3,
8
- } from "../agents/live-authority-v3.ts";
2
+ import { recordLiveTaskChangeV3 } from "../agents/live-authority-v3.ts";
9
3
  import { endSessionExplicitV3 } from "../agents/session-finalizer-v3.ts";
10
- import { heartbeatPath } from "../agents/state/heartbeat-reader.ts";
11
4
  import {
5
+ clearCoordinationCache,
12
6
  type Heartbeat,
13
- readHeartbeat,
14
7
  setIdentityCache,
15
8
  } from "../agents/state/heartbeat-writer.ts";
16
9
  import { readLiveCoordinationRow } from "../agents/state/live-coordination-view.ts";
@@ -19,17 +12,9 @@ import { assignName } from "../agents/state/names.ts";
19
12
  import { coordFreshnessSeconds } from "../config.ts";
20
13
  import { ensureEventLedgerV3 } from "../events/v3/bootstrap.ts";
21
14
  import {
22
- readCoordinationViewV3,
23
- requireAuthoritySafeCoordinationViewV3,
24
- } from "../events/v3/coordination-view.ts";
25
- import { withGenerationCustodyV3 } from "../events/v3/generation-custody.ts";
26
- import {
27
- liveInstanceIdV3,
28
15
  recordLiveHookSignalV3,
29
16
  resolveLiveEventLedgerRouteV3,
30
17
  } from "../events/v3/live-routing.ts";
31
- import { readHookProducerStateV3 } from "../events/v3/producers/recorder.ts";
32
- import { fsyncParentDirectory } from "./durable-record.ts";
33
18
  import { stableScopeId } from "./scope-id.ts";
34
19
 
35
20
  export interface WorkflowChildSessionV3Input {
@@ -92,45 +77,16 @@ export function startWorkflowChildSessionV3(input: WorkflowChildSessionV3Input):
92
77
  }
93
78
  const projected = readLiveCoordinationRow(input.coordRoot, input.instanceId);
94
79
  if (!projected) throw new Error("workflow_child_v3_projection_missing");
95
- const own = readHookProducerStateV3(input.coordRoot, adapter, sessionId);
96
- if (!own || own.terminal || own.generation_id !== projected.v3_generation_id) {
97
- throw new Error("workflow_child_v3_start_generation_changed");
98
- }
99
80
  return projected;
100
81
  }
101
82
 
102
83
  /** Record the authoritative terminal before removing the disposable coordination cache. */
103
84
  export function endWorkflowChildSessionV3(
104
- input: WorkflowChildSessionV3Input & { cleanExit: boolean; expectedGenerationId: string },
105
- ): void {
106
- withGenerationCustodyV3(input.coordRoot, [liveInstanceIdV3(input.instanceId)], () =>
107
- endExpectedChild(input),
108
- );
109
- }
110
-
111
- function endExpectedChild(
112
- input: WorkflowChildSessionV3Input & { cleanExit: boolean; expectedGenerationId: string },
85
+ input: WorkflowChildSessionV3Input & { cleanExit: boolean },
113
86
  ): void {
114
87
  requireV3Route(input.coordRoot);
115
- const view = requireAuthoritySafeCoordinationViewV3(readCoordinationViewV3(input.coordRoot));
116
- const instanceId = liveInstanceIdV3(input.instanceId);
117
- const current = view.instances[instanceId];
118
- const terminal = view.terminal_generations[input.expectedGenerationId];
119
- if (terminal?.instance_id === instanceId) recoverHeldCaches(input);
120
- if (current && current.generation_id !== input.expectedGenerationId) {
121
- throw new Error("workflow_child_v3_generation_changed");
122
- }
123
- if (!current) {
124
- if (!terminal || terminal.instance_id !== instanceId)
125
- throw new Error("workflow_child_v3_generation_missing");
126
- if (terminal.terminal?.outcome !== (input.cleanExit ? "succeeded" : "failed")) {
127
- throw new Error("workflow_child_v3_terminal_outcome_changed");
128
- }
129
- clearExpectedCache(input);
130
- return;
131
- }
132
88
  const row = readLiveCoordinationRow(input.coordRoot, input.instanceId);
133
- if (!row?.v3_instance_id || row.v3_generation_id !== input.expectedGenerationId) {
89
+ if (!row?.v3_instance_id || !row.v3_generation_id) {
134
90
  throw new Error("workflow_child_v3_generation_missing");
135
91
  }
136
92
  const result = endSessionExplicitV3({
@@ -145,104 +101,7 @@ function endExpectedChild(
145
101
  `workflow_child_v3_end_failed:${result.state}${"reason" in result ? `:${result.reason}` : ""}`,
146
102
  );
147
103
  }
148
- clearExpectedCache(input);
149
- }
150
-
151
- /** Persist unresolved terminal custody only for the generation this dispatch owns. */
152
- export function recordWorkflowChildTerminalFailureV3(
153
- input: WorkflowChildSessionV3Input & { expectedGenerationId: string; reason: string },
154
- ): void {
155
- withGenerationCustodyV3(input.coordRoot, [liveInstanceIdV3(input.instanceId)], () => {
156
- const row = readLiveCoordinationRow(input.coordRoot, input.instanceId);
157
- if (!row || row.v3_generation_id !== input.expectedGenerationId) return;
158
- recordLiveLifecycleChangeV3({
159
- coordRoot: input.coordRoot,
160
- owner: input.instanceId,
161
- nativeSessionId: input.sessionId ?? input.instanceId,
162
- adapter: workflowAdapter(input.adapter),
163
- expectedGenerationId: input.expectedGenerationId as `gen_${string}`,
164
- state: "blocked",
165
- reason: input.reason,
166
- });
167
- });
168
- }
169
-
170
- function heldPrefix(input: WorkflowChildSessionV3Input & { expectedGenerationId: string }): string {
171
- if (!/^gen_[a-zA-Z0-9._-]{1,128}$/.test(input.expectedGenerationId))
172
- throw new Error("workflow_child_v3_generation_invalid");
173
- return `${basename(heartbeatPath(input.coordRoot, input.instanceId))}.terminal-${input.expectedGenerationId}-`;
174
- }
175
-
176
- function recoverHeldCaches(
177
- input: WorkflowChildSessionV3Input & { expectedGenerationId: string },
178
- ): void {
179
- const path = heartbeatPath(input.coordRoot, input.instanceId);
180
- if (!existsSync(dirname(path))) return;
181
- const prefix = heldPrefix(input);
182
- for (const name of readdirSync(dirname(path)).filter((value) => value.startsWith(prefix))) {
183
- finishHeldCache(`${dirname(path)}/${name}`, path, input.expectedGenerationId);
184
- }
185
- }
186
-
187
- function clearExpectedCache(
188
- input: WorkflowChildSessionV3Input & { expectedGenerationId: string },
189
- ): void {
190
- recoverHeldCaches(input);
191
- if (
192
- readHeartbeat(input.coordRoot, input.instanceId)?.v3_generation_id !==
193
- input.expectedGenerationId
194
- )
195
- return;
196
- const path = heartbeatPath(input.coordRoot, input.instanceId);
197
- const held = `${dirname(path)}/${heldPrefix(input)}${randomUUID()}`;
198
- try {
199
- // A unique held name pins the actual bytes atomically. Checking before
200
- // unlink alone could consume a replacement published at this boundary.
201
- renameSync(path, held);
202
- fsyncParentDirectory(path);
203
- } catch (error) {
204
- if ((error as NodeJS.ErrnoException).code === "ENOENT") return;
205
- throw error;
206
- }
207
- finishHeldCache(held, path, input.expectedGenerationId);
208
- }
209
-
210
- function finishHeldCache(held: string, path: string, expectedGenerationId: string): void {
211
- let removed = false;
212
- let cleanupFailed = false;
213
- let cleanupError: unknown;
214
- try {
215
- const cache = JSON.parse(readFileSync(held, "utf8")) as Heartbeat;
216
- if (cache.v3_generation_id === expectedGenerationId) {
217
- unlinkSync(held);
218
- fsyncParentDirectory(held);
219
- removed = true;
220
- }
221
- } catch (error) {
222
- cleanupFailed = true;
223
- cleanupError = error;
224
- }
225
- if (!removed) {
226
- try {
227
- // Never overwrite a later publication. Keep the held evidence when
228
- // another cache already owns the live name.
229
- linkSync(held, path);
230
- fsyncParentDirectory(path);
231
- unlinkSync(held);
232
- fsyncParentDirectory(held);
233
- } catch (error) {
234
- if ((error as NodeJS.ErrnoException).code !== "EEXIST") {
235
- if (cleanupFailed) {
236
- throw new AggregateError(
237
- [cleanupError, error],
238
- `workflow_child_v3_cache_cleanup_and_restore_failed: ${String(cleanupError)}; restoration: ${String(error)}`,
239
- );
240
- }
241
- throw error;
242
- }
243
- }
244
- }
245
- if (cleanupFailed) throw cleanupError;
104
+ clearCoordinationCache(input.coordRoot, input.instanceId);
246
105
  }
247
106
 
248
107
  function requireV3Route(coordRoot: string) {