@lostgradient/weft 0.19.0 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. package/README.md +2 -2
  2. package/dist/cli/generated/operation-catalog.snapshot.json +1334 -0
  3. package/dist/cli/generated/operation-client.generated.d.ts +9 -2
  4. package/dist/cli/generated/operation-client.generated.js +2 -0
  5. package/dist/core/engine/activity-reconciliation.d.ts +2 -2
  6. package/dist/core/engine/activity-reconciliation.js +12 -8
  7. package/dist/core/engine/async-activity-completion.js +12 -4
  8. package/dist/core/engine/async-activity-records.d.ts +11 -1
  9. package/dist/core/engine/async-activity-records.js +4 -3
  10. package/dist/core/engine/attributes-tags.d.ts +11 -1
  11. package/dist/core/engine/attributes-tags.js +4 -3
  12. package/dist/core/engine/bulk-operations-purge.js +5 -2
  13. package/dist/core/engine/bulk-operations.js +17 -5
  14. package/dist/core/engine/callback-creators-bundles.js +1 -1
  15. package/dist/core/engine/callback-creators-core.js +4 -2
  16. package/dist/core/engine/checkpoint-io.js +1 -1
  17. package/dist/core/engine/child-workflow.js +2 -1
  18. package/dist/core/engine/completed-review-storage.js +1 -1
  19. package/dist/core/engine/condition-waiters.d.ts +45 -3
  20. package/dist/core/engine/condition-waiters.js +12 -3
  21. package/dist/core/engine/disposal.js +6 -0
  22. package/dist/core/engine/engine-internal-types.d.ts +12 -4
  23. package/dist/core/engine/fenced-write.d.ts +46 -15
  24. package/dist/core/engine/fenced-write.js +39 -6
  25. package/dist/core/engine/finalizer-state.d.ts +3 -1
  26. package/dist/core/engine/handle-result.d.ts +49 -1
  27. package/dist/core/engine/handle-result.js +95 -20
  28. package/dist/core/engine/index.d.ts +13 -5
  29. package/dist/core/engine/index.js +155 -5
  30. package/dist/core/engine/inline-launch-queue.js +9 -0
  31. package/dist/core/engine/internals.d.ts +73 -0
  32. package/dist/core/engine/invoke-update-handler.d.ts +19 -0
  33. package/dist/core/engine/invoke-update-handler.js +7 -0
  34. package/dist/core/engine/lease-deposition.d.ts +132 -0
  35. package/dist/core/engine/lease-deposition.js +31 -0
  36. package/dist/core/engine/lease-errors.d.ts +80 -9
  37. package/dist/core/engine/lease-errors.js +28 -2
  38. package/dist/core/engine/lifecycle/resume.js +3 -1
  39. package/dist/core/engine/lifecycle/standalone-claim-acquire.d.ts +33 -0
  40. package/dist/core/engine/lifecycle/standalone-claim-acquire.js +25 -0
  41. package/dist/core/engine/lifecycle/start-commit.js +20 -6
  42. package/dist/core/engine/lifecycle/transition.d.ts +15 -1
  43. package/dist/core/engine/lifecycle/transition.js +6 -3
  44. package/dist/core/engine/operations-activity.js +1 -1
  45. package/dist/core/engine/operations-time.js +21 -7
  46. package/dist/core/engine/owner-side-signal-poll.d.ts +132 -0
  47. package/dist/core/engine/owner-side-signal-poll.js +22 -0
  48. package/dist/core/engine/owner-side-update-poll.d.ts +116 -0
  49. package/dist/core/engine/owner-side-update-poll.js +22 -0
  50. package/dist/core/engine/ownership-bootstrap.d.ts +280 -0
  51. package/dist/core/engine/ownership-bootstrap.js +89 -0
  52. package/dist/core/engine/ownership-mode-marker.d.ts +82 -0
  53. package/dist/core/engine/ownership-mode-marker.js +52 -0
  54. package/dist/core/engine/ownership-options.d.ts +22 -6
  55. package/dist/core/engine/ownership-options.js +42 -12
  56. package/dist/core/engine/pending-updates.js +35 -24
  57. package/dist/core/engine/queries.d.ts +70 -0
  58. package/dist/core/engine/queries.js +58 -4
  59. package/dist/core/engine/schedules.js +1 -1
  60. package/dist/core/engine/signals.d.ts +11 -1
  61. package/dist/core/engine/signals.js +11 -7
  62. package/dist/core/engine/sleep-timer-acknowledgements.d.ts +68 -0
  63. package/dist/core/engine/sleep-timer-acknowledgements.js +19 -0
  64. package/dist/core/engine/storage-io.d.ts +58 -10
  65. package/dist/core/engine/storage-io.js +27 -9
  66. package/dist/core/engine/termination/cleanup.d.ts +5 -4
  67. package/dist/core/engine/termination/complete.js +23 -4
  68. package/dist/core/engine/termination/finalizer-claim.js +4 -4
  69. package/dist/core/engine/termination/state-commit-callbacks.d.ts +15 -0
  70. package/dist/core/engine/termination/state-commit-callbacks.js +0 -0
  71. package/dist/core/engine/termination/suspend.js +1 -1
  72. package/dist/core/engine/update-validation.d.ts +28 -0
  73. package/dist/core/engine/update-validation.js +37 -0
  74. package/dist/core/engine/updates.d.ts +17 -16
  75. package/dist/core/engine/updates.js +24 -47
  76. package/dist/core/engine/wake-ownership-check.d.ts +86 -0
  77. package/dist/core/engine/wake-ownership-check.js +19 -0
  78. package/dist/core/engine/wake-ownership-guard.d.ts +46 -0
  79. package/dist/core/engine/wake-ownership-guard.js +23 -0
  80. package/dist/core/engine/workflow-claim-codec.d.ts +64 -0
  81. package/dist/core/engine/workflow-claim-codec.js +44 -0
  82. package/dist/core/engine/workflow-claim-cooldown.d.ts +60 -0
  83. package/dist/core/engine/workflow-claim-cooldown.js +17 -0
  84. package/dist/core/engine/workflow-claim-fold.d.ts +82 -0
  85. package/dist/core/engine/workflow-claim-fold.js +53 -0
  86. package/dist/core/engine/workflow-claim-metrics.d.ts +131 -0
  87. package/dist/core/engine/workflow-claim-metrics.js +62 -0
  88. package/dist/core/engine/workflow-claim-reclaim-scan.d.ts +96 -0
  89. package/dist/core/engine/workflow-claim-reclaim-scan.js +66 -0
  90. package/dist/core/engine/workflow-claim-reclaim-target.d.ts +107 -0
  91. package/dist/core/engine/workflow-claim-reclaim-target.js +155 -0
  92. package/dist/core/engine/workflow-claim-registry.d.ts +227 -0
  93. package/dist/core/engine/workflow-claim-registry.js +198 -0
  94. package/dist/core/engine/workflow-claim-renewal-interval.d.ts +50 -0
  95. package/dist/core/engine/workflow-claim-renewal-interval.js +132 -0
  96. package/dist/core/engine/workflow-claim-renewal-subpasses.d.ts +265 -0
  97. package/dist/core/engine/workflow-claim-renewal-subpasses.js +104 -0
  98. package/dist/core/engine/workflow-claim-renewal-task.d.ts +159 -0
  99. package/dist/core/engine/workflow-claim-renewal-task.js +65 -0
  100. package/dist/core/engine/workflow-claim-transitions.d.ts +186 -0
  101. package/dist/core/engine/workflow-claim-transitions.js +120 -0
  102. package/dist/core/inline-execution-strategy.d.ts +20 -0
  103. package/dist/core/inline-execution-strategy.js +15 -4
  104. package/dist/core/scheduler/scheduler-class.js +3 -3
  105. package/dist/core/scheduler/timer-sources.d.ts +13 -0
  106. package/dist/core/scheduler/timer-sources.js +1 -1
  107. package/dist/core/types/options.d.ts +17 -1
  108. package/dist/core/weft-error.d.ts +1 -1
  109. package/dist/core/weft-error.js +2 -1
  110. package/dist/http.js +2 -2
  111. package/dist/index.d.ts +2 -1
  112. package/dist/index.js +8 -0
  113. package/dist/indexeddb.js +1 -1
  114. package/dist/json-schema.js +3 -3
  115. package/dist/observability/index.js +2 -2
  116. package/dist/server/operations/get-task-detail-schema.d.ts +341 -0
  117. package/dist/server/operations/get-task-detail-schema.js +118 -0
  118. package/dist/server/operations/get-task-detail.d.ts +346 -0
  119. package/dist/server/operations/get-task-detail.js +215 -0
  120. package/dist/server/operations/static-registrations.js +3 -0
  121. package/dist/storage/auto.js +1 -1
  122. package/dist/storage/bun-sql.js +113 -0
  123. package/dist/storage/compressed-storage.js +1 -1
  124. package/dist/storage/index.d.ts +38 -35
  125. package/dist/storage/interface.d.ts +87 -125
  126. package/dist/storage/interface.js +1 -1
  127. package/dist/storage/key-encoding.d.ts +51 -0
  128. package/dist/storage/key-encoding.js +13 -0
  129. package/dist/storage/key-prefixes.d.ts +1 -1
  130. package/dist/storage/key-prefixes.js +3 -0
  131. package/dist/storage/lmdb.js +1 -1
  132. package/dist/storage/memory.js +1 -1
  133. package/dist/storage/neon.js +2 -2
  134. package/dist/storage/node-sqlite.js +113 -0
  135. package/dist/storage/ownership-keys.d.ts +55 -0
  136. package/dist/storage/ownership-keys.js +6 -0
  137. package/dist/storage/postgres.js +2 -2
  138. package/dist/storage/resolve.js +1 -1
  139. package/dist/storage/scoped-storage.js +1 -1
  140. package/dist/storage/testing.js +1 -1
  141. package/dist/storage/turso.js +2 -2
  142. package/dist/version.d.ts +1 -1
  143. package/dist/version.js +1 -1
  144. package/dist/web-extension.js +1 -1
  145. package/dist/worker/protocol.js +1 -1
  146. package/package.json +1 -1
@@ -37,6 +37,7 @@ import {
37
37
  untagAll as untagAllWorkflows
38
38
  } from "./bulk-operations.js";
39
39
  import { createTimeOperationCallbacks as createTimeOperationCallbacksForEngine } from "./callback-creators-bundles.js";
40
+ import { createPendingUpdateCallbacks as createPendingUpdateCallbacksForEngine } from "./callback-creators-core.js";
40
41
  import {
41
42
  createBroadcastCallbacks as createBroadcastCallbacksForEngine,
42
43
  createInlineParkingCallbacks as createInlineParkingCallbacksForEngine,
@@ -88,7 +89,8 @@ import { recordFinalizerState } from "./finalizer-state.js";
88
89
  import { getFinalizerStatus as getFinalizerStatusFromInternals } from "./finalizer-status.js";
89
90
  import {
90
91
  createWorkflowHandleWithResultPromise as createWorkflowHandleWithResultPromiseFromInternals,
91
- getWorkflowResultPromise as getWorkflowResultPromiseFromInternals
92
+ getWorkflowResultPromise as getWorkflowResultPromiseFromInternals,
93
+ pollPendingCrossEngineResultWaiters
92
94
  } from "./handle-result.js";
93
95
  import { HANDLE_RESULT_PROMISE, WorkflowHandle } from "./handles.js";
94
96
  import { hasQueuedInlineWorkflowStart } from "./inline-launch-queue.js";
@@ -119,6 +121,12 @@ import { getStreamChunksFromInternals } from "./operations-stream.js";
119
121
  import {
120
122
  handleTimerFired as handleTimerFiredFromInternals
121
123
  } from "./operations-time.js";
124
+ import {
125
+ bootstrapWorkflowLeaseOwnership,
126
+ buildOwnerSideSignalPollTarget
127
+ } from "./ownership-bootstrap.js";
128
+ import { bootstrapOwnershipGates } from "./ownership-mode-marker.js";
129
+ import { processPendingUpdatesForHandlers as processPendingUpdatesForHandlersFromInternals } from "./pending-updates.js";
122
130
  import { assertCompatiblePersistedDataVersion } from "./persisted-data-version.js";
123
131
  import { query as queryWorkflow } from "./queries.js";
124
132
  import {
@@ -154,7 +162,11 @@ import {
154
162
  createSecondInstanceDetectionTick,
155
163
  createSecondInstanceDetector
156
164
  } from "./second-instance-detector.js";
157
- import { signal as signalWorkflow } from "./signals.js";
165
+ import {
166
+ hasBufferedSignal as hasBufferedSignalFromInternals,
167
+ releaseSignalWaiter,
168
+ signal as signalWorkflow
169
+ } from "./signals.js";
158
170
  import {
159
171
  loadScheduleState,
160
172
  loadWorkflowState,
@@ -173,12 +185,14 @@ import {
173
185
  timeoutWorkflow as timeoutWorkflowFromTermination
174
186
  } from "./termination.js";
175
187
  import {
188
+ deliverCoordinatedUpdateToWaiterIfAvailable as deliverCoordinatedUpdateToWaiterIfAvailableFromInternals,
176
189
  getUpdateResult as getUpdateResultFromInternals,
177
190
  submitCoordinatedUpdate as submitCoordinatedUpdateFromInternals,
178
191
  update as updateFromInternals
179
192
  } from "./updates.js";
180
193
  import { isTerminalWorkflowStatus } from "./validation.js";
181
194
  import { coerceScheduleId } from "./validation/schedule.js";
195
+ import { confirmWakeOwnership } from "./wake-ownership-guard.js";
182
196
  import {
183
197
  replayWorkflowFeed,
184
198
  snapshotWorkflowFeedTail,
@@ -209,10 +223,18 @@ export {
209
223
  WorkflowTypeNotRegisteredForRecoveryError
210
224
  } from "./errors.js";
211
225
  export { HANDLE_RESULT_PROMISE, WorkflowHandle } from "./handles.js";
226
+ export {
227
+ WeftWorkflowClaimLostWarning,
228
+ WeftWorkflowWakeDiscardedWarning,
229
+ WORKFLOW_CLAIM_LOST_WARNING_NAME,
230
+ WORKFLOW_WAKE_DISCARDED_WARNING_NAME
231
+ } from "./lease-deposition.js";
212
232
  export {
213
233
  EngineLeaseAcquisitionTimeoutError,
214
234
  EngineLeaseCorruptedError,
215
- EngineLeaseNotHeldError
235
+ EngineLeaseNotHeldError,
236
+ OwnershipModeMismatchError,
237
+ WorkflowClaimUnavailableError
216
238
  } from "./lease-errors.js";
217
239
  export { ScheduleHandle } from "./schedule-handle.js";
218
240
  export {
@@ -257,6 +279,7 @@ export class Engine extends EventTarget {
257
279
  throw new EngineCreateNameMismatchError("workflow", name, definition.name);
258
280
  engine.register(definition);
259
281
  }
282
+ await engine.#bootstrapOwnershipIfConfigured();
260
283
  await engine.#acquireLeaseIfConfigured();
261
284
  if (options.recover !== !1)
262
285
  await engine.recoverAll(options.acknowledgeUnknownWorkflowTypes !== void 0 ? { acknowledgeUnknownWorkflowTypes: options.acknowledgeUnknownWorkflowTypes } : {});
@@ -331,7 +354,7 @@ export class Engine extends EventTarget {
331
354
  onTimerFired: (entry) => handleTimerFiredFromInternals(getInternals(this), entry, this.#createTimeOperationCallbacks()),
332
355
  pollIntervalMs: resolvedOptions.schedulerPollIntervalMs,
333
356
  getNow,
334
- commitTimerCleanup: (operations) => commitFencedEngineWrite(getInternals(this), operations, [], () => Error("Timer cleanup lost its fenced CAS race"))
357
+ commitTimerCleanup: (operations) => commitFencedEngineWrite(getInternals(this), null, operations, [], () => Error("Timer cleanup lost its fenced CAS race"))
335
358
  });
336
359
  getInternals(this).strategy = strategyBundle.strategy;
337
360
  getInternals(this).inlineStrategy = strategyBundle.inlineStrategy;
@@ -359,9 +382,11 @@ export class Engine extends EventTarget {
359
382
  getInternals(this).parkedInlineWorkflows = new Set;
360
383
  getInternals(this).terminalizingWorkflows = new Set;
361
384
  getInternals(this).deliveredPendingUpdateIds = new Map;
385
+ getInternals(this).onRecoveredWorkflowHook = void 0;
362
386
  getInternals(this).cancelHandlersByWorkflow = new Map;
363
387
  getInternals(this).reviewTimerIds = new Map;
364
388
  getInternals(this).pendingWebhooks = new Set;
389
+ getInternals(this).pendingResultPollTimers = new Set;
365
390
  getInternals(this).pendingTimelineEntries = new Map;
366
391
  getInternals(this).pendingAtomicWorkflowCommitSideEffects = new Map;
367
392
  getInternals(this).cleanupIntervalDisposalTracker = null;
@@ -381,7 +406,11 @@ export class Engine extends EventTarget {
381
406
  getInternals(this).secondInstanceDetectionInterval = null;
382
407
  getInternals(this).secondInstanceDetector = null;
383
408
  getInternals(this).leaseManager = null;
409
+ getInternals(this).workflowClaimRegistry = null;
410
+ getInternals(this).workflowClaimRenewalTask = null;
411
+ getInternals(this).workflowClaimMetrics = null;
384
412
  getInternals(this).inFlightLeaseAcquire = null;
413
+ getInternals(this).inFlightOwnershipBootstrap = null;
385
414
  getInternals(this).deposed = !1;
386
415
  getInternals(this).tearDownAfterDeposition = () => {
387
416
  this.#disposeAfterDeposition();
@@ -458,15 +487,81 @@ export class Engine extends EventTarget {
458
487
  internals.inFlightLeaseAcquire = null;
459
488
  }
460
489
  }
490
+ async#bootstrapOwnershipIfConfigured() {
491
+ const internals = getInternals(this), ownershipMode = internals.options.ownershipMode;
492
+ if (ownershipMode === "none")
493
+ return;
494
+ if (internals.disposed)
495
+ throw new EngineDisposedError;
496
+ if (ownershipMode === "lease") {
497
+ await bootstrapOwnershipGates({
498
+ storage: internals.storage,
499
+ ownershipMode,
500
+ getNow: internals.options.getNow
501
+ });
502
+ return;
503
+ }
504
+ if (internals.inFlightOwnershipBootstrap !== null) {
505
+ await internals.inFlightOwnershipBootstrap;
506
+ return;
507
+ }
508
+ if (internals.workflowClaimRegistry !== null)
509
+ return;
510
+ const bootstrap = (async () => {
511
+ const result = await bootstrapWorkflowLeaseOwnership({
512
+ storage: internals.storage,
513
+ getNow: internals.options.getNow,
514
+ claimTtlMs: internals.options.workflowClaimTtlMs,
515
+ claimRenewIntervalMs: internals.options.workflowClaimRenewIntervalMs,
516
+ isWorkflowTypeRegistered: (workflowType) => internals.workflowDefinitionsByName.has(workflowType),
517
+ onWorkflowClaimReclaimed: async (workflowId) => {
518
+ internals.inlineStrategy?.evictContextForReclaim(workflowId);
519
+ internals.deliveredPendingUpdateIds.delete(workflowId);
520
+ await recoverPendingAsyncActivities(internals, workflowId);
521
+ if (internals.disposed)
522
+ return;
523
+ await resumeFromLifecycle(internals, workflowId, this.#createLifecycleCallbacks(), internals.onRecoveredWorkflowHook, { forceReplayFromStorage: !0 });
524
+ },
525
+ signalPollTarget: this.#buildOwnerSideSignalPollTarget(),
526
+ updatePollTarget: this.#buildOwnerSideUpdatePollTarget()
527
+ });
528
+ if (internals.disposed)
529
+ throw new EngineDisposedError;
530
+ internals.workflowClaimRegistry = result.registry;
531
+ internals.workflowClaimRenewalTask = result.renewalTask;
532
+ internals.workflowClaimMetrics = result.metrics;
533
+ if (internals.options.backgroundTaskMode === "automatic")
534
+ result.renewalTask.start();
535
+ })();
536
+ internals.inFlightOwnershipBootstrap = bootstrap;
537
+ try {
538
+ await bootstrap;
539
+ } finally {
540
+ if (internals.inFlightOwnershipBootstrap === bootstrap)
541
+ internals.inFlightOwnershipBootstrap = null;
542
+ }
543
+ }
544
+ #detachWorkflowClaimOwnership() {
545
+ const internals = getInternals(this), registry = internals.workflowClaimRegistry;
546
+ internals.workflowClaimRenewalTask?.stop();
547
+ internals.workflowClaimRenewalTask = null;
548
+ internals.workflowClaimRegistry = null;
549
+ internals.workflowClaimMetrics = null;
550
+ return { registry };
551
+ }
552
+ async#releaseWorkflowClaimsBestEffort(registry) {
553
+ await registry?.releaseAll().catch(() => {});
554
+ }
461
555
  #disposeAfterDeposition() {
462
556
  const internals = getInternals(this);
463
557
  if (internals.disposed)
464
558
  return;
465
- const leaseManager = internals.leaseManager;
559
+ const leaseManager = internals.leaseManager, { registry: workflowClaimRegistry } = this.#detachWorkflowClaimOwnership();
466
560
  disposeEngine(internals);
467
561
  if (this.#synchronousDisposeResult === null)
468
562
  this.#synchronousDisposeResult = (leaseManager?.release() ?? Promise.resolve(!0)).catch(() => !1);
469
563
  this.#synchronousDisposeResult;
564
+ this.#releaseWorkflowClaimsBestEffort(workflowClaimRegistry);
470
565
  }
471
566
  #startSecondInstanceDetection() {
472
567
  const internals = getInternals(this);
@@ -517,9 +612,55 @@ export class Engine extends EventTarget {
517
612
  #createInlineParkingCallbacks() {
518
613
  return createInlineParkingCallbacksForEngine(this);
519
614
  }
615
+ #buildOwnerSideSignalPollTarget() {
616
+ const internals = getInternals(this);
617
+ return buildOwnerSideSignalPollTarget({
618
+ listParkedInlineWorkflowIds: () => internals.parkedInlineWorkflows,
619
+ isParkedInlineWorkflow: (workflowId) => internals.parkedInlineWorkflows.has(workflowId),
620
+ parkedSignalName: (workflowId) => {
621
+ const pending = internals.pendingTimelineEntries.get(workflowId);
622
+ return pending?.entry.operationType === "wait-signal" ? pending.entry.operationLabel : void 0;
623
+ },
624
+ listSignalWaiterEntries: function* signalWaiterEntries() {
625
+ for (const [workflowId, waiterKeys] of internals.signalWaitersByWorkflow) {
626
+ if (typeof waiterKeys === "string") {
627
+ yield [workflowId, waiterKeys];
628
+ continue;
629
+ }
630
+ for (const waiterKey of waiterKeys)
631
+ yield [workflowId, waiterKey];
632
+ }
633
+ },
634
+ hasBufferedSignal: (workflowId, signalName) => hasBufferedSignalFromInternals(internals, workflowId, signalName),
635
+ resumeParkedInlineWorkflow: (workflowId) => resumeParkedInlineWorkflowFromInternals(internals, workflowId, this.#createInlineParkingCallbacks()),
636
+ confirmSignalWakeOwnership: (workflowId) => confirmWakeOwnership(internals, workflowId, "signal"),
637
+ wakeSignalWaiter: (workflowId, waiterKey) => {
638
+ const waiter = internals.signalWaiters.get(waiterKey);
639
+ if (waiter === void 0)
640
+ return;
641
+ if (releaseSignalWaiter(internals, workflowId, waiterKey, waiter))
642
+ waiter();
643
+ }
644
+ });
645
+ }
520
646
  #createUpdateCallbacks() {
521
647
  return createUpdateCallbacksForEngine(this);
522
648
  }
649
+ #buildOwnerSideUpdatePollTarget() {
650
+ const internals = getInternals(this);
651
+ return {
652
+ listHeldWorkflowIds: () => internals.workflowClaimRegistry?.listHeldWorkflowIds() ?? [],
653
+ hasPendingUpdates: async (workflowId) => {
654
+ return (await internals.updateCoordinator.getPendingUpdates(workflowId)).length > 0;
655
+ },
656
+ drainPendingUpdates: async (workflowId) => {
657
+ await processPendingUpdatesForHandlersFromInternals(internals, workflowId, createPendingUpdateCallbacksForEngine(this));
658
+ const stillPending = await internals.updateCoordinator.getPendingUpdates(workflowId), updateCallbacks = this.#createUpdateCallbacks();
659
+ for (const updateRequest of stillPending)
660
+ await deliverCoordinatedUpdateToWaiterIfAvailableFromInternals(internals, workflowId, updateRequest, !1, updateCallbacks);
661
+ }
662
+ };
663
+ }
523
664
  #createTimeOperationCallbacks() {
524
665
  return createTimeOperationCallbacksForEngine(this);
525
666
  }
@@ -629,6 +770,9 @@ export class Engine extends EventTarget {
629
770
  }
630
771
  async runMaintenance(now = getInternals(this).options.getNow()) {
631
772
  const internals = getInternals(this);
773
+ await this.#bootstrapOwnershipIfConfigured();
774
+ await internals.workflowClaimRenewalTask?.runOnce();
775
+ await pollPendingCrossEngineResultWaiters(internals);
632
776
  await internals.scheduler.tick(now);
633
777
  try {
634
778
  await internals.updateCoordinator.cleanupExpiredResponses();
@@ -768,11 +912,13 @@ export class Engine extends EventTarget {
768
912
  return resumeFromLifecycle(getInternals(this), workflowId, this.#createLifecycleCallbacks());
769
913
  }
770
914
  async recoverAll(options) {
915
+ await this.#bootstrapOwnershipIfConfigured();
771
916
  await this.#acquireLeaseIfConfigured();
772
917
  if (getInternals(this).disposed)
773
918
  throw new EngineDisposedError;
774
919
  await recoverPendingAsyncActivities(getInternals(this));
775
920
  await recoverOrphanedScheduleTimers(getInternals(this));
921
+ getInternals(this).onRecoveredWorkflowHook = options?.onRecoveredWorkflow;
776
922
  return recoverAllFromLifecycle(getInternals(this), this.#createLifecycleCallbacks(), options);
777
923
  }
778
924
  async completeAsyncActivity(token, result) {
@@ -934,10 +1080,12 @@ export class Engine extends EventTarget {
934
1080
  const leaseManager = getInternals(this).leaseManager;
935
1081
  if (leaseManager !== null && leaseManager.currentEpochBytes() !== null)
936
1082
  process.emitWarning("engine ownership lease disposed synchronously; lease release is fire-and-forget, so exiting before the release completes can make the next instance wait for leaseTtl. Use await engine.shutdown(), await using, or await engine[Symbol.asyncDispose]() for prompt rolling-deploy handoff.", ENGINE_LEASE_SYNCHRONOUS_DISPOSE_WARNING_NAME);
1083
+ const { registry: workflowClaimRegistry } = this.#detachWorkflowClaimOwnership();
937
1084
  disposeEngine(getInternals(this));
938
1085
  if (this.#synchronousDisposeResult === null)
939
1086
  this.#synchronousDisposeResult = leaseManager?.release() ?? Promise.resolve(!0);
940
1087
  this.#synchronousDisposeResult;
1088
+ this.#releaseWorkflowClaimsBestEffort(workflowClaimRegistry);
941
1089
  }
942
1090
  async#disposeAsyncWithLeaseResult() {
943
1091
  if (!getInternals(this).disposed) {
@@ -950,9 +1098,11 @@ export class Engine extends EventTarget {
950
1098
  } catch (error) {
951
1099
  drainFailure = { error };
952
1100
  } finally {
1101
+ const { registry: workflowClaimRegistry } = this.#detachWorkflowClaimOwnership();
953
1102
  disposeEngine(getInternals(this));
954
1103
  await getInternals(this).inFlightLeaseAcquire?.catch(() => {});
955
1104
  leaseReleased = await (this.#synchronousDisposeResult ?? leaseManager?.release() ?? Promise.resolve(!0));
1105
+ await this.#releaseWorkflowClaimsBestEffort(workflowClaimRegistry);
956
1106
  }
957
1107
  if (drainFailure !== null)
958
1108
  throw new EngineDisposalError(drainFailure.error, leaseReleased);
@@ -1,6 +1,7 @@
1
1
  import { WorkflowStartedEvent } from "../events.js";
2
2
  import { startWorkflowExecution } from "./lifecycle.js";
3
3
  import { loadWorkflowState } from "./storage-io.js";
4
+ import { confirmWakeOwnership } from "./wake-ownership-guard.js";
4
5
  async function yieldQueuedShutdownAdvanceOpportunity() {
5
6
  await new Promise((resolve) => {
6
7
  setTimeout(resolve, 0);
@@ -83,11 +84,19 @@ export async function drainQueuedInlineWorkflowStarts(internals, callbacks, opti
83
84
  }
84
85
  }
85
86
  }
87
+ async function isQueuedStartStillTheLiveRun(internals, start) {
88
+ const currentState = await loadWorkflowState(internals, start.workflowId);
89
+ return currentState !== null && currentState.status === "running" && (start.workflowExecutionToken === void 0 || currentState.workflowExecutionToken === start.workflowExecutionToken);
90
+ }
86
91
  async function startQueuedInlineWorkflowExecution(internals, start, callbacks, options) {
87
92
  try {
88
93
  const state = await loadWorkflowState(internals, start.workflowId);
89
94
  if (!state || state.status !== "running")
90
95
  return;
96
+ if (await confirmWakeOwnership(internals, start.workflowId, "inline-macrotask-drive") === "discard")
97
+ return;
98
+ if (!await isQueuedStartStillTheLiveRun(internals, start))
99
+ return;
91
100
  internals.queuedInlineWorkflowStartIds.delete(start.workflowId);
92
101
  internals.engine.dispatchEvent(new WorkflowStartedEvent(start.workflowId, start.workflowType, start.input));
93
102
  startWorkflowExecution(internals, start.workflowId, state.workflowExecutionToken, start.workflowType, start.input, start.checkpoint, start.nestingDepth, start.executionDeadline, start.executionStateOwnerId);
@@ -32,6 +32,7 @@ import type { Scheduler } from '../scheduler.ts';
32
32
  import type { Checkpoint, StartWorkflowOptions } from '../types.ts';
33
33
  import type { UpdateCoordinator } from '../updates.ts';
34
34
  import type { WorkflowVersionTuple } from '../workflow-version-tuple.ts';
35
+ import type { RecoveredWorkflowInfo } from './lifecycle/shared.ts';
35
36
  import type { ActivityHeartbeatKey } from './activity-heartbeat-tracking.ts';
36
37
  import type { PendingTimelineEntry, QueuedInlineWorkflowExecutionStart, RegistrationEntry, ResolvedOptions, TrackedWaiterKeys, WorkflowResultWaiter } from './engine-internal-types.ts';
37
38
  import type { EngineCleanupIntervalDisposalTracker } from './engine-leak-warnings.ts';
@@ -40,6 +41,9 @@ import type { WorkflowFeedListener } from './index.ts';
40
41
  import type { LeaseManager } from './lease-manager.ts';
41
42
  import type { ScheduleHandleEngine } from './schedule-handle.ts';
42
43
  import type { SecondInstanceDetector } from './second-instance-detector.ts';
44
+ import type { WorkflowClaimMetricsRecorder } from './workflow-claim-metrics.ts';
45
+ import type { WorkflowClaimRegistry } from './workflow-claim-registry.ts';
46
+ import type { WorkflowClaimRenewalTask } from './workflow-claim-renewal-task.ts';
43
47
  export type SleepTimerAcknowledgementWaiter = {
44
48
  fireAt: number;
45
49
  operationId: string;
@@ -239,6 +243,43 @@ export interface EngineInternals {
239
243
  * Step 1; epoch fencing of durable writes (Step 2) is what makes it enforceable.
240
244
  */
241
245
  leaseManager: LeaseManager | null;
246
+ /**
247
+ * The active per-workflow claim registry for `ownership: 'workflow-lease'`;
248
+ * `null` under `'none'`/`'lease'` and, for now, ALSO under `'workflow-lease'`
249
+ * itself — wiring construction (Gate 1/Gate 2, actually instantiating the
250
+ * registry, and folding `acquire()` into start/resume/delayed-start-fire) is a
251
+ * later stage. Its presence here lets {@link commitFencedEngineWrite} read
252
+ * {@link WorkflowClaimRegistry.currentEpochBytes} for a workflow-scoped write
253
+ * without that later stage having to touch the fencing path again. Until it is
254
+ * populated, every workflow-scoped write under `'workflow-lease'` fails closed
255
+ * with {@link EngineDeposedError} — correct, not a bug: this engine holds no
256
+ * claim for any workflow yet.
257
+ */
258
+ workflowClaimRegistry: WorkflowClaimRegistry | null;
259
+ /**
260
+ * The active per-workflow claim-renewal task for `ownership: 'workflow-lease'`;
261
+ * `null` under `'none'`/`'lease'` and until `#bootstrapOwnershipIfConfigured`
262
+ * completes. Renews every claim this engine holds (active or parked) on its
263
+ * own cadence — independent of the durable-timer scheduler, so `startScheduler:
264
+ * false` still renews claims — or via `runMaintenance()` under
265
+ * `backgroundTasks: 'manual'`, which never starts its interval. Stopped
266
+ * (interval cleared, never awaited — stopping is synchronous) and detached at
267
+ * every `disposeEngine` call site; releasing the claims it was renewing is the
268
+ * separate, best-effort `workflowClaimRegistry.releaseAll()` step each
269
+ * disposal path drives on its own schedule.
270
+ */
271
+ workflowClaimRenewalTask: WorkflowClaimRenewalTask | null;
272
+ /**
273
+ * Bounded-cardinality observability recorder for the `ownership:
274
+ * 'workflow-lease'` claim protocol (ADR 0002 § Observability); `null` under
275
+ * `'none'`/`'lease'` and until `#bootstrapOwnershipIfConfigured` completes.
276
+ * One instance per engine process, shared by this engine's claim-renewal task
277
+ * (which feeds it renewal-failure counts and the active-claim gauge after
278
+ * every pass) and, in a later stage, its claim-acquiring entry points (start,
279
+ * resume, delayed-start fire), which would record `acquired`/`takeover`/
280
+ * `lost_race`/`deposed`/`backoff_skipped` attempts into the same instance.
281
+ */
282
+ workflowClaimMetrics: WorkflowClaimMetricsRecorder | null;
242
283
  /**
243
284
  * The in-flight lease acquisition, or `null` when none is running. Set while
244
285
  * `#acquireLeaseIfConfigured` awaits `acquire()` (which can park for the whole
@@ -248,6 +289,15 @@ export interface EngineInternals {
248
289
  * than leaking until TTL on an already-disposed engine.
249
290
  */
250
291
  inFlightLeaseAcquire: Promise<void> | null;
292
+ /**
293
+ * The in-flight `ownership: 'workflow-lease'` bootstrap (Gate 1, Gate 2, and
294
+ * claim-registry/renewal-task construction), or `null` when none is running.
295
+ * Mirrors {@link inFlightLeaseAcquire}'s idempotency shape: concurrent
296
+ * `Engine.create` + `recoverAll` (and repeated `recoverAll`/`runMaintenance`)
297
+ * callers await this same promise rather than racing the gates twice. Always
298
+ * `null` under `ownership: 'none'`/`'lease'`, which never run this bootstrap.
299
+ */
300
+ inFlightOwnershipBootstrap: Promise<void> | null;
251
301
  /**
252
302
  * Set to `true` the instant this engine is detected as deposed under
253
303
  * `ownership: 'lease'` — either a fenced durable write's CAS failed against a
@@ -296,9 +346,32 @@ export interface EngineInternals {
296
346
  * updates whose delete never committed).
297
347
  */
298
348
  deliveredPendingUpdateIds: Map<string, Set<string>>;
349
+ /**
350
+ * The most recent `recoverAll({ onRecoveredWorkflow })` hook installed by
351
+ * this host, retained so a LATER same-engine reclaim (ADR 0002's recurring
352
+ * scan, after `recover: false` skipped this workflow entirely at startup
353
+ * because another engine held it then) still runs it before resuming.
354
+ * Without this, `index.ts`'s `onWorkflowClaimReclaimed` callback would pass
355
+ * `undefined` for every reclaim-driven resume, silently skipping whatever
356
+ * host initialization/validation the hook contract guarantees runs before
357
+ * recovered execution. `undefined` until `recoverAll()` is ever called with
358
+ * this option.
359
+ */
360
+ onRecoveredWorkflowHook: ((info: RecoveredWorkflowInfo) => void | Promise<void>) | undefined;
299
361
  cancelHandlersByWorkflow: Map<string, Array<() => Promise<void> | void>>;
300
362
  reviewTimerIds: Map<string, string[]>;
301
363
  pendingWebhooks: Set<AbortController>;
364
+ /**
365
+ * Every `setTimeout` handle `scheduleCrossEngineResultPollIfPending`
366
+ * (`handle-result.ts`) currently has pending, so disposal can cancel them.
367
+ * Without this, disposing an automatic-mode engine while a cross-engine
368
+ * result waiter is parked rejects and clears the waiter but leaves this
369
+ * untracked timer alive — the timer queue retains the waiter, engine
370
+ * internals, and storage reference until `workflowClaimRenewIntervalMs`
371
+ * elapses, and its callback then calls `bootstrapWorkflowResultResolver`
372
+ * against already-disposed storage.
373
+ */
374
+ pendingResultPollTimers: Set<ReturnType<typeof setTimeout>>;
302
375
  alertManager: AlertManager | null;
303
376
  eventLogHeads: Map<string, Readonly<EventHeadRecord>>;
304
377
  workflowFeedListeners: Map<string, Set<WorkflowFeedListener>>;
@@ -0,0 +1,19 @@
1
+ /**
2
+ * Invoking an `ctx.onUpdate()` handler, and the type of the handler itself.
3
+ *
4
+ * Separate from `updates.ts` because both the inline dispatch path and
5
+ * `pending-updates.ts`'s replay path call it, and because it is pure handler
6
+ * invocation with no coupling to the in-memory waiter maps or the durable
7
+ * coordinated-update protocol that make up the rest of update delivery.
8
+ *
9
+ * @module core/engine/invoke-update-handler
10
+ */
11
+ import type { EngineInternals } from './internals.ts';
12
+ /** An `ctx.onUpdate()` handler as stored on a live workflow context. */
13
+ export type InlineUpdateHandler = (payload: unknown) => unknown;
14
+ /**
15
+ * Invoke an update handler, checking that it does not return a generator.
16
+ * Centralises the runtime generator guard for both the inline-handler path
17
+ * in `update()` and the pending-drain path on resume.
18
+ */
19
+ export declare function invokeUpdateHandler(_internals: EngineInternals, name: string, handler: (payload: unknown) => unknown, payload: unknown): Promise<unknown>;
@@ -0,0 +1,7 @@
1
+ import { isGeneratorResult } from "../step-context.js";
2
+ export async function invokeUpdateHandler(_internals, name, handler, payload) {
3
+ const result = handler(payload);
4
+ if (isGeneratorResult(result))
5
+ throw TypeError(`Update handler "${name}" returned a generator. Update handlers must return a plain value or a Promise, not a generator.`);
6
+ return await result;
7
+ }
@@ -64,3 +64,135 @@ export declare const ENGINE_LEASE_LOST_WARNING_NAME = "WeftEngineLeaseLostWarnin
64
64
  * not advance in-memory state past a durable write that did not land.
65
65
  */
66
66
  export declare function handleDeposition(internals: EngineInternals): void;
67
+ /**
68
+ * `name` of the operator warning emitted when `ownership: 'workflow-lease'`
69
+ * loses one workflow's claim — a renewal CAS failure (a successor stole the
70
+ * claim) or a fenced write rejected because a newer epoch now holds it. Unlike
71
+ * {@link ENGINE_LEASE_LOST_WARNING_NAME}, this is per-workflow, not per-engine:
72
+ * the rest of this engine's claimed workflows are unaffected and keep running.
73
+ *
74
+ * @example
75
+ * ```ts
76
+ * import { WORKFLOW_CLAIM_LOST_WARNING_NAME } from '@lostgradient/weft';
77
+ *
78
+ * process.on('warning', (warning) => {
79
+ * if (warning.name === WORKFLOW_CLAIM_LOST_WARNING_NAME) {
80
+ * console.error('lost a workflow claim', warning.message);
81
+ * }
82
+ * });
83
+ * ```
84
+ */
85
+ export declare const WORKFLOW_CLAIM_LOST_WARNING_NAME = "WeftWorkflowClaimLostWarning";
86
+ /**
87
+ * Operator diagnostic for a lost per-workflow ownership claim under
88
+ * `ownership: 'workflow-lease'` (see
89
+ * [ADR 0002](../../../documentation/contributing/architecture-decisions/0002-multiengine-per-workflow-ownership.md)).
90
+ * Carries the affected `workflowId` as a real field — `process.emitWarning(message,
91
+ * name)` only accepts strings, so this is emitted as the warning object itself
92
+ * (`process.emitWarning(warning)`) rather than passed as a message, which lets
93
+ * consumers read `warning.workflowId` directly instead of parsing it back out
94
+ * of the message text.
95
+ *
96
+ * @example
97
+ * ```ts
98
+ * import { WeftWorkflowClaimLostWarning } from '@lostgradient/weft';
99
+ *
100
+ * process.on('warning', (warning) => {
101
+ * if (warning instanceof WeftWorkflowClaimLostWarning) {
102
+ * console.error('deposed workflow:', warning.workflowId);
103
+ * }
104
+ * });
105
+ * ```
106
+ */
107
+ export declare class WeftWorkflowClaimLostWarning extends Error {
108
+ readonly workflowId: string;
109
+ constructor(workflowId: string);
110
+ }
111
+ /**
112
+ * The wake path a stale in-memory resolver was discarded from, for
113
+ * {@link WeftWorkflowWakeDiscardedWarning}. Mirrors the `wakeOwnershipCheck`
114
+ * call sites named in ADR 0002: a durable timer firing, a re-evaluated
115
+ * `ctx.waitUntil()` condition, a delivered signal, an async-activity
116
+ * completion/failure, a child workflow's termination, or a deferred inline
117
+ * macrotask drive.
118
+ *
119
+ * @example
120
+ * ```ts
121
+ * import type { WorkflowWakeKind } from '@lostgradient/weft';
122
+ *
123
+ * const kind: WorkflowWakeKind = 'signal';
124
+ * void kind;
125
+ * ```
126
+ */
127
+ export type WorkflowWakeKind = 'sleep' | 'wait-condition' | 'signal' | 'async-activity' | 'child-completion' | 'update' | 'inline-macrotask-drive';
128
+ /**
129
+ * `name` of the operator warning emitted when `wakeOwnershipCheck` discards a
130
+ * stale in-memory wake resolver — it re-read the durable holder record and
131
+ * found this engine no longer holds the generation (engine id AND epoch) it
132
+ * parked the workflow under, so the resolver is dropped without driving the
133
+ * generator.
134
+ *
135
+ * @example
136
+ * ```ts
137
+ * import { WORKFLOW_WAKE_DISCARDED_WARNING_NAME } from '@lostgradient/weft';
138
+ *
139
+ * process.on('warning', (warning) => {
140
+ * if (warning.name === WORKFLOW_WAKE_DISCARDED_WARNING_NAME) {
141
+ * console.error('discarded a stale wake', warning.message);
142
+ * }
143
+ * });
144
+ * ```
145
+ */
146
+ export declare const WORKFLOW_WAKE_DISCARDED_WARNING_NAME = "WeftWorkflowWakeDiscardedWarning";
147
+ /**
148
+ * Operator diagnostic emitted whenever `wakeOwnershipCheck` discards a stale
149
+ * in-memory wake resolver under `ownership: 'workflow-lease'` (see
150
+ * [ADR 0002](../../../documentation/contributing/architecture-decisions/0002-multiengine-per-workflow-ownership.md)).
151
+ * Carries the affected `workflowId` and the `wakeKind` that was discarded as
152
+ * real fields, for the same reason {@link WeftWorkflowClaimLostWarning} does —
153
+ * `process.emitWarning(message, name)` cannot carry structured data, so this is
154
+ * emitted as the warning object itself.
155
+ *
156
+ * @example
157
+ * ```ts
158
+ * import { WeftWorkflowWakeDiscardedWarning } from '@lostgradient/weft';
159
+ *
160
+ * process.on('warning', (warning) => {
161
+ * if (warning instanceof WeftWorkflowWakeDiscardedWarning) {
162
+ * console.error('discarded wake', warning.workflowId, warning.wakeKind);
163
+ * }
164
+ * });
165
+ * ```
166
+ */
167
+ export declare class WeftWorkflowWakeDiscardedWarning extends Error {
168
+ readonly workflowId: string;
169
+ readonly wakeKind: WorkflowWakeKind;
170
+ constructor(workflowId: string, wakeKind: WorkflowWakeKind);
171
+ }
172
+ /**
173
+ * Emit one of the two per-workflow warnings above. Injected as a seam
174
+ * (defaulting to `process.emitWarning(warning)`) so later stages — the claim
175
+ * renewal loop and `wakeOwnershipCheck` — get deterministic test coverage of
176
+ * their emission sites, mirroring the `warn` option
177
+ * {@link createSecondInstanceDetector} already takes. Unlike that seam, which
178
+ * is message-only because its warning name is a single fixed constant, this
179
+ * one takes the warning *instance* — `WeftWorkflowClaimLostWarning` and
180
+ * `WeftWorkflowWakeDiscardedWarning` carry different fields, so a caller
181
+ * filtering or asserting on a specific warning needs the real object, not a
182
+ * pre-flattened string.
183
+ */
184
+ export type EmitWorkflowLeaseWarning = (warning: WeftWorkflowClaimLostWarning | WeftWorkflowWakeDiscardedWarning) => void;
185
+ /**
186
+ * Emit {@link WeftWorkflowClaimLostWarning} for `workflowId`. `emit` defaults
187
+ * to `process.emitWarning(warning)`; pass a test double to assert on the
188
+ * emitted instance deterministically instead of scraping `process`'s global
189
+ * `warning` event.
190
+ */
191
+ export declare function emitWorkflowClaimLostWarning(workflowId: string, emit?: EmitWorkflowLeaseWarning): void;
192
+ /**
193
+ * Emit {@link WeftWorkflowWakeDiscardedWarning} for `workflowId`/`wakeKind`.
194
+ * `emit` defaults to `process.emitWarning(warning)`; pass a test double to
195
+ * assert on the emitted instance deterministically instead of scraping
196
+ * `process`'s global `warning` event.
197
+ */
198
+ export declare function emitWorkflowWakeDiscardedWarning(workflowId: string, wakeKind: WorkflowWakeKind, emit?: EmitWorkflowLeaseWarning): void;
@@ -8,3 +8,34 @@ export function handleDeposition(internals) {
8
8
  if (tearDown !== null)
9
9
  Promise.resolve().then(tearDown).catch(() => {});
10
10
  }
11
+ export const WORKFLOW_CLAIM_LOST_WARNING_NAME = "WeftWorkflowClaimLostWarning";
12
+
13
+ export class WeftWorkflowClaimLostWarning extends Error {
14
+ workflowId;
15
+ constructor(workflowId) {
16
+ super(`workflow "${workflowId}" lost its ownership claim: another engine now holds it at a newer epoch. This engine stops driving that workflow; the rest of its claimed workflows are unaffected.`);
17
+ this.name = WORKFLOW_CLAIM_LOST_WARNING_NAME;
18
+ this.workflowId = workflowId;
19
+ }
20
+ }
21
+ export const WORKFLOW_WAKE_DISCARDED_WARNING_NAME = "WeftWorkflowWakeDiscardedWarning";
22
+
23
+ export class WeftWorkflowWakeDiscardedWarning extends Error {
24
+ workflowId;
25
+ wakeKind;
26
+ constructor(workflowId, wakeKind) {
27
+ super(`discarded a stale "${wakeKind}" wake for workflow "${workflowId}": this engine no longer holds the ownership-claim generation it parked under. The in-memory resolver is dropped without driving the generator.`);
28
+ this.name = WORKFLOW_WAKE_DISCARDED_WARNING_NAME;
29
+ this.workflowId = workflowId;
30
+ this.wakeKind = wakeKind;
31
+ }
32
+ }
33
+ const defaultEmitWorkflowLeaseWarning = (warning) => {
34
+ process.emitWarning(warning);
35
+ };
36
+ export function emitWorkflowClaimLostWarning(workflowId, emit = defaultEmitWorkflowLeaseWarning) {
37
+ emit(new WeftWorkflowClaimLostWarning(workflowId));
38
+ }
39
+ export function emitWorkflowWakeDiscardedWarning(workflowId, wakeKind, emit = defaultEmitWorkflowLeaseWarning) {
40
+ emit(new WeftWorkflowWakeDiscardedWarning(workflowId, wakeKind));
41
+ }