@lostgradient/weft 0.20.0 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. package/README.md +2 -2
  2. package/dist/core/engine/activity-reconciliation.d.ts +2 -2
  3. package/dist/core/engine/activity-reconciliation.js +12 -8
  4. package/dist/core/engine/async-activity-completion.js +12 -4
  5. package/dist/core/engine/async-activity-records.d.ts +11 -1
  6. package/dist/core/engine/async-activity-records.js +4 -3
  7. package/dist/core/engine/attributes-tags.d.ts +11 -1
  8. package/dist/core/engine/attributes-tags.js +4 -3
  9. package/dist/core/engine/bulk-operations-purge.js +5 -2
  10. package/dist/core/engine/bulk-operations.js +17 -5
  11. package/dist/core/engine/callback-creators-bundles.js +1 -1
  12. package/dist/core/engine/callback-creators-core.js +4 -2
  13. package/dist/core/engine/checkpoint-io.js +1 -1
  14. package/dist/core/engine/child-workflow.js +2 -1
  15. package/dist/core/engine/completed-review-storage.js +1 -1
  16. package/dist/core/engine/condition-waiters.d.ts +45 -3
  17. package/dist/core/engine/condition-waiters.js +12 -3
  18. package/dist/core/engine/disposal.js +6 -0
  19. package/dist/core/engine/engine-internal-types.d.ts +12 -4
  20. package/dist/core/engine/fenced-write.d.ts +46 -15
  21. package/dist/core/engine/fenced-write.js +39 -6
  22. package/dist/core/engine/finalizer-state.d.ts +3 -1
  23. package/dist/core/engine/handle-result.d.ts +49 -1
  24. package/dist/core/engine/handle-result.js +95 -20
  25. package/dist/core/engine/index.d.ts +13 -5
  26. package/dist/core/engine/index.js +155 -5
  27. package/dist/core/engine/inline-launch-queue.js +9 -0
  28. package/dist/core/engine/internals.d.ts +73 -0
  29. package/dist/core/engine/invoke-update-handler.d.ts +19 -0
  30. package/dist/core/engine/invoke-update-handler.js +7 -0
  31. package/dist/core/engine/lease-deposition.d.ts +132 -0
  32. package/dist/core/engine/lease-deposition.js +31 -0
  33. package/dist/core/engine/lease-errors.d.ts +80 -9
  34. package/dist/core/engine/lease-errors.js +28 -2
  35. package/dist/core/engine/lifecycle/resume.js +3 -1
  36. package/dist/core/engine/lifecycle/standalone-claim-acquire.d.ts +33 -0
  37. package/dist/core/engine/lifecycle/standalone-claim-acquire.js +25 -0
  38. package/dist/core/engine/lifecycle/start-commit.js +20 -6
  39. package/dist/core/engine/lifecycle/transition.d.ts +15 -1
  40. package/dist/core/engine/lifecycle/transition.js +6 -3
  41. package/dist/core/engine/operations-activity.js +1 -1
  42. package/dist/core/engine/operations-time.js +21 -7
  43. package/dist/core/engine/owner-side-signal-poll.d.ts +132 -0
  44. package/dist/core/engine/owner-side-signal-poll.js +22 -0
  45. package/dist/core/engine/owner-side-update-poll.d.ts +116 -0
  46. package/dist/core/engine/owner-side-update-poll.js +22 -0
  47. package/dist/core/engine/ownership-bootstrap.d.ts +280 -0
  48. package/dist/core/engine/ownership-bootstrap.js +89 -0
  49. package/dist/core/engine/ownership-mode-marker.d.ts +82 -0
  50. package/dist/core/engine/ownership-mode-marker.js +52 -0
  51. package/dist/core/engine/ownership-options.d.ts +22 -6
  52. package/dist/core/engine/ownership-options.js +42 -12
  53. package/dist/core/engine/pending-updates.js +35 -24
  54. package/dist/core/engine/queries.d.ts +70 -0
  55. package/dist/core/engine/queries.js +58 -4
  56. package/dist/core/engine/schedules.js +1 -1
  57. package/dist/core/engine/signals.d.ts +11 -1
  58. package/dist/core/engine/signals.js +11 -7
  59. package/dist/core/engine/sleep-timer-acknowledgements.d.ts +68 -0
  60. package/dist/core/engine/sleep-timer-acknowledgements.js +19 -0
  61. package/dist/core/engine/storage-io.d.ts +58 -10
  62. package/dist/core/engine/storage-io.js +27 -9
  63. package/dist/core/engine/termination/cleanup.d.ts +5 -4
  64. package/dist/core/engine/termination/complete.js +23 -4
  65. package/dist/core/engine/termination/finalizer-claim.js +4 -4
  66. package/dist/core/engine/termination/state-commit-callbacks.d.ts +15 -0
  67. package/dist/core/engine/termination/state-commit-callbacks.js +0 -0
  68. package/dist/core/engine/termination/suspend.js +1 -1
  69. package/dist/core/engine/update-validation.d.ts +28 -0
  70. package/dist/core/engine/update-validation.js +37 -0
  71. package/dist/core/engine/updates.d.ts +17 -16
  72. package/dist/core/engine/updates.js +24 -47
  73. package/dist/core/engine/wake-ownership-check.d.ts +86 -0
  74. package/dist/core/engine/wake-ownership-check.js +19 -0
  75. package/dist/core/engine/wake-ownership-guard.d.ts +46 -0
  76. package/dist/core/engine/wake-ownership-guard.js +23 -0
  77. package/dist/core/engine/workflow-claim-codec.d.ts +64 -0
  78. package/dist/core/engine/workflow-claim-codec.js +44 -0
  79. package/dist/core/engine/workflow-claim-cooldown.d.ts +60 -0
  80. package/dist/core/engine/workflow-claim-cooldown.js +17 -0
  81. package/dist/core/engine/workflow-claim-fold.d.ts +82 -0
  82. package/dist/core/engine/workflow-claim-fold.js +53 -0
  83. package/dist/core/engine/workflow-claim-metrics.d.ts +131 -0
  84. package/dist/core/engine/workflow-claim-metrics.js +62 -0
  85. package/dist/core/engine/workflow-claim-reclaim-scan.d.ts +96 -0
  86. package/dist/core/engine/workflow-claim-reclaim-scan.js +66 -0
  87. package/dist/core/engine/workflow-claim-reclaim-target.d.ts +107 -0
  88. package/dist/core/engine/workflow-claim-reclaim-target.js +155 -0
  89. package/dist/core/engine/workflow-claim-registry.d.ts +227 -0
  90. package/dist/core/engine/workflow-claim-registry.js +198 -0
  91. package/dist/core/engine/workflow-claim-renewal-interval.d.ts +50 -0
  92. package/dist/core/engine/workflow-claim-renewal-interval.js +132 -0
  93. package/dist/core/engine/workflow-claim-renewal-subpasses.d.ts +265 -0
  94. package/dist/core/engine/workflow-claim-renewal-subpasses.js +104 -0
  95. package/dist/core/engine/workflow-claim-renewal-task.d.ts +159 -0
  96. package/dist/core/engine/workflow-claim-renewal-task.js +65 -0
  97. package/dist/core/engine/workflow-claim-transitions.d.ts +186 -0
  98. package/dist/core/engine/workflow-claim-transitions.js +120 -0
  99. package/dist/core/inline-execution-strategy.d.ts +20 -0
  100. package/dist/core/inline-execution-strategy.js +15 -4
  101. package/dist/core/scheduler/scheduler-class.js +3 -3
  102. package/dist/core/scheduler/timer-sources.d.ts +13 -0
  103. package/dist/core/scheduler/timer-sources.js +1 -1
  104. package/dist/core/types/options.d.ts +17 -1
  105. package/dist/core/weft-error.d.ts +1 -1
  106. package/dist/core/weft-error.js +2 -1
  107. package/dist/http.js +2 -2
  108. package/dist/index.d.ts +2 -1
  109. package/dist/index.js +8 -0
  110. package/dist/indexeddb.js +1 -1
  111. package/dist/json-schema.js +3 -3
  112. package/dist/observability/index.js +2 -2
  113. package/dist/storage/auto.js +1 -1
  114. package/dist/storage/bun-sql.js +113 -0
  115. package/dist/storage/compressed-storage.js +1 -1
  116. package/dist/storage/index.d.ts +38 -35
  117. package/dist/storage/interface.d.ts +87 -125
  118. package/dist/storage/interface.js +1 -1
  119. package/dist/storage/key-encoding.d.ts +51 -0
  120. package/dist/storage/key-encoding.js +13 -0
  121. package/dist/storage/key-prefixes.d.ts +1 -1
  122. package/dist/storage/key-prefixes.js +3 -0
  123. package/dist/storage/lmdb.js +1 -1
  124. package/dist/storage/memory.js +1 -1
  125. package/dist/storage/neon.js +2 -2
  126. package/dist/storage/node-sqlite.js +113 -0
  127. package/dist/storage/ownership-keys.d.ts +55 -0
  128. package/dist/storage/ownership-keys.js +6 -0
  129. package/dist/storage/postgres.js +2 -2
  130. package/dist/storage/resolve.js +1 -1
  131. package/dist/storage/scoped-storage.js +1 -1
  132. package/dist/storage/testing.js +1 -1
  133. package/dist/storage/turso.js +2 -2
  134. package/dist/version.d.ts +1 -1
  135. package/dist/version.js +1 -1
  136. package/dist/web-extension.js +1 -1
  137. package/dist/worker/protocol.js +1 -1
  138. package/package.json +1 -1
@@ -128,7 +128,7 @@ export async function executeActivityOperationResult(internals, workflowId, oper
128
128
  if (executionOptions.reconciliationCompletion === "immediate-fenced") {
129
129
  const finishImmediateReconciliationCommit = executionOptions.beforeImmediateReconciliationCommit?.();
130
130
  try {
131
- await commitActivityReconciliationTransitionWithFencedWrite(internals, reference, started, completedRecord);
131
+ await commitActivityReconciliationTransitionWithFencedWrite(internals, workflowId, reference, started, completedRecord);
132
132
  } finally {
133
133
  finishImmediateReconciliationCommit?.();
134
134
  }
@@ -2,13 +2,16 @@ import { KEYS, storageHas } from "../../storage/interface.js";
2
2
  import { deserializeCheckpoint } from "../checkpoint.js";
3
3
  import { encode } from "../codec.js";
4
4
  import { buildTimerBatchOperations, normalizeStorageTimestamp } from "../scheduler.js";
5
+ import { notifyConditionWaiters, notifyConditionWaitersForTimerFire } from "./condition-waiters.js";
5
6
  import { commitFencedEngineWrite } from "./fenced-write.js";
6
7
  import { reprovideRecoveredServices } from "./lifecycle/recovered-services.js";
7
8
  import {
8
9
  acknowledgeSupersededSleepTimers,
9
10
  handleSleepTimerWithAcknowledgement,
10
- resolveSleepTimer
11
+ resolveSleepTimer,
12
+ retainDiscardedDurableTimer
11
13
  } from "./sleep-timer-acknowledgements.js";
14
+ import { commitWithWorkflowClaimFold, prepareWorkflowClaimFold } from "./workflow-claim-fold.js";
12
15
  import { buildWorkflowVisibilityIndexTransition } from "./workflow-indexes.js";
13
16
  export function createDelayedStartTimerEntry(_internals, workflowId, scheduledStartAt, options, callbacks) {
14
17
  return {
@@ -118,7 +121,10 @@ export async function startDelayedWorkflow(internals, entry, callbacks) {
118
121
  fireAt: executionDeadline,
119
122
  kind: "execution-deadline"
120
123
  }));
121
- await commitFencedEngineWrite(internals, operations, [], () => Error(`Delayed-start transition for workflow "${entry.workflowId}" lost its CAS race.`));
124
+ const claimFold = await prepareWorkflowClaimFold(internals, entry.workflowId);
125
+ if (claimFold)
126
+ return (await commitWithWorkflowClaimFold(internals, claimFold, operations, [], "delayed-start workflow claim acquisition")).status === "committed" ? nextRunningState : null;
127
+ await commitFencedEngineWrite(internals, entry.workflowId, operations, [], () => Error(`Delayed-start transition for workflow "${entry.workflowId}" lost its CAS race.`));
122
128
  return nextRunningState;
123
129
  });
124
130
  if (!runningState)
@@ -158,7 +164,7 @@ async function failInvalidDelayedExecutionTimeout(entry, callbacks) {
158
164
  await callbacks.failWorkflow(entry.workflowId, Error(`Invalid delayed execution timeout for workflow "${entry.workflowId}"`));
159
165
  }
160
166
  export async function handleTimerFired(internals, entry, callbacks) {
161
- if (entry.id.startsWith("review-escalation:") || entry.id.startsWith("review-timeout:")) {
167
+ if (isReviewTimerEntry(entry)) {
162
168
  await handleReviewTimer(internals, entry, callbacks);
163
169
  return;
164
170
  }
@@ -181,14 +187,22 @@ export async function handleTimerFired(internals, entry, callbacks) {
181
187
  if (entry.kind === "sleep")
182
188
  await handleSleepTimerWithAcknowledgement(internals, entry, callbacks.loadWorkflowState);
183
189
  else if (entry.kind === "wait-condition")
184
- resolveConditionTimer(internals, entry);
190
+ if (internals.workflowClaimRegistry === null)
191
+ resolveConditionTimer(internals, entry);
192
+ else
193
+ await resolveConditionTimerConfirmingOwnership(internals, entry, callbacks.loadWorkflowState);
185
194
  else if (entry.kind === "execution-deadline")
186
195
  await callbacks.timeout(entry.workflowId);
187
196
  }
197
+ function isReviewTimerEntry(entry) {
198
+ return entry.id.startsWith("review-escalation:") || entry.id.startsWith("review-timeout:");
199
+ }
188
200
  function resolveConditionTimer(internals, entry) {
189
- const resolver = internals.conditionWaiters.get(entry.workflowId);
190
- if (resolver)
191
- resolver();
201
+ notifyConditionWaiters(internals, entry.workflowId);
202
+ }
203
+ async function resolveConditionTimerConfirmingOwnership(internals, entry, loadWorkflowState) {
204
+ if (await notifyConditionWaitersForTimerFire(internals, entry.workflowId) === "discard")
205
+ await retainDiscardedDurableTimer(entry.id, entry.workflowId, loadWorkflowState);
192
206
  }
193
207
  async function handleReviewTimer(internals, entry, callbacks) {
194
208
  const reviewId = entry.id.split(":")[1];
@@ -0,0 +1,132 @@
1
+ /**
2
+ * Owner-side signal polling, specified in
3
+ * [ADR 0002 § Entry point classification](../../../documentation/contributing/architecture-decisions/0002-multiengine-per-workflow-ownership.md)
4
+ * (the paragraph beginning "Signal delivery needs more than a
5
+ * classification").
6
+ *
7
+ * **The problem.** Weft has no cross-engine RPC. A signal sent to any engine
8
+ * is durably buffered — but under `ownership: 'workflow-lease'`, only the
9
+ * engine that owns a workflow inspects its in-memory waiters when a signal
10
+ * arrives. If the signal lands on a NON-owning engine, the durable buffer is
11
+ * written and nobody wakes the owner: a workflow parked on
12
+ * `waitForSignal()` (including a `waitForSignal` branch inside `ctx.race()`)
13
+ * hangs until something unrelated happens to drive it.
14
+ *
15
+ * **The fix.** The owning engine re-checks the durable signal buffer for its
16
+ * OWN parked workflows on the same cadence it renews their claims, and wakes
17
+ * any whose awaited signal has since been buffered — by any engine. This
18
+ * bounds cross-engine signal delivery latency at one renewal interval rather
19
+ * than leaving it unbounded. Owner-side polling is the accepted mechanism;
20
+ * real cross-engine notification would be a later latency optimization, not
21
+ * a correctness fix.
22
+ *
23
+ * **Scope and independence.** This module is the poll pass alone: given a
24
+ * list of this engine's parked signal waits, probe the durable buffer for
25
+ * each, and wake the ones that are ready. It does not read
26
+ * `src/core/engine/signals.ts` or `src/core/engine/inline-parking.ts`
27
+ * directly — the dependencies below are small structural types this module
28
+ * owns, so it stays independent of those modules' concrete shapes. Each
29
+ * type's doc comment names the existing function it satisfies. This module
30
+ * is also NOT a recurring task by itself (compare
31
+ * `workflow-claim-renewal-task.ts`) — it is production-wired by
32
+ * `ownership-bootstrap.ts`'s {@link buildOwnerSideSignalPollTarget}, which
33
+ * composes {@link OwnerSideSignalPollSources} (from `EngineInternals` and
34
+ * `inline-parking.ts`) into a real {@link OwnerSideSignalPollTarget}, and by
35
+ * `workflow-claim-renewal-task.ts`, which calls {@link runOwnerSideSignalPoll}
36
+ * against that target from the same lifecycle cadence that drives claim
37
+ * renewal.
38
+ *
39
+ * @module core/engine/owner-side-signal-poll
40
+ */
41
+ /** One workflow parked on `waitForSignal(signalName)` (or a `ctx.race()` branch awaiting it), that this engine owns. */
42
+ export type ParkedSignalWait = {
43
+ workflowId: string;
44
+ signalName: string;
45
+ };
46
+ /**
47
+ * The minimal structural shape this poll needs. Satisfied in production by
48
+ * `ownership-bootstrap.ts`'s `buildOwnerSideSignalPollTarget`, composed from
49
+ * `EngineInternals` and `signals.ts` / `inline-parking.ts` — without this
50
+ * module importing either.
51
+ */
52
+ export type OwnerSideSignalPollTarget = {
53
+ /**
54
+ * Every signal wait this engine's currently-parked workflows are awaiting.
55
+ * Read fresh at the start of every pass — implementations may return a
56
+ * live or a defensive-copy array; this poll never mutates it and takes its
57
+ * own snapshot before iterating.
58
+ *
59
+ * Expected to be satisfied by deriving from `EngineInternals.signalWaiters`
60
+ * (a `Map<string, () => void>` keyed by `` `${workflowId}:${signalName}` ``,
61
+ * the same format `src/core/engine/signals.ts`'s `deliverBufferedSignals`
62
+ * reads), splitting each key back into its `workflowId`/`signalName` pair.
63
+ */
64
+ listParkedSignalWaits(): readonly ParkedSignalWait[];
65
+ /**
66
+ * Probe whether `signalName` has already been durably buffered for
67
+ * `workflowId`, without consuming it — this poll only decides WHETHER to
68
+ * wake, never consumes the signal itself, so a losing `ctx.race()` branch
69
+ * is never silently dropped.
70
+ *
71
+ * Expected to be satisfied by `hasBufferedSignal` from
72
+ * `src/core/engine/signals.ts`.
73
+ */
74
+ hasBufferedSignal(workflowId: string, signalName: string): Promise<boolean>;
75
+ /**
76
+ * Wake `workflowId` so it re-evaluates its parked state and, ultimately,
77
+ * consumes the buffered signal and resumes its generator. Resolves once
78
+ * the wake attempt has been dispatched; rejects on failure, which this
79
+ * poll catches and records without stopping the remaining wakes in the
80
+ * pass.
81
+ *
82
+ * Expected to be satisfied by `resumeParkedInlineWorkflow` from
83
+ * `src/core/engine/inline-parking.ts` (already surfaced to callers today
84
+ * through `SignalCallbacks.resumeParkedInlineWorkflow` in `signals.ts`).
85
+ * The real implementation is expected to perform its own
86
+ * `wakeOwnershipCheck` immediately before driving the generator, per the
87
+ * ADR — that check is this poll's sibling module, not this poll's
88
+ * responsibility.
89
+ */
90
+ wakeWorkflow(workflowId: string): Promise<void>;
91
+ };
92
+ /** One parked signal wait's outcome within a single poll pass. */
93
+ export type OwnerSideSignalPollOutcome = {
94
+ workflowId: string;
95
+ signalName: string;
96
+ status: 'woken';
97
+ } | {
98
+ workflowId: string;
99
+ signalName: string;
100
+ status: 'not-buffered';
101
+ } | {
102
+ workflowId: string;
103
+ signalName: string;
104
+ status: 'wake-failed';
105
+ error: unknown;
106
+ };
107
+ /** The result of one full owner-side signal poll pass ({@link runOwnerSideSignalPoll}). */
108
+ export type OwnerSideSignalPollResult = {
109
+ /** `getNow()` read at the start of the pass, before any probe. */
110
+ startedAt: number;
111
+ /** `getNow()` read after every probe/wake has settled. */
112
+ finishedAt: number;
113
+ /** One entry per parked signal wait the pass attempted, in iteration order. */
114
+ outcomes: OwnerSideSignalPollOutcome[];
115
+ /** `outcomes.filter(o => o.status === 'woken').length`, precomputed for observability consumers. */
116
+ wokenCount: number;
117
+ };
118
+ /** Options for {@link runOwnerSideSignalPoll}. */
119
+ export type OwnerSideSignalPollOptions = {
120
+ target: OwnerSideSignalPollTarget;
121
+ /** Wall-clock source (ms), injected so tests never depend on real time. */
122
+ getNow: () => number;
123
+ };
124
+ /**
125
+ * Run exactly one owner-side signal poll pass: snapshot this engine's
126
+ * currently-parked signal waits, probe the durable buffer for each, and wake
127
+ * the ones that are ready. A wake failure for one parked workflow is
128
+ * captured as a `'wake-failed'` outcome and never stops the remaining
129
+ * parked workflows in the same pass from being probed and, where ready,
130
+ * woken.
131
+ */
132
+ export declare function runOwnerSideSignalPoll(options: OwnerSideSignalPollOptions): Promise<OwnerSideSignalPollResult>;
@@ -0,0 +1,22 @@
1
+ export async function runOwnerSideSignalPoll(options) {
2
+ const { target, getNow } = options, startedAt = getNow(), parkedWaits = target.listParkedSignalWaits(), outcomes = [];
3
+ for (const { workflowId, signalName } of parkedWaits) {
4
+ if (!await target.hasBufferedSignal(workflowId, signalName)) {
5
+ outcomes.push({ workflowId, signalName, status: "not-buffered" });
6
+ continue;
7
+ }
8
+ try {
9
+ await target.wakeWorkflow(workflowId);
10
+ outcomes.push({ workflowId, signalName, status: "woken" });
11
+ } catch (error) {
12
+ outcomes.push({ workflowId, signalName, status: "wake-failed", error });
13
+ }
14
+ }
15
+ const finishedAt = getNow();
16
+ return {
17
+ startedAt,
18
+ finishedAt,
19
+ outcomes,
20
+ wokenCount: outcomes.filter((outcome) => outcome.status === "woken").length
21
+ };
22
+ }
@@ -0,0 +1,116 @@
1
+ /**
2
+ * Owner-side coordinated-update polling (WFT-79), mirroring
3
+ * `owner-side-signal-poll.ts`'s mechanism and rationale.
4
+ *
5
+ * **The problem.** `updates.ts`'s `deliverCoordinatedUpdateToWaiterIfAvailable`
6
+ * discards a delivery attempt when this engine no longer holds the claim
7
+ * generation it parked the waiter under (`confirmWakeOwnership`), leaving the
8
+ * durable coordinated-update request in place for the true owner to deliver.
9
+ * But nothing DRIVES the true owner to look: `schedulePendingInlineUpdateDrain`
10
+ * only fires a `setTimeout(0)` drain on the engine that RECEIVED the
11
+ * `engine.update()`/`submitCoordinatedUpdate()` call, and the owning engine's
12
+ * maintenance task polls the durable signal buffer and child-result state,
13
+ * never pending coordinated updates. If the true owner is parked on
14
+ * `ctx.waitForUpdate()`, or simply has no reason to re-check because nothing
15
+ * else is advancing it, the caller's request sits durable and undelivered
16
+ * until something unrelated happens to drive that workflow.
17
+ *
18
+ * **The fix.** The owning engine re-checks its own held workflows' pending
19
+ * coordinated-update queues on the same cadence it renews their claims, and
20
+ * drains any workflow with at least one buffered request — by any engine.
21
+ * This bounds cross-engine update delivery latency at one renewal interval,
22
+ * the same bound ADR 0002 already gives cross-engine signal delivery.
23
+ *
24
+ * **Scope and independence.** This module is the poll pass alone: given this
25
+ * engine's held workflow ids, probe each for pending coordinated updates, and
26
+ * drain the ones that have any. It does not read `pending-updates.ts` or
27
+ * `updates.ts` directly — the dependencies below are small structural types
28
+ * this module owns, so it stays independent of those modules' concrete
29
+ * shapes. Draining is expected to be satisfied by composing
30
+ * `processPendingUpdatesForHandlers` (drains inline `ctx.onUpdate()`
31
+ * handlers) and `deliverCoordinatedUpdateToWaiterIfAvailable`-per-pending-
32
+ * update (drains `ctx.waitForUpdate()` waiters) — both already idempotent and
33
+ * fenced on this engine's own claim generation, so a poll racing a
34
+ * concurrent delivery (the normal `setTimeout(0)` drain, or another poll
35
+ * tick) is always safe to re-run. This module is also NOT a recurring task
36
+ * by itself (compare `workflow-claim-renewal-task.ts`) — it is production-
37
+ * wired by `ownership-bootstrap.ts`, which composes `EngineInternals` into a
38
+ * real {@link OwnerSideUpdatePollTarget}, and by
39
+ * `workflow-claim-renewal-task.ts`, which calls {@link runOwnerSideUpdatePoll}
40
+ * against that target from the same lifecycle cadence that drives claim
41
+ * renewal and signal polling.
42
+ *
43
+ * @module core/engine/owner-side-update-poll
44
+ */
45
+ /**
46
+ * The minimal structural shape this poll needs. Satisfied in production by
47
+ * `ownership-bootstrap.ts`'s adapter, composed from `EngineInternals` —
48
+ * without this module importing it.
49
+ */
50
+ export type OwnerSideUpdatePollTarget = {
51
+ /**
52
+ * Every workflow id this engine currently holds a live claim for, active or
53
+ * parked. Read fresh at the start of every pass — implementations may
54
+ * return a live or a defensive-copy array; this poll never mutates it and
55
+ * takes its own snapshot before iterating.
56
+ *
57
+ * Expected to be satisfied by `WorkflowClaimRegistry.listHeldWorkflowIds`,
58
+ * the same source `workflow-claim-renewal-task.ts`'s own renewal sub-pass
59
+ * uses.
60
+ */
61
+ listHeldWorkflowIds(): readonly string[];
62
+ /**
63
+ * Whether `workflowId` currently has at least one durably-buffered
64
+ * coordinated-update request, without consuming any of them.
65
+ *
66
+ * Expected to be satisfied by
67
+ * `internals.updateCoordinator.getPendingUpdates(workflowId).length > 0`.
68
+ */
69
+ hasPendingUpdates(workflowId: string): Promise<boolean>;
70
+ /**
71
+ * Drain every pending coordinated update currently buffered for
72
+ * `workflowId`: deliver to a registered `ctx.onUpdate()` handler, or to a
73
+ * parked `ctx.waitForUpdate()` waiter, whichever applies. Idempotent and
74
+ * self-fenced on this engine's own current claim generation — safe to call
75
+ * even when a concurrent drain (the normal post-request `setTimeout(0)`
76
+ * trigger, or another poll tick) is racing it.
77
+ */
78
+ drainPendingUpdates(workflowId: string): Promise<void>;
79
+ };
80
+ /** One held workflow's outcome within a single update-poll pass. */
81
+ export type OwnerSideUpdatePollOutcome = {
82
+ workflowId: string;
83
+ status: 'drained';
84
+ } | {
85
+ workflowId: string;
86
+ status: 'no-pending-updates';
87
+ } | {
88
+ workflowId: string;
89
+ status: 'drain-failed';
90
+ error: unknown;
91
+ };
92
+ /** The result of one full owner-side update poll pass ({@link runOwnerSideUpdatePoll}). */
93
+ export type OwnerSideUpdatePollResult = {
94
+ /** `getNow()` read at the start of the pass, before any probe. */
95
+ startedAt: number;
96
+ /** `getNow()` read after every probe/drain has settled. */
97
+ finishedAt: number;
98
+ /** One entry per held workflow id the pass attempted, in iteration order. */
99
+ outcomes: OwnerSideUpdatePollOutcome[];
100
+ /** `outcomes.filter(o => o.status === 'drained').length`, precomputed for observability consumers. */
101
+ drainedCount: number;
102
+ };
103
+ /** Options for {@link runOwnerSideUpdatePoll}. */
104
+ export type OwnerSideUpdatePollOptions = {
105
+ target: OwnerSideUpdatePollTarget;
106
+ /** Wall-clock source (ms), injected so tests never depend on real time. */
107
+ getNow: () => number;
108
+ };
109
+ /**
110
+ * Run exactly one owner-side update poll pass: snapshot this engine's
111
+ * currently-held workflow ids, probe each for pending coordinated updates,
112
+ * and drain the ones that have any. A drain failure for one workflow is
113
+ * captured as a `'drain-failed'` outcome and never stops the remaining held
114
+ * workflows in the same pass from being probed and, where pending, drained.
115
+ */
116
+ export declare function runOwnerSideUpdatePoll(options: OwnerSideUpdatePollOptions): Promise<OwnerSideUpdatePollResult>;
@@ -0,0 +1,22 @@
1
+ export async function runOwnerSideUpdatePoll(options) {
2
+ const { target, getNow } = options, startedAt = getNow(), heldWorkflowIds = target.listHeldWorkflowIds(), outcomes = [];
3
+ for (const workflowId of heldWorkflowIds) {
4
+ if (!await target.hasPendingUpdates(workflowId)) {
5
+ outcomes.push({ workflowId, status: "no-pending-updates" });
6
+ continue;
7
+ }
8
+ try {
9
+ await target.drainPendingUpdates(workflowId);
10
+ outcomes.push({ workflowId, status: "drained" });
11
+ } catch (error) {
12
+ outcomes.push({ workflowId, status: "drain-failed", error });
13
+ }
14
+ }
15
+ const finishedAt = getNow();
16
+ return {
17
+ startedAt,
18
+ finishedAt,
19
+ outcomes,
20
+ drainedCount: outcomes.filter((outcome) => outcome.status === "drained").length
21
+ };
22
+ }
@@ -0,0 +1,280 @@
1
+ /**
2
+ * Engine-level bootstrap for `ownership: 'workflow-lease'`
3
+ * ([ADR 0002](../../../documentation/contributing/architecture-decisions/0002-multiengine-per-workflow-ownership.md)):
4
+ * runs the two construction-time capability gates ({@link bootstrapOwnershipGates})
5
+ * and, once they pass, constructs this engine's {@link WorkflowClaimRegistry}
6
+ * and its claim-renewal task. Standalone from `Engine`/`EngineInternals` so the
7
+ * idempotency, disposal-race, and background-task-mode wiring in
8
+ * `src/core/engine/index.ts` stay thin call sites over one tested unit.
9
+ *
10
+ * **Both fencing modes run the gates.** This module builds the
11
+ * `workflow-lease` machinery, but `Engine`'s ownership bootstrap also runs
12
+ * Gate 1 and Gate 2 for `ownership: 'lease'` before it acquires the global
13
+ * lease, so the store-wide mode marker rejects a mismatched pairing in either
14
+ * direction. `'lease'` acquisition, fencing, health, and disposal are
15
+ * otherwise unchanged.
16
+ *
17
+ * **The renewal task's reclaim scan is fully wired here**, via
18
+ * {@link createWorkflowClaimReclaimTarget}: candidate discovery
19
+ * (`workflow-claim-reclaim-scan.ts`, including its ownerless-but-running
20
+ * scan — ADR 0002 § "Reclaiming stranded claims") plus a bounded-retry
21
+ * `registry.takeover`/`registry.acquire` loop, per the ADR's "bounded at 5
22
+ * attempts" row. A reclaimed or freshly-acquired claim's `onReclaimed` drive
23
+ * is retried on every later pass if it throws — see
24
+ * {@link createWorkflowClaimReclaimTarget}'s doc for why release-on-failure
25
+ * is unsafe here.
26
+ *
27
+ * **Owner-side signal polling is composed here from small structural
28
+ * sources** ({@link buildOwnerSideSignalPollTarget}) rather than built
29
+ * against `EngineInternals` directly — this module stays independent of
30
+ * that module's concrete shape, matching `owner-side-signal-poll.ts`'s own
31
+ * decoupling discipline. `src/core/engine/index.ts` is the only expected
32
+ * caller: it already has `EngineInternals` in scope and supplies
33
+ * {@link OwnerSideSignalPollSources} as one-line closures over
34
+ * `getInternals(this)`, then passes the composed target through
35
+ * {@link WorkflowLeaseOwnershipBootstrapOptions.signalPollTarget}.
36
+ *
37
+ * @module core/engine/ownership-bootstrap
38
+ */
39
+ import type { Storage } from '../../storage/interface.ts';
40
+ import type { OwnerSideSignalPollTarget } from './owner-side-signal-poll.ts';
41
+ import type { OwnerSideUpdatePollTarget } from './owner-side-update-poll.ts';
42
+ import { WorkflowClaimMetricsCollector } from './workflow-claim-metrics.ts';
43
+ import { WorkflowClaimRegistry } from './workflow-claim-registry.ts';
44
+ import type { WorkflowClaimRenewalTarget } from './workflow-claim-renewal-subpasses.ts';
45
+ import { type WorkflowClaimRenewalTask } from './workflow-claim-renewal-task.ts';
46
+ export { createWorkflowClaimReclaimTarget, WORKFLOW_CLAIM_TAKEOVER_MAX_ATTEMPTS, type WorkflowClaimReclaimTargetHandle, } from './workflow-claim-reclaim-target.ts';
47
+ /** Input to {@link bootstrapWorkflowLeaseOwnership}. */
48
+ export type WorkflowLeaseOwnershipBootstrapOptions = {
49
+ storage: Storage;
50
+ /** Engine-clock source (ms), injected so tests can drive it deterministically. */
51
+ getNow: () => number;
52
+ /** Resolved `workflowClaimTtl` (ms). */
53
+ claimTtlMs: number;
54
+ /** Resolved `workflowClaimRenewInterval` (ms). */
55
+ claimRenewIntervalMs: number;
56
+ /**
57
+ * Optional owner-side signal-poll target, built by a caller with access to
58
+ * `EngineInternals`/`inline-parking.ts` — typically
59
+ * {@link buildOwnerSideSignalPollTarget} composed from
60
+ * {@link OwnerSideSignalPollSources}. See the module doc's signal-poll
61
+ * paragraph. Omitted leaves `result.signalPoll` `undefined` on every pass.
62
+ */
63
+ signalPollTarget?: OwnerSideSignalPollTarget;
64
+ /**
65
+ * Optional owner-side update-poll target (WFT-79), built by a caller with
66
+ * access to `EngineInternals` — typically `index.ts`'s
67
+ * `#buildOwnerSideUpdatePollTarget`. Omitted leaves `result.updatePoll`
68
+ * `undefined` on every pass. See `owner-side-update-poll.ts`'s module doc
69
+ * for why the owning engine must re-check its own held workflows' pending
70
+ * coordinated-update queues rather than relying solely on the
71
+ * `setTimeout(0)` drain the receiving engine fires.
72
+ */
73
+ updatePollTarget?: OwnerSideUpdatePollTarget;
74
+ /**
75
+ * Invoked after this engine reclaims a stranded claim, to actually drive the
76
+ * workflow. Taking the claim only moves durable ownership keys; without this
77
+ * the reclaimed workflow makes no progress while this engine's renewal keeps
78
+ * its claim alive, shielding it from any engine that would have resumed it.
79
+ */
80
+ onWorkflowClaimReclaimed?: (workflowId: string) => Promise<void>;
81
+ /**
82
+ * Optional workflow-type eligibility check, forwarded to
83
+ * {@link createWorkflowClaimReclaimTarget}'s `isTypeRegistered` parameter.
84
+ * Omitted skips the check, matching that function's own default. See its
85
+ * doc for why an engine that cannot execute a candidate's registered type
86
+ * must never win its claim in the first place.
87
+ */
88
+ isWorkflowTypeRegistered?: (workflowType: string) => boolean;
89
+ };
90
+ /**
91
+ * Everything `Engine` needs to hold on `EngineInternals` once bootstrap
92
+ * succeeds. `metrics` is the concrete {@link WorkflowClaimMetricsCollector} —
93
+ * not just the structural `WorkflowClaimMetricsRecorder` interface it also
94
+ * satisfies — so a caller (tests, a later Prometheus-exporter bridge) can read
95
+ * `snapshot()` directly instead of needing a second recorder reference.
96
+ */
97
+ export type WorkflowLeaseOwnershipBootstrapResult = {
98
+ registry: WorkflowClaimRegistry;
99
+ renewalTask: WorkflowClaimRenewalTask;
100
+ metrics: WorkflowClaimMetricsCollector;
101
+ };
102
+ /**
103
+ * Structural sources {@link buildOwnerSideSignalPollTarget} composes into a
104
+ * real {@link OwnerSideSignalPollTarget} (ADR 0002 § "Signal delivery needs
105
+ * more than a classification"). Defined as small closures rather than
106
+ * accepting `EngineInternals` directly — see this module's doc comment.
107
+ * `src/core/engine/index.ts` is the only expected caller.
108
+ *
109
+ * A parked `waitForSignal()` lands in one of two disjoint in-memory
110
+ * populations, and both need owner-side polling to survive a signal
111
+ * delivered on a non-owning engine:
112
+ *
113
+ * - **Checkpoint-parked** (`listParkedInlineWorkflowIds`/`parkedSignalName`):
114
+ * a top-level `waitForSignal()` with no live `ctx.onQuery`/update handler
115
+ * is evicted from memory (`inline-parking.ts`'s `parkInlineWorkflowAfterCheckpoint`)
116
+ * and tracked only as `EngineInternals.parkedInlineWorkflows` plus the
117
+ * checkpointed operation itself — recovered here via
118
+ * `EngineInternals.pendingTimelineEntries`, which every checkpoint commit
119
+ * sets to the just-persisted operation's timeline summary
120
+ * (`checkpoint-io.ts`) and only clears once that operation is finalized.
121
+ * For a still-parked `wait-signal` entry, `operationLabel` IS the awaited
122
+ * signal name (`state-utilities.ts`'s `getTimelineOperationLabel`).
123
+ * - **Live in-memory waiters** (`listSignalWaiterEntries`): a `ctx.race`/
124
+ * `ctx.all` wait-signal branch, or a top-level `waitForSignal()` on a
125
+ * workflow WITH live query/update handlers, registers into
126
+ * `EngineInternals.signalWaiters`/`signalWaitersByWorkflow` instead
127
+ * (`operations-coordination.ts`, `coordination-branch-executors.ts`) — the
128
+ * generator's turn stays alive, blocked on an unresolved in-process
129
+ * promise, rather than being evicted.
130
+ */
131
+ export type OwnerSideSignalPollSources = {
132
+ /** `EngineInternals.parkedInlineWorkflows` — this engine's checkpoint-parked (memory-evicted) workflow ids. */
133
+ listParkedInlineWorkflowIds(): Iterable<string>;
134
+ /** Whether `workflowId` is currently checkpoint-parked. */
135
+ isParkedInlineWorkflow(workflowId: string): boolean;
136
+ /**
137
+ * The signal name a checkpoint-parked workflow is currently awaiting —
138
+ * `undefined` when `workflowId` is not parked on a `wait-signal`
139
+ * operation. See this type's doc comment for the `pendingTimelineEntries`
140
+ * source.
141
+ */
142
+ parkedSignalName(workflowId: string): string | undefined;
143
+ /**
144
+ * Every `[workflowId, waiterKey]` pair from
145
+ * `EngineInternals.signalWaitersByWorkflow`. `waiterKey` is always
146
+ * `` `${workflowId}:${signalName}` ``.
147
+ */
148
+ listSignalWaiterEntries(): Iterable<readonly [workflowId: string, waiterKey: string]>;
149
+ /** `signals.ts`'s `hasBufferedSignal`. */
150
+ hasBufferedSignal(workflowId: string, signalName: string): Promise<boolean>;
151
+ /** `inline-parking.ts`'s `resumeParkedInlineWorkflow`, bound to this engine. */
152
+ resumeParkedInlineWorkflow(workflowId: string): Promise<void>;
153
+ /**
154
+ * Wake exactly the in-memory signal waiter registered under `waiterKey` —
155
+ * mirrors `signals.ts`'s (unexported) `deliverBufferedSignals` waiter
156
+ * branch: release the waiter via the exported `releaseSignalWaiter`, then
157
+ * invoke it. A no-op when no waiter is currently registered under that
158
+ * exact key (already consumed by something else since discovery).
159
+ */
160
+ wakeSignalWaiter(workflowId: string, waiterKey: string): void;
161
+ /**
162
+ * Optional pre-wake ownership confirmation for the LIVE in-memory waiter
163
+ * branch only (WFT-79 Finding 1). Mirrors `wake-ownership-guard.ts`'s
164
+ * `confirmWakeOwnership` three-way decision tree: `'discard'` means this
165
+ * engine no longer holds the claim generation it registered the waiter
166
+ * under, so `wakeSignalWaiter` must NOT be called — doing so would let a
167
+ * signal buffered for a `ctx.race()`/`ctx.all()` branch wake a generator
168
+ * this engine has since been deposed from, driving it concurrently with
169
+ * whichever engine now legitimately owns the workflow (the exact
170
+ * duplicate-execution hazard ADR 0002 exists to close). `'proceed'` (or
171
+ * this source being omitted entirely) preserves the pre-guard behavior:
172
+ * every confirmed-buffered live waiter is woken unconditionally.
173
+ *
174
+ * **Not called for the checkpoint-parked branch above.** A checkpoint-parked
175
+ * resume goes through `resumeParkedInlineWorkflow` →
176
+ * `resumeWorkflowFromStorage`'s `acquireStandaloneClaimBeforeResume`, which
177
+ * re-confirms this engine's cached claim generation against the durable
178
+ * holder (via `wakeOwnershipCheck`) before resuming, and throws
179
+ * `WorkflowClaimUnavailableError` rather than replaying against a stale
180
+ * generation when that check fails — guarding it here too would be
181
+ * redundant, not unsafe, but the finding is scoped to the live-waiter
182
+ * release this module makes directly.
183
+ *
184
+ * **Required, deliberately.** This module cannot call `confirmWakeOwnership`
185
+ * directly: that helper takes `EngineInternals`, and this module stays
186
+ * independent of that concrete shape (see this module's doc comment) so it
187
+ * never couples to `src/core/engine/index.ts`'s internals layout. The caller
188
+ * therefore supplies it — `src/core/engine/index.ts` passes
189
+ * `(workflowId) => confirmWakeOwnership(internals, workflowId, 'signal')`.
190
+ *
191
+ * It is NOT optional, because an optional fence is one that silently does
192
+ * nothing when a call site forgets it — which is exactly how the owner-side
193
+ * signal poll shipped unwired in the first place. Making it required turns a
194
+ * missing fence into a compile error rather than a duplicate-execution bug
195
+ * that only a reviewer can catch. A caller that genuinely needs no fence
196
+ * (`ownership: 'none'`/`'lease'`) still passes one; `confirmWakeOwnership`
197
+ * returns `'proceed'` when no claim registry is installed.
198
+ */
199
+ confirmSignalWakeOwnership(workflowId: string): Promise<'proceed' | 'discard'>;
200
+ };
201
+ /**
202
+ * Compose {@link OwnerSideSignalPollSources} into a real
203
+ * {@link OwnerSideSignalPollTarget}.
204
+ *
205
+ * `wakeWorkflow` re-confirms `hasBufferedSignal` for each live waiter key it
206
+ * owns before firing it, rather than trusting the outer poll's per-entry
207
+ * probe: `runOwnerSideSignalPoll` calls `wakeWorkflow(workflowId)` with no
208
+ * `signalName` (`owner-side-signal-poll.ts`'s fixed contract), and one
209
+ * workflow can have multiple live waiter keys at once (distinct `ctx.race`
210
+ * branches waiting on different signals). Firing all of them because ONE was
211
+ * confirmed buffered would spuriously "win" a branch whose signal never
212
+ * arrived — `coordination-branch-executors.ts`'s race branch resolves
213
+ * unconditionally once its waiter fires, with no re-check.
214
+ */
215
+ export declare function buildOwnerSideSignalPollTarget(sources: OwnerSideSignalPollSources): OwnerSideSignalPollTarget;
216
+ /**
217
+ * Adapt a {@link WorkflowClaimRegistry} to the renewal task's
218
+ * {@link WorkflowClaimRenewalTarget} contract. `registry.renew()`'s
219
+ * discriminated result never throws on a lost CAS — background scanning must
220
+ * isolate a per-workflow failure and continue, so every `WorkflowClaimRegistry`
221
+ * method returns a result instead of throwing. This adapter is where that gets
222
+ * converted to the renewal task's reject-on-failure contract, without adding a
223
+ * throwing method to the registry itself, so the registry's "none throw"
224
+ * contract stays intact for every other caller.
225
+ *
226
+ * `'renewed'` resolves. `'not-held'` also resolves rather than rejects: it is
227
+ * the benign race against this engine's own `release()` (the registry stops
228
+ * new renewals for a workflow before it reads the bytes `release()` conditions
229
+ * on — see `workflow-claim-registry.ts`'s module doc), not a lost claim, and
230
+ * counting it as a renewal failure would inflate
231
+ * `weft_workflow_claim_renewal_failures_total` on ordinary terminal shutdown.
232
+ * Only `'lost'` rejects. The registry's own `renew()` already emits
233
+ * `WeftWorkflowClaimLostWarning` on a `'lost'` result, so this adapter does not
234
+ * double-emit.
235
+ *
236
+ * Exported (only) so its three-branch mapping can be pinned directly against a
237
+ * canned `WorkflowClaimRenewResult`, without needing a storage-timing race to
238
+ * reach `'not-held'` through a real registry.
239
+ */
240
+ export declare function createWorkflowClaimRenewalTarget(registry: WorkflowClaimRegistry, metrics?: WorkflowClaimMetricsCollector): WorkflowClaimRenewalTarget;
241
+ /**
242
+ * Run Gate 1 + Gate 2 for `ownership: 'workflow-lease'`, then construct this
243
+ * engine's {@link WorkflowClaimRegistry} and claim-renewal task. Does not
244
+ * start the renewal task's interval — the caller decides interval-mode vs.
245
+ * `runMaintenance()`-driven mode, since only it knows `backgroundTaskMode`.
246
+ * Throws whatever {@link bootstrapOwnershipGates} throws (a bare `Error` for
247
+ * Gate 1, `OwnershipModeMismatchError` for Gate 2) before constructing
248
+ * anything durable-adjacent — a failed gate leaves no registry, no renewal
249
+ * task, and no metrics recorder behind.
250
+ *
251
+ * The renewal task's `onPassComplete` seam bridges every pass into the
252
+ * returned {@link WorkflowClaimMetricsRecorder}: each `'failed'` renewal
253
+ * outcome records one `weft_workflow_claim_renewal_failures_total`, and the
254
+ * registry's own currently-held-id count (read fresh after the pass, since a
255
+ * failed renewal can drop a claim mid-pass) sets `weft_workflow_claims_active`.
256
+ *
257
+ * **`renewalTask.stop()` also quiesces the reclaim target (WFT-79 Finding 2).**
258
+ * `createWorkflowClaimRenewalTask`'s own `stop()` only clears the interval —
259
+ * a reclaim pass already in flight (started by the last tick before `stop()`)
260
+ * still runs to completion, per that module's own doc. Left alone, a
261
+ * disposal that races an in-flight interval-driven pass could let the
262
+ * {@link createWorkflowClaimReclaimTarget reclaim target} land a fresh
263
+ * takeover/acquire CAS, or drive a freshly reclaimed workflow's `onReclaimed`
264
+ * against a host that is mid-teardown, AFTER `Engine`'s disposal path has
265
+ * already snapshotted the registry for `WorkflowClaimRegistry.releaseAll()`.
266
+ * The `renewalTask` this function returns wraps the raw task's `stop` so it
267
+ * synchronously calls {@link WorkflowClaimReclaimTargetHandle.markDisposing}
268
+ * first: since JS has no preemption, that flag flip happens strictly before
269
+ * any other code runs, so every reclaim-target checkpoint after an `await`
270
+ * (including one a suspended continuation resumes into after `stop()` was
271
+ * called) observes it. A checkpoint that finds itself disposing self-releases
272
+ * any claim it just landed rather than driving it — see that module's doc —
273
+ * so the hazard closes without `Engine`'s disposal path (`index.ts`, out of
274
+ * this module's scope) needing to await anything new. This wrapping makes
275
+ * `stop()` permanent: a later `start()` on the returned task still clears
276
+ * `disposing` for renewal-interval purposes, but the reclaim target itself
277
+ * never un-disposes. That is intentional — `index.ts` never restarts a
278
+ * renewal task after detaching it — and is pinned by a test.
279
+ */
280
+ export declare function bootstrapWorkflowLeaseOwnership(options: WorkflowLeaseOwnershipBootstrapOptions): Promise<WorkflowLeaseOwnershipBootstrapResult>;