@lostgradient/weft 0.20.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/core/engine/activity-reconciliation.d.ts +2 -2
- package/dist/core/engine/activity-reconciliation.js +12 -8
- package/dist/core/engine/async-activity-completion.js +12 -4
- package/dist/core/engine/async-activity-records.d.ts +11 -1
- package/dist/core/engine/async-activity-records.js +4 -3
- package/dist/core/engine/attributes-tags.d.ts +11 -1
- package/dist/core/engine/attributes-tags.js +4 -3
- package/dist/core/engine/bulk-operations-purge.js +5 -2
- package/dist/core/engine/bulk-operations.js +17 -5
- package/dist/core/engine/callback-creators-bundles.js +1 -1
- package/dist/core/engine/callback-creators-core.js +4 -2
- package/dist/core/engine/checkpoint-io.js +1 -1
- package/dist/core/engine/child-workflow.js +2 -1
- package/dist/core/engine/completed-review-storage.js +1 -1
- package/dist/core/engine/condition-waiters.d.ts +45 -3
- package/dist/core/engine/condition-waiters.js +12 -3
- package/dist/core/engine/disposal.js +6 -0
- package/dist/core/engine/engine-internal-types.d.ts +12 -4
- package/dist/core/engine/fenced-write.d.ts +46 -15
- package/dist/core/engine/fenced-write.js +39 -6
- package/dist/core/engine/finalizer-state.d.ts +3 -1
- package/dist/core/engine/handle-result.d.ts +49 -1
- package/dist/core/engine/handle-result.js +95 -20
- package/dist/core/engine/index.d.ts +13 -5
- package/dist/core/engine/index.js +155 -5
- package/dist/core/engine/inline-launch-queue.js +9 -0
- package/dist/core/engine/internals.d.ts +73 -0
- package/dist/core/engine/invoke-update-handler.d.ts +19 -0
- package/dist/core/engine/invoke-update-handler.js +7 -0
- package/dist/core/engine/lease-deposition.d.ts +132 -0
- package/dist/core/engine/lease-deposition.js +31 -0
- package/dist/core/engine/lease-errors.d.ts +80 -9
- package/dist/core/engine/lease-errors.js +28 -2
- package/dist/core/engine/lifecycle/resume.js +3 -1
- package/dist/core/engine/lifecycle/standalone-claim-acquire.d.ts +33 -0
- package/dist/core/engine/lifecycle/standalone-claim-acquire.js +25 -0
- package/dist/core/engine/lifecycle/start-commit.js +20 -6
- package/dist/core/engine/lifecycle/transition.d.ts +15 -1
- package/dist/core/engine/lifecycle/transition.js +6 -3
- package/dist/core/engine/operations-activity.js +1 -1
- package/dist/core/engine/operations-time.js +21 -7
- package/dist/core/engine/owner-side-signal-poll.d.ts +132 -0
- package/dist/core/engine/owner-side-signal-poll.js +22 -0
- package/dist/core/engine/owner-side-update-poll.d.ts +116 -0
- package/dist/core/engine/owner-side-update-poll.js +22 -0
- package/dist/core/engine/ownership-bootstrap.d.ts +280 -0
- package/dist/core/engine/ownership-bootstrap.js +89 -0
- package/dist/core/engine/ownership-mode-marker.d.ts +82 -0
- package/dist/core/engine/ownership-mode-marker.js +52 -0
- package/dist/core/engine/ownership-options.d.ts +22 -6
- package/dist/core/engine/ownership-options.js +42 -12
- package/dist/core/engine/pending-updates.js +35 -24
- package/dist/core/engine/queries.d.ts +70 -0
- package/dist/core/engine/queries.js +58 -4
- package/dist/core/engine/schedules.js +1 -1
- package/dist/core/engine/signals.d.ts +11 -1
- package/dist/core/engine/signals.js +11 -7
- package/dist/core/engine/sleep-timer-acknowledgements.d.ts +68 -0
- package/dist/core/engine/sleep-timer-acknowledgements.js +19 -0
- package/dist/core/engine/storage-io.d.ts +58 -10
- package/dist/core/engine/storage-io.js +27 -9
- package/dist/core/engine/termination/cleanup.d.ts +5 -4
- package/dist/core/engine/termination/complete.js +23 -4
- package/dist/core/engine/termination/finalizer-claim.js +4 -4
- package/dist/core/engine/termination/state-commit-callbacks.d.ts +15 -0
- package/dist/core/engine/termination/state-commit-callbacks.js +0 -0
- package/dist/core/engine/termination/suspend.js +1 -1
- package/dist/core/engine/update-validation.d.ts +28 -0
- package/dist/core/engine/update-validation.js +37 -0
- package/dist/core/engine/updates.d.ts +17 -16
- package/dist/core/engine/updates.js +24 -47
- package/dist/core/engine/wake-ownership-check.d.ts +86 -0
- package/dist/core/engine/wake-ownership-check.js +19 -0
- package/dist/core/engine/wake-ownership-guard.d.ts +46 -0
- package/dist/core/engine/wake-ownership-guard.js +23 -0
- package/dist/core/engine/workflow-claim-codec.d.ts +64 -0
- package/dist/core/engine/workflow-claim-codec.js +44 -0
- package/dist/core/engine/workflow-claim-cooldown.d.ts +60 -0
- package/dist/core/engine/workflow-claim-cooldown.js +17 -0
- package/dist/core/engine/workflow-claim-fold.d.ts +82 -0
- package/dist/core/engine/workflow-claim-fold.js +53 -0
- package/dist/core/engine/workflow-claim-metrics.d.ts +131 -0
- package/dist/core/engine/workflow-claim-metrics.js +62 -0
- package/dist/core/engine/workflow-claim-reclaim-scan.d.ts +96 -0
- package/dist/core/engine/workflow-claim-reclaim-scan.js +66 -0
- package/dist/core/engine/workflow-claim-reclaim-target.d.ts +107 -0
- package/dist/core/engine/workflow-claim-reclaim-target.js +155 -0
- package/dist/core/engine/workflow-claim-registry.d.ts +227 -0
- package/dist/core/engine/workflow-claim-registry.js +198 -0
- package/dist/core/engine/workflow-claim-renewal-interval.d.ts +50 -0
- package/dist/core/engine/workflow-claim-renewal-interval.js +132 -0
- package/dist/core/engine/workflow-claim-renewal-subpasses.d.ts +265 -0
- package/dist/core/engine/workflow-claim-renewal-subpasses.js +104 -0
- package/dist/core/engine/workflow-claim-renewal-task.d.ts +159 -0
- package/dist/core/engine/workflow-claim-renewal-task.js +65 -0
- package/dist/core/engine/workflow-claim-transitions.d.ts +186 -0
- package/dist/core/engine/workflow-claim-transitions.js +120 -0
- package/dist/core/inline-execution-strategy.d.ts +20 -0
- package/dist/core/inline-execution-strategy.js +15 -4
- package/dist/core/scheduler/scheduler-class.js +3 -3
- package/dist/core/scheduler/timer-sources.d.ts +13 -0
- package/dist/core/scheduler/timer-sources.js +1 -1
- package/dist/core/types/options.d.ts +17 -1
- package/dist/core/weft-error.d.ts +1 -1
- package/dist/core/weft-error.js +2 -1
- package/dist/http.js +2 -2
- package/dist/index.d.ts +2 -1
- package/dist/index.js +8 -0
- package/dist/indexeddb.js +1 -1
- package/dist/json-schema.js +3 -3
- package/dist/observability/index.js +2 -2
- package/dist/storage/auto.js +1 -1
- package/dist/storage/bun-sql.js +113 -0
- package/dist/storage/compressed-storage.js +1 -1
- package/dist/storage/index.d.ts +38 -35
- package/dist/storage/interface.d.ts +87 -125
- package/dist/storage/interface.js +1 -1
- package/dist/storage/key-encoding.d.ts +51 -0
- package/dist/storage/key-encoding.js +13 -0
- package/dist/storage/key-prefixes.d.ts +1 -1
- package/dist/storage/key-prefixes.js +3 -0
- package/dist/storage/lmdb.js +1 -1
- package/dist/storage/memory.js +1 -1
- package/dist/storage/neon.js +2 -2
- package/dist/storage/node-sqlite.js +113 -0
- package/dist/storage/ownership-keys.d.ts +55 -0
- package/dist/storage/ownership-keys.js +6 -0
- package/dist/storage/postgres.js +2 -2
- package/dist/storage/resolve.js +1 -1
- package/dist/storage/scoped-storage.js +1 -1
- package/dist/storage/testing.js +1 -1
- package/dist/storage/turso.js +2 -2
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/web-extension.js +1 -1
- package/dist/worker/protocol.js +1 -1
- package/package.json +1 -1
|
@@ -32,6 +32,7 @@ import type { Scheduler } from '../scheduler.ts';
|
|
|
32
32
|
import type { Checkpoint, StartWorkflowOptions } from '../types.ts';
|
|
33
33
|
import type { UpdateCoordinator } from '../updates.ts';
|
|
34
34
|
import type { WorkflowVersionTuple } from '../workflow-version-tuple.ts';
|
|
35
|
+
import type { RecoveredWorkflowInfo } from './lifecycle/shared.ts';
|
|
35
36
|
import type { ActivityHeartbeatKey } from './activity-heartbeat-tracking.ts';
|
|
36
37
|
import type { PendingTimelineEntry, QueuedInlineWorkflowExecutionStart, RegistrationEntry, ResolvedOptions, TrackedWaiterKeys, WorkflowResultWaiter } from './engine-internal-types.ts';
|
|
37
38
|
import type { EngineCleanupIntervalDisposalTracker } from './engine-leak-warnings.ts';
|
|
@@ -40,6 +41,9 @@ import type { WorkflowFeedListener } from './index.ts';
|
|
|
40
41
|
import type { LeaseManager } from './lease-manager.ts';
|
|
41
42
|
import type { ScheduleHandleEngine } from './schedule-handle.ts';
|
|
42
43
|
import type { SecondInstanceDetector } from './second-instance-detector.ts';
|
|
44
|
+
import type { WorkflowClaimMetricsRecorder } from './workflow-claim-metrics.ts';
|
|
45
|
+
import type { WorkflowClaimRegistry } from './workflow-claim-registry.ts';
|
|
46
|
+
import type { WorkflowClaimRenewalTask } from './workflow-claim-renewal-task.ts';
|
|
43
47
|
export type SleepTimerAcknowledgementWaiter = {
|
|
44
48
|
fireAt: number;
|
|
45
49
|
operationId: string;
|
|
@@ -239,6 +243,43 @@ export interface EngineInternals {
|
|
|
239
243
|
* Step 1; epoch fencing of durable writes (Step 2) is what makes it enforceable.
|
|
240
244
|
*/
|
|
241
245
|
leaseManager: LeaseManager | null;
|
|
246
|
+
/**
|
|
247
|
+
* The active per-workflow claim registry for `ownership: 'workflow-lease'`;
|
|
248
|
+
* `null` under `'none'`/`'lease'` and, for now, ALSO under `'workflow-lease'`
|
|
249
|
+
* itself — wiring construction (Gate 1/Gate 2, actually instantiating the
|
|
250
|
+
* registry, and folding `acquire()` into start/resume/delayed-start-fire) is a
|
|
251
|
+
* later stage. Its presence here lets {@link commitFencedEngineWrite} read
|
|
252
|
+
* {@link WorkflowClaimRegistry.currentEpochBytes} for a workflow-scoped write
|
|
253
|
+
* without that later stage having to touch the fencing path again. Until it is
|
|
254
|
+
* populated, every workflow-scoped write under `'workflow-lease'` fails closed
|
|
255
|
+
* with {@link EngineDeposedError} — correct, not a bug: this engine holds no
|
|
256
|
+
* claim for any workflow yet.
|
|
257
|
+
*/
|
|
258
|
+
workflowClaimRegistry: WorkflowClaimRegistry | null;
|
|
259
|
+
/**
|
|
260
|
+
* The active per-workflow claim-renewal task for `ownership: 'workflow-lease'`;
|
|
261
|
+
* `null` under `'none'`/`'lease'` and until `#bootstrapOwnershipIfConfigured`
|
|
262
|
+
* completes. Renews every claim this engine holds (active or parked) on its
|
|
263
|
+
* own cadence — independent of the durable-timer scheduler, so `startScheduler:
|
|
264
|
+
* false` still renews claims — or via `runMaintenance()` under
|
|
265
|
+
* `backgroundTasks: 'manual'`, which never starts its interval. Stopped
|
|
266
|
+
* (interval cleared, never awaited — stopping is synchronous) and detached at
|
|
267
|
+
* every `disposeEngine` call site; releasing the claims it was renewing is the
|
|
268
|
+
* separate, best-effort `workflowClaimRegistry.releaseAll()` step each
|
|
269
|
+
* disposal path drives on its own schedule.
|
|
270
|
+
*/
|
|
271
|
+
workflowClaimRenewalTask: WorkflowClaimRenewalTask | null;
|
|
272
|
+
/**
|
|
273
|
+
* Bounded-cardinality observability recorder for the `ownership:
|
|
274
|
+
* 'workflow-lease'` claim protocol (ADR 0002 § Observability); `null` under
|
|
275
|
+
* `'none'`/`'lease'` and until `#bootstrapOwnershipIfConfigured` completes.
|
|
276
|
+
* One instance per engine process, shared by this engine's claim-renewal task
|
|
277
|
+
* (which feeds it renewal-failure counts and the active-claim gauge after
|
|
278
|
+
* every pass) and, in a later stage, its claim-acquiring entry points (start,
|
|
279
|
+
* resume, delayed-start fire), which would record `acquired`/`takeover`/
|
|
280
|
+
* `lost_race`/`deposed`/`backoff_skipped` attempts into the same instance.
|
|
281
|
+
*/
|
|
282
|
+
workflowClaimMetrics: WorkflowClaimMetricsRecorder | null;
|
|
242
283
|
/**
|
|
243
284
|
* The in-flight lease acquisition, or `null` when none is running. Set while
|
|
244
285
|
* `#acquireLeaseIfConfigured` awaits `acquire()` (which can park for the whole
|
|
@@ -248,6 +289,15 @@ export interface EngineInternals {
|
|
|
248
289
|
* than leaking until TTL on an already-disposed engine.
|
|
249
290
|
*/
|
|
250
291
|
inFlightLeaseAcquire: Promise<void> | null;
|
|
292
|
+
/**
|
|
293
|
+
* The in-flight `ownership: 'workflow-lease'` bootstrap (Gate 1, Gate 2, and
|
|
294
|
+
* claim-registry/renewal-task construction), or `null` when none is running.
|
|
295
|
+
* Mirrors {@link inFlightLeaseAcquire}'s idempotency shape: concurrent
|
|
296
|
+
* `Engine.create` + `recoverAll` (and repeated `recoverAll`/`runMaintenance`)
|
|
297
|
+
* callers await this same promise rather than racing the gates twice. Always
|
|
298
|
+
* `null` under `ownership: 'none'`/`'lease'`, which never run this bootstrap.
|
|
299
|
+
*/
|
|
300
|
+
inFlightOwnershipBootstrap: Promise<void> | null;
|
|
251
301
|
/**
|
|
252
302
|
* Set to `true` the instant this engine is detected as deposed under
|
|
253
303
|
* `ownership: 'lease'` — either a fenced durable write's CAS failed against a
|
|
@@ -296,9 +346,32 @@ export interface EngineInternals {
|
|
|
296
346
|
* updates whose delete never committed).
|
|
297
347
|
*/
|
|
298
348
|
deliveredPendingUpdateIds: Map<string, Set<string>>;
|
|
349
|
+
/**
|
|
350
|
+
* The most recent `recoverAll({ onRecoveredWorkflow })` hook installed by
|
|
351
|
+
* this host, retained so a LATER same-engine reclaim (ADR 0002's recurring
|
|
352
|
+
* scan, after `recover: false` skipped this workflow entirely at startup
|
|
353
|
+
* because another engine held it then) still runs it before resuming.
|
|
354
|
+
* Without this, `index.ts`'s `onWorkflowClaimReclaimed` callback would pass
|
|
355
|
+
* `undefined` for every reclaim-driven resume, silently skipping whatever
|
|
356
|
+
* host initialization/validation the hook contract guarantees runs before
|
|
357
|
+
* recovered execution. `undefined` until `recoverAll()` is ever called with
|
|
358
|
+
* this option.
|
|
359
|
+
*/
|
|
360
|
+
onRecoveredWorkflowHook: ((info: RecoveredWorkflowInfo) => void | Promise<void>) | undefined;
|
|
299
361
|
cancelHandlersByWorkflow: Map<string, Array<() => Promise<void> | void>>;
|
|
300
362
|
reviewTimerIds: Map<string, string[]>;
|
|
301
363
|
pendingWebhooks: Set<AbortController>;
|
|
364
|
+
/**
|
|
365
|
+
* Every `setTimeout` handle `scheduleCrossEngineResultPollIfPending`
|
|
366
|
+
* (`handle-result.ts`) currently has pending, so disposal can cancel them.
|
|
367
|
+
* Without this, disposing an automatic-mode engine while a cross-engine
|
|
368
|
+
* result waiter is parked rejects and clears the waiter but leaves this
|
|
369
|
+
* untracked timer alive — the timer queue retains the waiter, engine
|
|
370
|
+
* internals, and storage reference until `workflowClaimRenewIntervalMs`
|
|
371
|
+
* elapses, and its callback then calls `bootstrapWorkflowResultResolver`
|
|
372
|
+
* against already-disposed storage.
|
|
373
|
+
*/
|
|
374
|
+
pendingResultPollTimers: Set<ReturnType<typeof setTimeout>>;
|
|
302
375
|
alertManager: AlertManager | null;
|
|
303
376
|
eventLogHeads: Map<string, Readonly<EventHeadRecord>>;
|
|
304
377
|
workflowFeedListeners: Map<string, Set<WorkflowFeedListener>>;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Invoking an `ctx.onUpdate()` handler, and the type of the handler itself.
|
|
3
|
+
*
|
|
4
|
+
* Separate from `updates.ts` because both the inline dispatch path and
|
|
5
|
+
* `pending-updates.ts`'s replay path call it, and because it is pure handler
|
|
6
|
+
* invocation with no coupling to the in-memory waiter maps or the durable
|
|
7
|
+
* coordinated-update protocol that make up the rest of update delivery.
|
|
8
|
+
*
|
|
9
|
+
* @module core/engine/invoke-update-handler
|
|
10
|
+
*/
|
|
11
|
+
import type { EngineInternals } from './internals.ts';
|
|
12
|
+
/** An `ctx.onUpdate()` handler as stored on a live workflow context. */
|
|
13
|
+
export type InlineUpdateHandler = (payload: unknown) => unknown;
|
|
14
|
+
/**
|
|
15
|
+
* Invoke an update handler, checking that it does not return a generator.
|
|
16
|
+
* Centralises the runtime generator guard for both the inline-handler path
|
|
17
|
+
* in `update()` and the pending-drain path on resume.
|
|
18
|
+
*/
|
|
19
|
+
export declare function invokeUpdateHandler(_internals: EngineInternals, name: string, handler: (payload: unknown) => unknown, payload: unknown): Promise<unknown>;
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { isGeneratorResult } from "../step-context.js";
|
|
2
|
+
export async function invokeUpdateHandler(_internals, name, handler, payload) {
|
|
3
|
+
const result = handler(payload);
|
|
4
|
+
if (isGeneratorResult(result))
|
|
5
|
+
throw TypeError(`Update handler "${name}" returned a generator. Update handlers must return a plain value or a Promise, not a generator.`);
|
|
6
|
+
return await result;
|
|
7
|
+
}
|
|
@@ -64,3 +64,135 @@ export declare const ENGINE_LEASE_LOST_WARNING_NAME = "WeftEngineLeaseLostWarnin
|
|
|
64
64
|
* not advance in-memory state past a durable write that did not land.
|
|
65
65
|
*/
|
|
66
66
|
export declare function handleDeposition(internals: EngineInternals): void;
|
|
67
|
+
/**
|
|
68
|
+
* `name` of the operator warning emitted when `ownership: 'workflow-lease'`
|
|
69
|
+
* loses one workflow's claim — a renewal CAS failure (a successor stole the
|
|
70
|
+
* claim) or a fenced write rejected because a newer epoch now holds it. Unlike
|
|
71
|
+
* {@link ENGINE_LEASE_LOST_WARNING_NAME}, this is per-workflow, not per-engine:
|
|
72
|
+
* the rest of this engine's claimed workflows are unaffected and keep running.
|
|
73
|
+
*
|
|
74
|
+
* @example
|
|
75
|
+
* ```ts
|
|
76
|
+
* import { WORKFLOW_CLAIM_LOST_WARNING_NAME } from '@lostgradient/weft';
|
|
77
|
+
*
|
|
78
|
+
* process.on('warning', (warning) => {
|
|
79
|
+
* if (warning.name === WORKFLOW_CLAIM_LOST_WARNING_NAME) {
|
|
80
|
+
* console.error('lost a workflow claim', warning.message);
|
|
81
|
+
* }
|
|
82
|
+
* });
|
|
83
|
+
* ```
|
|
84
|
+
*/
|
|
85
|
+
export declare const WORKFLOW_CLAIM_LOST_WARNING_NAME = "WeftWorkflowClaimLostWarning";
|
|
86
|
+
/**
|
|
87
|
+
* Operator diagnostic for a lost per-workflow ownership claim under
|
|
88
|
+
* `ownership: 'workflow-lease'` (see
|
|
89
|
+
* [ADR 0002](../../../documentation/contributing/architecture-decisions/0002-multiengine-per-workflow-ownership.md)).
|
|
90
|
+
* Carries the affected `workflowId` as a real field — `process.emitWarning(message,
|
|
91
|
+
* name)` only accepts strings, so this is emitted as the warning object itself
|
|
92
|
+
* (`process.emitWarning(warning)`) rather than passed as a message, which lets
|
|
93
|
+
* consumers read `warning.workflowId` directly instead of parsing it back out
|
|
94
|
+
* of the message text.
|
|
95
|
+
*
|
|
96
|
+
* @example
|
|
97
|
+
* ```ts
|
|
98
|
+
* import { WeftWorkflowClaimLostWarning } from '@lostgradient/weft';
|
|
99
|
+
*
|
|
100
|
+
* process.on('warning', (warning) => {
|
|
101
|
+
* if (warning instanceof WeftWorkflowClaimLostWarning) {
|
|
102
|
+
* console.error('deposed workflow:', warning.workflowId);
|
|
103
|
+
* }
|
|
104
|
+
* });
|
|
105
|
+
* ```
|
|
106
|
+
*/
|
|
107
|
+
export declare class WeftWorkflowClaimLostWarning extends Error {
|
|
108
|
+
readonly workflowId: string;
|
|
109
|
+
constructor(workflowId: string);
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* The wake path a stale in-memory resolver was discarded from, for
|
|
113
|
+
* {@link WeftWorkflowWakeDiscardedWarning}. Mirrors the `wakeOwnershipCheck`
|
|
114
|
+
* call sites named in ADR 0002: a durable timer firing, a re-evaluated
|
|
115
|
+
* `ctx.waitUntil()` condition, a delivered signal, an async-activity
|
|
116
|
+
* completion/failure, a child workflow's termination, or a deferred inline
|
|
117
|
+
* macrotask drive.
|
|
118
|
+
*
|
|
119
|
+
* @example
|
|
120
|
+
* ```ts
|
|
121
|
+
* import type { WorkflowWakeKind } from '@lostgradient/weft';
|
|
122
|
+
*
|
|
123
|
+
* const kind: WorkflowWakeKind = 'signal';
|
|
124
|
+
* void kind;
|
|
125
|
+
* ```
|
|
126
|
+
*/
|
|
127
|
+
export type WorkflowWakeKind = 'sleep' | 'wait-condition' | 'signal' | 'async-activity' | 'child-completion' | 'update' | 'inline-macrotask-drive';
|
|
128
|
+
/**
|
|
129
|
+
* `name` of the operator warning emitted when `wakeOwnershipCheck` discards a
|
|
130
|
+
* stale in-memory wake resolver — it re-read the durable holder record and
|
|
131
|
+
* found this engine no longer holds the generation (engine id AND epoch) it
|
|
132
|
+
* parked the workflow under, so the resolver is dropped without driving the
|
|
133
|
+
* generator.
|
|
134
|
+
*
|
|
135
|
+
* @example
|
|
136
|
+
* ```ts
|
|
137
|
+
* import { WORKFLOW_WAKE_DISCARDED_WARNING_NAME } from '@lostgradient/weft';
|
|
138
|
+
*
|
|
139
|
+
* process.on('warning', (warning) => {
|
|
140
|
+
* if (warning.name === WORKFLOW_WAKE_DISCARDED_WARNING_NAME) {
|
|
141
|
+
* console.error('discarded a stale wake', warning.message);
|
|
142
|
+
* }
|
|
143
|
+
* });
|
|
144
|
+
* ```
|
|
145
|
+
*/
|
|
146
|
+
export declare const WORKFLOW_WAKE_DISCARDED_WARNING_NAME = "WeftWorkflowWakeDiscardedWarning";
|
|
147
|
+
/**
|
|
148
|
+
* Operator diagnostic emitted whenever `wakeOwnershipCheck` discards a stale
|
|
149
|
+
* in-memory wake resolver under `ownership: 'workflow-lease'` (see
|
|
150
|
+
* [ADR 0002](../../../documentation/contributing/architecture-decisions/0002-multiengine-per-workflow-ownership.md)).
|
|
151
|
+
* Carries the affected `workflowId` and the `wakeKind` that was discarded as
|
|
152
|
+
* real fields, for the same reason {@link WeftWorkflowClaimLostWarning} does —
|
|
153
|
+
* `process.emitWarning(message, name)` cannot carry structured data, so this is
|
|
154
|
+
* emitted as the warning object itself.
|
|
155
|
+
*
|
|
156
|
+
* @example
|
|
157
|
+
* ```ts
|
|
158
|
+
* import { WeftWorkflowWakeDiscardedWarning } from '@lostgradient/weft';
|
|
159
|
+
*
|
|
160
|
+
* process.on('warning', (warning) => {
|
|
161
|
+
* if (warning instanceof WeftWorkflowWakeDiscardedWarning) {
|
|
162
|
+
* console.error('discarded wake', warning.workflowId, warning.wakeKind);
|
|
163
|
+
* }
|
|
164
|
+
* });
|
|
165
|
+
* ```
|
|
166
|
+
*/
|
|
167
|
+
export declare class WeftWorkflowWakeDiscardedWarning extends Error {
|
|
168
|
+
readonly workflowId: string;
|
|
169
|
+
readonly wakeKind: WorkflowWakeKind;
|
|
170
|
+
constructor(workflowId: string, wakeKind: WorkflowWakeKind);
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* Emit one of the two per-workflow warnings above. Injected as a seam
|
|
174
|
+
* (defaulting to `process.emitWarning(warning)`) so later stages — the claim
|
|
175
|
+
* renewal loop and `wakeOwnershipCheck` — get deterministic test coverage of
|
|
176
|
+
* their emission sites, mirroring the `warn` option
|
|
177
|
+
* {@link createSecondInstanceDetector} already takes. Unlike that seam, which
|
|
178
|
+
* is message-only because its warning name is a single fixed constant, this
|
|
179
|
+
* one takes the warning *instance* — `WeftWorkflowClaimLostWarning` and
|
|
180
|
+
* `WeftWorkflowWakeDiscardedWarning` carry different fields, so a caller
|
|
181
|
+
* filtering or asserting on a specific warning needs the real object, not a
|
|
182
|
+
* pre-flattened string.
|
|
183
|
+
*/
|
|
184
|
+
export type EmitWorkflowLeaseWarning = (warning: WeftWorkflowClaimLostWarning | WeftWorkflowWakeDiscardedWarning) => void;
|
|
185
|
+
/**
|
|
186
|
+
* Emit {@link WeftWorkflowClaimLostWarning} for `workflowId`. `emit` defaults
|
|
187
|
+
* to `process.emitWarning(warning)`; pass a test double to assert on the
|
|
188
|
+
* emitted instance deterministically instead of scraping `process`'s global
|
|
189
|
+
* `warning` event.
|
|
190
|
+
*/
|
|
191
|
+
export declare function emitWorkflowClaimLostWarning(workflowId: string, emit?: EmitWorkflowLeaseWarning): void;
|
|
192
|
+
/**
|
|
193
|
+
* Emit {@link WeftWorkflowWakeDiscardedWarning} for `workflowId`/`wakeKind`.
|
|
194
|
+
* `emit` defaults to `process.emitWarning(warning)`; pass a test double to
|
|
195
|
+
* assert on the emitted instance deterministically instead of scraping
|
|
196
|
+
* `process`'s global `warning` event.
|
|
197
|
+
*/
|
|
198
|
+
export declare function emitWorkflowWakeDiscardedWarning(workflowId: string, wakeKind: WorkflowWakeKind, emit?: EmitWorkflowLeaseWarning): void;
|
|
@@ -8,3 +8,34 @@ export function handleDeposition(internals) {
|
|
|
8
8
|
if (tearDown !== null)
|
|
9
9
|
Promise.resolve().then(tearDown).catch(() => {});
|
|
10
10
|
}
|
|
11
|
+
export const WORKFLOW_CLAIM_LOST_WARNING_NAME = "WeftWorkflowClaimLostWarning";
|
|
12
|
+
|
|
13
|
+
export class WeftWorkflowClaimLostWarning extends Error {
|
|
14
|
+
workflowId;
|
|
15
|
+
constructor(workflowId) {
|
|
16
|
+
super(`workflow "${workflowId}" lost its ownership claim: another engine now holds it at a newer epoch. This engine stops driving that workflow; the rest of its claimed workflows are unaffected.`);
|
|
17
|
+
this.name = WORKFLOW_CLAIM_LOST_WARNING_NAME;
|
|
18
|
+
this.workflowId = workflowId;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
export const WORKFLOW_WAKE_DISCARDED_WARNING_NAME = "WeftWorkflowWakeDiscardedWarning";
|
|
22
|
+
|
|
23
|
+
export class WeftWorkflowWakeDiscardedWarning extends Error {
|
|
24
|
+
workflowId;
|
|
25
|
+
wakeKind;
|
|
26
|
+
constructor(workflowId, wakeKind) {
|
|
27
|
+
super(`discarded a stale "${wakeKind}" wake for workflow "${workflowId}": this engine no longer holds the ownership-claim generation it parked under. The in-memory resolver is dropped without driving the generator.`);
|
|
28
|
+
this.name = WORKFLOW_WAKE_DISCARDED_WARNING_NAME;
|
|
29
|
+
this.workflowId = workflowId;
|
|
30
|
+
this.wakeKind = wakeKind;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
const defaultEmitWorkflowLeaseWarning = (warning) => {
|
|
34
|
+
process.emitWarning(warning);
|
|
35
|
+
};
|
|
36
|
+
export function emitWorkflowClaimLostWarning(workflowId, emit = defaultEmitWorkflowLeaseWarning) {
|
|
37
|
+
emit(new WeftWorkflowClaimLostWarning(workflowId));
|
|
38
|
+
}
|
|
39
|
+
export function emitWorkflowWakeDiscardedWarning(workflowId, wakeKind, emit = defaultEmitWorkflowLeaseWarning) {
|
|
40
|
+
emit(new WeftWorkflowWakeDiscardedWarning(workflowId, wakeKind));
|
|
41
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { WeftError } from '../weft-error.ts';
|
|
2
|
+
import type { FencingOwnershipMode } from './workflow-claim-codec.ts';
|
|
2
3
|
/**
|
|
3
4
|
* Thrown when `ownership: 'lease'` is configured and the engine cannot acquire the
|
|
4
5
|
* storage lease within the configured wait window — another live instance still
|
|
@@ -82,18 +83,88 @@ export declare class EngineLeaseNotHeldError extends WeftError<'EngineLeaseNotHe
|
|
|
82
83
|
constructor();
|
|
83
84
|
}
|
|
84
85
|
/**
|
|
85
|
-
* Thrown internally on the durable-write path when `ownership
|
|
86
|
-
* configured and a fenced write's CAS fails because the held
|
|
87
|
-
* stale — a successor instance has taken
|
|
88
|
-
*
|
|
89
|
-
*
|
|
90
|
-
*
|
|
91
|
-
*
|
|
86
|
+
* Thrown internally on the durable-write path when either fencing `ownership`
|
|
87
|
+
* mode is configured and a fenced write's CAS fails because the held epoch is
|
|
88
|
+
* stale — a successor instance has taken over, so this instance has been
|
|
89
|
+
* deposed. It is the local unwind that stops the commit from advancing
|
|
90
|
+
* in-memory state past a durable write that did not land; the actual "this
|
|
91
|
+
* engine is deposed" reaction (set the `deposed` flag, warn the operator, tear
|
|
92
|
+
* the engine down) is driven by {@link handleDeposition} at the detection
|
|
92
93
|
* site, not by this error propagating — the inline strategy swallows turn
|
|
93
94
|
* rejections, so the throw never reaches a handler. Internal-only: it is never
|
|
94
95
|
* surfaced to user code (deposition reaches operators through the
|
|
95
|
-
* `WeftEngineLeaseLostWarning` process warning
|
|
96
|
+
* `WeftEngineLeaseLostWarning` process warning, or, under `workflow-lease`,
|
|
97
|
+
* through `WeftWorkflowClaimLostWarning`).
|
|
98
|
+
*
|
|
99
|
+
* `workflowId` is additive: under the global `ownership: 'lease'` mode
|
|
100
|
+
* deposition is store-wide and no single workflow is implicated, so existing
|
|
101
|
+
* call sites that construct `new EngineDeposedError()` with no argument keep
|
|
102
|
+
* working unchanged. Under `ownership: 'workflow-lease'`, deposition is scoped
|
|
103
|
+
* to one workflow's fenced epoch, so callers on that path supply the id and it
|
|
104
|
+
* folds into the message.
|
|
96
105
|
*/
|
|
97
106
|
export declare class EngineDeposedError extends WeftError<'EngineDeposedError'> {
|
|
98
|
-
|
|
107
|
+
readonly workflowId: string | undefined;
|
|
108
|
+
constructor(workflowId?: string);
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Thrown when an explicit, single-workflow public API — {@link Engine.resume},
|
|
112
|
+
* or a per-workflow step inside a bulk operation — loses the `acquire`,
|
|
113
|
+
* `takeover`, or standalone-resume CAS for a workflow's ownership claim under
|
|
114
|
+
* `ownership: 'workflow-lease'`, because another engine still holds a live,
|
|
115
|
+
* unexpired claim on it. Never thrown from background scanning: `recoverAll`
|
|
116
|
+
* and the scheduler tick isolate the loss to that one workflow and continue
|
|
117
|
+
* with the rest of the sweep, exactly as a background scan skips a workflow it
|
|
118
|
+
* cannot acquire rather than failing the whole pass.
|
|
119
|
+
*
|
|
120
|
+
* Like the other lease errors in this module, this does not carry a stable
|
|
121
|
+
* {@link WeftErrorCode} — match by `instanceof` rather than `.code`.
|
|
122
|
+
*
|
|
123
|
+
* @example
|
|
124
|
+
* ```ts
|
|
125
|
+
* import { WorkflowClaimUnavailableError } from '@lostgradient/weft';
|
|
126
|
+
*
|
|
127
|
+
* function isClaimContested(error: unknown): boolean {
|
|
128
|
+
* return error instanceof WorkflowClaimUnavailableError;
|
|
129
|
+
* }
|
|
130
|
+
* ```
|
|
131
|
+
*/
|
|
132
|
+
export declare class WorkflowClaimUnavailableError extends WeftError<'WorkflowClaimUnavailableError'> {
|
|
133
|
+
readonly workflowId: string;
|
|
134
|
+
readonly heldBy: string | null;
|
|
135
|
+
constructor(workflowId: string, heldBy: string | null);
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Thrown at engine construction when Gate 2 (the store-wide
|
|
139
|
+
* `ownership-mode-marker` check) finds the store already stamped with a
|
|
140
|
+
* different fencing mode than this engine is configured with. Fired
|
|
141
|
+
* immediately after Gate 1 (storage `conditionalBatch` capability) passes, for
|
|
142
|
+
* `ownership: 'lease'` or `ownership: 'workflow-lease'` — `ownership: 'none'`
|
|
143
|
+
* engines never touch the marker. Raised before `recoverAll()` or any storage
|
|
144
|
+
* write beyond the marker read/CAS attempt, so construction fails closed
|
|
145
|
+
* rather than letting two incompatible fencing modes coexist against one
|
|
146
|
+
* store.
|
|
147
|
+
*
|
|
148
|
+
* Unlike the other lease errors in this module, this one DOES carry a stable
|
|
149
|
+
* {@link WeftErrorCode} — it is a configuration-time boot blocker an operator
|
|
150
|
+
* needs to route and alert on, not an internal fencing unwind.
|
|
151
|
+
*
|
|
152
|
+
* @example
|
|
153
|
+
* ```ts
|
|
154
|
+
* import { OwnershipModeMismatchError } from '@lostgradient/weft';
|
|
155
|
+
*
|
|
156
|
+
* function describeMismatch(error: OwnershipModeMismatchError): string {
|
|
157
|
+
* return `configured "${error.configuredMode}" but store is "${error.storedMode}" ` +
|
|
158
|
+
* `(established ${new Date(error.establishedAt).toISOString()})`;
|
|
159
|
+
* }
|
|
160
|
+
* ```
|
|
161
|
+
*/
|
|
162
|
+
export declare class OwnershipModeMismatchError extends WeftError<'OwnershipModeMismatchError'> {
|
|
163
|
+
/** This engine's configured `ownership` option. */
|
|
164
|
+
readonly configuredMode: FencingOwnershipMode;
|
|
165
|
+
/** The fencing mode read from the store's `ownership-mode-marker`. */
|
|
166
|
+
readonly storedMode: FencingOwnershipMode;
|
|
167
|
+
/** The marker's `establishedAt` timestamp, for diagnosing when the mismatch was introduced. */
|
|
168
|
+
readonly establishedAt: number;
|
|
169
|
+
constructor(configuredMode: FencingOwnershipMode, storedMode: FencingOwnershipMode, establishedAt: number);
|
|
99
170
|
}
|
|
@@ -24,7 +24,33 @@ export class EngineLeaseNotHeldError extends WeftError {
|
|
|
24
24
|
}
|
|
25
25
|
|
|
26
26
|
export class EngineDeposedError extends WeftError {
|
|
27
|
-
|
|
28
|
-
|
|
27
|
+
workflowId;
|
|
28
|
+
constructor(workflowId) {
|
|
29
|
+
const scopeClause = workflowId === void 0 ? "another instance now holds the ownership lease at a newer epoch" : `another engine now holds workflow "${workflowId}"'s ownership claim at a newer epoch`;
|
|
30
|
+
super("EngineDeposedError", `Engine was deposed: a fenced durable write lost its CAS race because ${scopeClause}. This engine is halting.`);
|
|
31
|
+
this.workflowId = workflowId;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export class WorkflowClaimUnavailableError extends WeftError {
|
|
36
|
+
workflowId;
|
|
37
|
+
heldBy;
|
|
38
|
+
constructor(workflowId, heldBy) {
|
|
39
|
+
const heldClause = heldBy === null ? "" : ` (currently held by "${heldBy}")`;
|
|
40
|
+
super("WorkflowClaimUnavailableError", `Could not acquire the ownership claim for workflow "${workflowId}"${heldClause}. Another engine still holds a live, unexpired claim for it. Retry later, or investigate whether the holding engine has crashed without releasing (it will become eligible for takeover once its claim expires).`);
|
|
41
|
+
this.workflowId = workflowId;
|
|
42
|
+
this.heldBy = heldBy;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export class OwnershipModeMismatchError extends WeftError {
|
|
47
|
+
configuredMode;
|
|
48
|
+
storedMode;
|
|
49
|
+
establishedAt;
|
|
50
|
+
constructor(configuredMode, storedMode, establishedAt) {
|
|
51
|
+
super("OwnershipModeMismatchError", `This engine is configured with ownership: '${configuredMode}', but the store's ownership-mode-marker records '${storedMode}' (established at ${new Date(establishedAt).toISOString()}). Every engine sharing a store under a fencing mode must agree on that mode. Stop every engine pointed at this store, pick one mode, ` + "and restart them all \u2014 mixing modes against one store is unsafe.");
|
|
52
|
+
this.configuredMode = configuredMode;
|
|
53
|
+
this.storedMode = storedMode;
|
|
54
|
+
this.establishedAt = establishedAt;
|
|
29
55
|
}
|
|
30
56
|
}
|
|
@@ -25,6 +25,7 @@ import {
|
|
|
25
25
|
loadWorkflowStartHeaders,
|
|
26
26
|
setWorkflowStartHeaders
|
|
27
27
|
} from "./shared.js";
|
|
28
|
+
import { acquireStandaloneClaimBeforeResume } from "./standalone-claim-acquire.js";
|
|
28
29
|
async function prepareRecoveredServicesOrFail(internals, state, callbacks, resolverInfo) {
|
|
29
30
|
return reprovideRecoveredServices(internals, state, callbacks.failWorkflowForUnavailableServices, callbacks.handleCleanupError, callbacks.dispatchEvent, resolverInfo);
|
|
30
31
|
}
|
|
@@ -159,7 +160,7 @@ async function reactivateSuspendedWorkflowState(internals, state) {
|
|
|
159
160
|
const previousState = { ...state };
|
|
160
161
|
state.status = "running";
|
|
161
162
|
state.updatedAt = internals.options.getNow();
|
|
162
|
-
await commitFencedEngineWrite(internals, [
|
|
163
|
+
await commitFencedEngineWrite(internals, state.id, [
|
|
163
164
|
{ type: "put", key: KEYS.workflow(state.id), value: encode(state) },
|
|
164
165
|
...buildWorkflowVisibilityIndexTransition(state.id, previousState, state).batchOps,
|
|
165
166
|
...state.executionDeadline !== void 0 ? buildTimerBatchOperations({
|
|
@@ -177,6 +178,7 @@ export async function resumeWorkflowFromStorage(internals, workflowId, dispatchR
|
|
|
177
178
|
const state = decodeWorkflowState(stateBytes);
|
|
178
179
|
if (state.status !== "running" && state.status !== "suspended")
|
|
179
180
|
throw Error(`Cannot resume workflow "${workflowId}": status is "${state.status}", expected "running" or "suspended"`);
|
|
181
|
+
await acquireStandaloneClaimBeforeResume(internals, workflowId);
|
|
180
182
|
const checkpointBytes = await internals.storage.get(KEYS.checkpoint(workflowId));
|
|
181
183
|
if (!checkpointBytes)
|
|
182
184
|
throw Error(`Checkpoint not found for workflow "${workflowId}"`);
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ADR 0002 § "Two additional transitions": `acquire (standalone resume)`. A
|
|
3
|
+
* workflow found `running` OR `suspended` at recovery has no other enabling
|
|
4
|
+
* write here to fold a claim into, so this commits the acquire fragment
|
|
5
|
+
* ALONE — the one named exception to "never a standalone commit". Attempted
|
|
6
|
+
* BEFORE `prepareResumeState`, `onRecoveredWorkflow`, and generator relaunch,
|
|
7
|
+
* so no user code — and no earlier fenced write this function makes, such as
|
|
8
|
+
* the history-policy circuit breaker or a services-unavailable failure —
|
|
9
|
+
* can run without a held claim.
|
|
10
|
+
*
|
|
11
|
+
* Deliberately covers `running` and `suspended` the same way. The ADR
|
|
12
|
+
* describes `suspended` as folding `acquire` into the status flip itself
|
|
13
|
+
* (`reactivateSuspendedWorkflowState`'s commit in `resume.ts`), but the
|
|
14
|
+
* EARLIER fenced writes above need a held claim regardless of final status,
|
|
15
|
+
* so one early standalone acquire is the pragmatic shape for both — a
|
|
16
|
+
* documented deviation from that row, not an oversight.
|
|
17
|
+
*
|
|
18
|
+
* A no-op when folding does not apply (`ownership !== 'workflow-lease'`, no
|
|
19
|
+
* registry constructed yet) or when this engine already tracks a claim for
|
|
20
|
+
* `workflowId` — a parked signal-driven wake (`inline-parking.ts`) or a bulk
|
|
21
|
+
* retry that folded `acquire` into its own reactivation write before calling
|
|
22
|
+
* `resume` (`bulk-operations.ts`) — PROVIDED `wakeOwnershipCheck` confirms
|
|
23
|
+
* the durable holder still matches the cached generation. A stalled, expired
|
|
24
|
+
* engine keeps `currentEpoch(workflowId) !== null` until its next renewal
|
|
25
|
+
* CAS detects the loss; trusting the cache alone would replay the parked
|
|
26
|
+
* generator against a successor. `acquire()` itself is unsafe as the
|
|
27
|
+
* re-check (it can win the CAS against a still-valid claim), so only the
|
|
28
|
+
* durable holder is re-read (no write) and compared.
|
|
29
|
+
*
|
|
30
|
+
* @module core/engine/lifecycle/standalone-claim-acquire
|
|
31
|
+
*/
|
|
32
|
+
import type { EngineInternals } from '../internals.ts';
|
|
33
|
+
export declare function acquireStandaloneClaimBeforeResume(internals: EngineInternals, workflowId: string): Promise<void>;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { WorkflowClaimUnavailableError } from "../lease-errors.js";
|
|
2
|
+
import { wakeOwnershipCheck } from "../wake-ownership-check.js";
|
|
3
|
+
export async function acquireStandaloneClaimBeforeResume(internals, workflowId) {
|
|
4
|
+
if (internals.options.ownershipMode !== "workflow-lease")
|
|
5
|
+
return;
|
|
6
|
+
const registry = internals.workflowClaimRegistry;
|
|
7
|
+
if (registry === null)
|
|
8
|
+
return;
|
|
9
|
+
const cachedEpoch = registry.currentEpoch(workflowId);
|
|
10
|
+
if (cachedEpoch !== null) {
|
|
11
|
+
const check = await wakeOwnershipCheck({
|
|
12
|
+
storage: internals.storage,
|
|
13
|
+
workflowId,
|
|
14
|
+
wakeKind: "signal",
|
|
15
|
+
expectedEngineId: registry.engineId,
|
|
16
|
+
expectedEpoch: cachedEpoch
|
|
17
|
+
});
|
|
18
|
+
if (check.status === "discarded")
|
|
19
|
+
throw new WorkflowClaimUnavailableError(workflowId, check.observedEngineId);
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
const result = await registry.acquire(workflowId);
|
|
23
|
+
if (result.status === "lost-race")
|
|
24
|
+
throw new WorkflowClaimUnavailableError(workflowId, result.heldBy);
|
|
25
|
+
}
|
|
@@ -4,6 +4,11 @@ import {
|
|
|
4
4
|
commitFencedEngineWrite,
|
|
5
5
|
commitFencedEngineWriteAllowingPreconditionFailure
|
|
6
6
|
} from "../fenced-write.js";
|
|
7
|
+
import {
|
|
8
|
+
commitWithWorkflowClaimFold,
|
|
9
|
+
prepareWorkflowClaimFold,
|
|
10
|
+
throwWorkflowClaimUnavailable
|
|
11
|
+
} from "../workflow-claim-fold.js";
|
|
7
12
|
import { buildStartBatchOperations } from "./start-batch.js";
|
|
8
13
|
|
|
9
14
|
export class StartIdempotencyRaceLostError extends Error {
|
|
@@ -13,13 +18,20 @@ export class StartIdempotencyRaceLostError extends Error {
|
|
|
13
18
|
}
|
|
14
19
|
}
|
|
15
20
|
const WORKFLOW_CONCURRENCY_ADMISSION_MAX_ATTEMPTS = 5;
|
|
16
|
-
async function persistStartBatch(internals, startOperations, conditions) {
|
|
21
|
+
async function persistStartBatch(internals, workflowId, startOperations, conditions, claimFold, isDelayedStart) {
|
|
22
|
+
if (claimFold) {
|
|
23
|
+
const result = await commitWithWorkflowClaimFold(internals, claimFold, startOperations, conditions.map((entry) => entry.condition), "workflow claim acquisition");
|
|
24
|
+
if (result.status === "committed")
|
|
25
|
+
return "committed";
|
|
26
|
+
return result.claimConflict ? "claim-lost" : "precondition-lost";
|
|
27
|
+
}
|
|
28
|
+
const fenceWorkflowId = isDelayedStart ? null : workflowId;
|
|
17
29
|
if (conditions.length === 0) {
|
|
18
|
-
await commitFencedEngineWrite(internals, startOperations, [], () => Error("Workflow start lost its CAS race."));
|
|
19
|
-
return
|
|
30
|
+
await commitFencedEngineWrite(internals, fenceWorkflowId, startOperations, [], () => Error("Workflow start lost its CAS race."));
|
|
31
|
+
return "committed";
|
|
20
32
|
}
|
|
21
33
|
requireStorageCapability(internals.storage, "conditionalBatch", "start preconditions");
|
|
22
|
-
return commitFencedEngineWriteAllowingPreconditionFailure(internals, startOperations, conditions.map((entry) => entry.condition));
|
|
34
|
+
return await commitFencedEngineWriteAllowingPreconditionFailure(internals, fenceWorkflowId, startOperations, conditions.map((entry) => entry.condition)) ? "committed" : "precondition-lost";
|
|
23
35
|
}
|
|
24
36
|
async function hasStartPreconditionConflict(internals, conditions) {
|
|
25
37
|
for (const entry of conditions) {
|
|
@@ -57,11 +69,13 @@ export async function buildAndCommitStartBatch(context, buildIdempotentStartOper
|
|
|
57
69
|
const startOperations = buildStartBatchOperations(internals, workflowId, state, checkpoint, registration, options, state.executionDeadline, context.delayedStartTimer, context.persistedWorkflowStartHeaders, mergeAdditionalStartOperations(context.additionalStartOperations, mergeAdditionalStartOperations(idempotent?.operations, workflowConcurrency?.operations)), context.callbacks, context.purgeDeleteOperations), conditions = [
|
|
58
70
|
...tagStartPreconditions(idempotent?.conditions),
|
|
59
71
|
...tagWorkflowConcurrencyConditions(workflowConcurrency?.conditions ?? [])
|
|
60
|
-
];
|
|
61
|
-
if (
|
|
72
|
+
], isDelayedStart = context.delayedStartTimer !== void 0, claimFold = isDelayedStart ? void 0 : await prepareWorkflowClaimFold(internals, workflowId), outcome = await persistStartBatch(internals, workflowId, startOperations, conditions, claimFold, isDelayedStart);
|
|
73
|
+
if (outcome === "committed")
|
|
62
74
|
return;
|
|
63
75
|
if (await hasStartPreconditionConflict(internals, conditions))
|
|
64
76
|
throw new StartIdempotencyRaceLostError;
|
|
77
|
+
if (outcome === "claim-lost")
|
|
78
|
+
return throwWorkflowClaimUnavailable(internals, workflowId);
|
|
65
79
|
if (workflowConcurrency === void 0)
|
|
66
80
|
throw new StartIdempotencyRaceLostError;
|
|
67
81
|
}
|
|
@@ -3,6 +3,20 @@ import { type WorkflowHandle } from '../handles.ts';
|
|
|
3
3
|
import type { EngineInternals } from '../internals.ts';
|
|
4
4
|
import { type LifecycleCallbacks, type RecoverAllOptions, type RegistrationEntry } from './shared.ts';
|
|
5
5
|
export declare function recoverAll(internals: EngineInternals, callbacks: LifecycleCallbacks, options?: RecoverAllOptions): Promise<WorkflowHandle[]>;
|
|
6
|
-
|
|
6
|
+
/** Options for {@link resume}. */
|
|
7
|
+
export type ResumeOptions = {
|
|
8
|
+
/**
|
|
9
|
+
* Skip the local-ownership fast path and always replay from durable storage.
|
|
10
|
+
*
|
|
11
|
+
* Required by ADR 0002 reclaim-driven resume: deposition drops only the
|
|
12
|
+
* registry's claim entry, so local checkpoints, parked markers, contexts and
|
|
13
|
+
* generators survive. Without this, a reclaim by the same engine returns the
|
|
14
|
+
* pre-deposition handle without reaching `resumeWorkflowFromStorage()` and
|
|
15
|
+
* renews a run that never restarted from durable state. Ordinary
|
|
16
|
+
* `engine.resume()` leaves it `false` — nothing was deposed there.
|
|
17
|
+
*/
|
|
18
|
+
readonly forceReplayFromStorage?: boolean;
|
|
19
|
+
};
|
|
20
|
+
export declare function resume(internals: EngineInternals, workflowId: string, callbacks: LifecycleCallbacks, onRecoveredWorkflow?: RecoverAllOptions['onRecoveredWorkflow'], options?: ResumeOptions): Promise<WorkflowHandle>;
|
|
7
21
|
export declare function fork(internals: EngineInternals, sourceWorkflowId: string, options: ForkOptions | undefined, callbacks: LifecycleCallbacks): Promise<WorkflowHandle>;
|
|
8
22
|
export declare function launchWorkflowFromCheckpoint(internals: EngineInternals, workflowId: string, state: WorkflowState, checkpoint: Checkpoint, registration: RegistrationEntry, callbacks: LifecycleCallbacks): WorkflowHandle;
|
|
@@ -11,6 +11,7 @@ import { hydrateCheckpointReplayState } from "../checkpoint-replay.js";
|
|
|
11
11
|
import { WorkflowTypeNotRegisteredForRecoveryError } from "../errors.js";
|
|
12
12
|
import { commitFencedEngineWrite } from "../fenced-write.js";
|
|
13
13
|
import { getWorkflowExecutionStartedAt } from "../handles.js";
|
|
14
|
+
import { WorkflowClaimUnavailableError } from "../lease-errors.js";
|
|
14
15
|
import { normalizeForkStep, selectPersistedWorkflowStartHeaders } from "../state-utilities.js";
|
|
15
16
|
import { loadWorkflowState } from "../storage-io.js";
|
|
16
17
|
import { getComposedWorkflowInterceptor } from "../strategy-helpers.js";
|
|
@@ -73,6 +74,8 @@ async function recoverEntryOrIsolateFailure(internals, workflowId, callbacks, op
|
|
|
73
74
|
await callbacks.failWorkflowForVersionMismatch(workflowId, error);
|
|
74
75
|
return null;
|
|
75
76
|
}
|
|
77
|
+
if (error instanceof WorkflowClaimUnavailableError)
|
|
78
|
+
return null;
|
|
76
79
|
throw error;
|
|
77
80
|
}
|
|
78
81
|
}
|
|
@@ -98,9 +101,9 @@ export async function recoverAll(internals, callbacks, options) {
|
|
|
98
101
|
}
|
|
99
102
|
return handles;
|
|
100
103
|
}
|
|
101
|
-
export async function resume(internals, workflowId, callbacks, onRecoveredWorkflow) {
|
|
104
|
+
export async function resume(internals, workflowId, callbacks, onRecoveredWorkflow, options) {
|
|
102
105
|
const workflowState = await loadWorkflowState(internals, workflowId);
|
|
103
|
-
if (workflowState !== null) {
|
|
106
|
+
if (workflowState !== null && options?.forceReplayFromStorage !== !0) {
|
|
104
107
|
if (callbacks.isInlineWorkflowLocallyOwned(workflowId, workflowState.status) || callbacks.hasLocalCheckpointOwnership(workflowId, workflowState.status)) {
|
|
105
108
|
await enforceHistoryPolicyBeforeReplayById(internals, workflowId, callbacks);
|
|
106
109
|
return callbacks.getHandle(workflowId);
|
|
@@ -130,7 +133,7 @@ export async function fork(internals, sourceWorkflowId, options, callbacks) {
|
|
|
130
133
|
let forkStarted = !1;
|
|
131
134
|
try {
|
|
132
135
|
const forkCheckpointBytes = serializeCheckpoint(forkCheckpoint);
|
|
133
|
-
await commitFencedEngineWrite(internals, buildForkBatchOperations(internals, workflowId, forkState, forkCheckpoint, forkCheckpointBytes, persistedWorkflowStartHeaders, callbacks), [], () => Error(`Fork of workflow "${workflowId}" lost its CAS race.`));
|
|
136
|
+
await commitFencedEngineWrite(internals, workflowId, buildForkBatchOperations(internals, workflowId, forkState, forkCheckpoint, forkCheckpointBytes, persistedWorkflowStartHeaders, callbacks), [], () => Error(`Fork of workflow "${workflowId}" lost its CAS race.`));
|
|
134
137
|
internals.eventLogHeads.set(workflowId, EMPTY_EVENT_HEAD);
|
|
135
138
|
setWorkflowStartHeaders(internals, workflowId, persistedWorkflowStartHeaders, callbacks);
|
|
136
139
|
const handle = launchWorkflowFromCheckpoint(internals, workflowId, forkState, forkCheckpoint, registration, callbacks);
|