@lostgradient/weft 0.20.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/core/engine/activity-reconciliation.d.ts +2 -2
- package/dist/core/engine/activity-reconciliation.js +12 -8
- package/dist/core/engine/async-activity-completion.js +12 -4
- package/dist/core/engine/async-activity-records.d.ts +11 -1
- package/dist/core/engine/async-activity-records.js +4 -3
- package/dist/core/engine/attributes-tags.d.ts +11 -1
- package/dist/core/engine/attributes-tags.js +4 -3
- package/dist/core/engine/bulk-operations-purge.js +5 -2
- package/dist/core/engine/bulk-operations.js +17 -5
- package/dist/core/engine/callback-creators-bundles.js +1 -1
- package/dist/core/engine/callback-creators-core.js +4 -2
- package/dist/core/engine/checkpoint-io.js +1 -1
- package/dist/core/engine/child-workflow.js +2 -1
- package/dist/core/engine/completed-review-storage.js +1 -1
- package/dist/core/engine/condition-waiters.d.ts +45 -3
- package/dist/core/engine/condition-waiters.js +12 -3
- package/dist/core/engine/disposal.js +6 -0
- package/dist/core/engine/engine-internal-types.d.ts +12 -4
- package/dist/core/engine/fenced-write.d.ts +46 -15
- package/dist/core/engine/fenced-write.js +39 -6
- package/dist/core/engine/finalizer-state.d.ts +3 -1
- package/dist/core/engine/handle-result.d.ts +49 -1
- package/dist/core/engine/handle-result.js +95 -20
- package/dist/core/engine/index.d.ts +13 -5
- package/dist/core/engine/index.js +155 -5
- package/dist/core/engine/inline-launch-queue.js +9 -0
- package/dist/core/engine/internals.d.ts +73 -0
- package/dist/core/engine/invoke-update-handler.d.ts +19 -0
- package/dist/core/engine/invoke-update-handler.js +7 -0
- package/dist/core/engine/lease-deposition.d.ts +132 -0
- package/dist/core/engine/lease-deposition.js +31 -0
- package/dist/core/engine/lease-errors.d.ts +80 -9
- package/dist/core/engine/lease-errors.js +28 -2
- package/dist/core/engine/lifecycle/resume.js +3 -1
- package/dist/core/engine/lifecycle/standalone-claim-acquire.d.ts +33 -0
- package/dist/core/engine/lifecycle/standalone-claim-acquire.js +25 -0
- package/dist/core/engine/lifecycle/start-commit.js +20 -6
- package/dist/core/engine/lifecycle/transition.d.ts +15 -1
- package/dist/core/engine/lifecycle/transition.js +6 -3
- package/dist/core/engine/operations-activity.js +1 -1
- package/dist/core/engine/operations-time.js +21 -7
- package/dist/core/engine/owner-side-signal-poll.d.ts +132 -0
- package/dist/core/engine/owner-side-signal-poll.js +22 -0
- package/dist/core/engine/owner-side-update-poll.d.ts +116 -0
- package/dist/core/engine/owner-side-update-poll.js +22 -0
- package/dist/core/engine/ownership-bootstrap.d.ts +280 -0
- package/dist/core/engine/ownership-bootstrap.js +89 -0
- package/dist/core/engine/ownership-mode-marker.d.ts +82 -0
- package/dist/core/engine/ownership-mode-marker.js +52 -0
- package/dist/core/engine/ownership-options.d.ts +22 -6
- package/dist/core/engine/ownership-options.js +42 -12
- package/dist/core/engine/pending-updates.js +35 -24
- package/dist/core/engine/queries.d.ts +70 -0
- package/dist/core/engine/queries.js +58 -4
- package/dist/core/engine/schedules.js +1 -1
- package/dist/core/engine/signals.d.ts +11 -1
- package/dist/core/engine/signals.js +11 -7
- package/dist/core/engine/sleep-timer-acknowledgements.d.ts +68 -0
- package/dist/core/engine/sleep-timer-acknowledgements.js +19 -0
- package/dist/core/engine/storage-io.d.ts +58 -10
- package/dist/core/engine/storage-io.js +27 -9
- package/dist/core/engine/termination/cleanup.d.ts +5 -4
- package/dist/core/engine/termination/complete.js +23 -4
- package/dist/core/engine/termination/finalizer-claim.js +4 -4
- package/dist/core/engine/termination/state-commit-callbacks.d.ts +15 -0
- package/dist/core/engine/termination/state-commit-callbacks.js +0 -0
- package/dist/core/engine/termination/suspend.js +1 -1
- package/dist/core/engine/update-validation.d.ts +28 -0
- package/dist/core/engine/update-validation.js +37 -0
- package/dist/core/engine/updates.d.ts +17 -16
- package/dist/core/engine/updates.js +24 -47
- package/dist/core/engine/wake-ownership-check.d.ts +86 -0
- package/dist/core/engine/wake-ownership-check.js +19 -0
- package/dist/core/engine/wake-ownership-guard.d.ts +46 -0
- package/dist/core/engine/wake-ownership-guard.js +23 -0
- package/dist/core/engine/workflow-claim-codec.d.ts +64 -0
- package/dist/core/engine/workflow-claim-codec.js +44 -0
- package/dist/core/engine/workflow-claim-cooldown.d.ts +60 -0
- package/dist/core/engine/workflow-claim-cooldown.js +17 -0
- package/dist/core/engine/workflow-claim-fold.d.ts +82 -0
- package/dist/core/engine/workflow-claim-fold.js +53 -0
- package/dist/core/engine/workflow-claim-metrics.d.ts +131 -0
- package/dist/core/engine/workflow-claim-metrics.js +62 -0
- package/dist/core/engine/workflow-claim-reclaim-scan.d.ts +96 -0
- package/dist/core/engine/workflow-claim-reclaim-scan.js +66 -0
- package/dist/core/engine/workflow-claim-reclaim-target.d.ts +107 -0
- package/dist/core/engine/workflow-claim-reclaim-target.js +155 -0
- package/dist/core/engine/workflow-claim-registry.d.ts +227 -0
- package/dist/core/engine/workflow-claim-registry.js +198 -0
- package/dist/core/engine/workflow-claim-renewal-interval.d.ts +50 -0
- package/dist/core/engine/workflow-claim-renewal-interval.js +132 -0
- package/dist/core/engine/workflow-claim-renewal-subpasses.d.ts +265 -0
- package/dist/core/engine/workflow-claim-renewal-subpasses.js +104 -0
- package/dist/core/engine/workflow-claim-renewal-task.d.ts +159 -0
- package/dist/core/engine/workflow-claim-renewal-task.js +65 -0
- package/dist/core/engine/workflow-claim-transitions.d.ts +186 -0
- package/dist/core/engine/workflow-claim-transitions.js +120 -0
- package/dist/core/inline-execution-strategy.d.ts +20 -0
- package/dist/core/inline-execution-strategy.js +15 -4
- package/dist/core/scheduler/scheduler-class.js +3 -3
- package/dist/core/scheduler/timer-sources.d.ts +13 -0
- package/dist/core/scheduler/timer-sources.js +1 -1
- package/dist/core/types/options.d.ts +17 -1
- package/dist/core/weft-error.d.ts +1 -1
- package/dist/core/weft-error.js +2 -1
- package/dist/http.js +2 -2
- package/dist/index.d.ts +2 -1
- package/dist/index.js +8 -0
- package/dist/indexeddb.js +1 -1
- package/dist/json-schema.js +3 -3
- package/dist/observability/index.js +2 -2
- package/dist/storage/auto.js +1 -1
- package/dist/storage/bun-sql.js +113 -0
- package/dist/storage/compressed-storage.js +1 -1
- package/dist/storage/index.d.ts +38 -35
- package/dist/storage/interface.d.ts +87 -125
- package/dist/storage/interface.js +1 -1
- package/dist/storage/key-encoding.d.ts +51 -0
- package/dist/storage/key-encoding.js +13 -0
- package/dist/storage/key-prefixes.d.ts +1 -1
- package/dist/storage/key-prefixes.js +3 -0
- package/dist/storage/lmdb.js +1 -1
- package/dist/storage/memory.js +1 -1
- package/dist/storage/neon.js +2 -2
- package/dist/storage/node-sqlite.js +113 -0
- package/dist/storage/ownership-keys.d.ts +55 -0
- package/dist/storage/ownership-keys.js +6 -0
- package/dist/storage/postgres.js +2 -2
- package/dist/storage/resolve.js +1 -1
- package/dist/storage/scoped-storage.js +1 -1
- package/dist/storage/testing.js +1 -1
- package/dist/storage/turso.js +2 -2
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/web-extension.js +1 -1
- package/dist/worker/protocol.js +1 -1
- package/package.json +1 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { KEYS, storageConditionalBatch } from "../../storage/interface.js";
|
|
2
2
|
import { decodeEpoch } from "./lease-codec.js";
|
|
3
|
-
import { handleDeposition } from "./lease-deposition.js";
|
|
3
|
+
import { emitWorkflowClaimLostWarning, handleDeposition } from "./lease-deposition.js";
|
|
4
4
|
import { EngineDeposedError, EngineLeaseNotHeldError } from "./lease-errors.js";
|
|
5
5
|
export function assertLeaseHeldForEngineWork(internals) {
|
|
6
6
|
if (internals.options.ownershipMode !== "lease")
|
|
@@ -8,7 +8,9 @@ export function assertLeaseHeldForEngineWork(internals) {
|
|
|
8
8
|
if (!(!internals.deposed && internals.leaseManager !== null && internals.leaseManager.currentEpochBytes() !== null))
|
|
9
9
|
throw new EngineLeaseNotHeldError;
|
|
10
10
|
}
|
|
11
|
-
async function fencedCommit(internals, operations, baseConditions) {
|
|
11
|
+
async function fencedCommit(internals, workflowId, operations, baseConditions) {
|
|
12
|
+
if (internals.options.ownershipMode === "workflow-lease" && workflowId !== null)
|
|
13
|
+
return fencedCommitForWorkflow(internals, workflowId, operations, baseConditions);
|
|
12
14
|
const epochBytes = resolveFenceEpochOrHalt(internals);
|
|
13
15
|
if (epochBytes === null) {
|
|
14
16
|
if (baseConditions.length === 0) {
|
|
@@ -41,12 +43,30 @@ function resolveFenceEpochOrHalt(internals) {
|
|
|
41
43
|
}
|
|
42
44
|
return epochBytes;
|
|
43
45
|
}
|
|
44
|
-
|
|
45
|
-
|
|
46
|
+
async function fencedCommitForWorkflow(internals, workflowId, operations, baseConditions) {
|
|
47
|
+
const epochBytes = internals.workflowClaimRegistry?.currentEpochBytes(workflowId) ?? null;
|
|
48
|
+
if (epochBytes === null)
|
|
49
|
+
haltWorkflowClaim(workflowId);
|
|
50
|
+
const conditions = [
|
|
51
|
+
...baseConditions,
|
|
52
|
+
{ key: KEYS.workflowOwnerEpoch(workflowId), expectedValue: epochBytes }
|
|
53
|
+
];
|
|
54
|
+
if (await storageConditionalBatch(internals.storage, conditions, operations))
|
|
55
|
+
return "committed";
|
|
56
|
+
if (await isWorkflowDeposed(internals, workflowId, epochBytes))
|
|
57
|
+
haltWorkflowClaim(workflowId);
|
|
58
|
+
return "lost-race";
|
|
59
|
+
}
|
|
60
|
+
function haltWorkflowClaim(workflowId) {
|
|
61
|
+
emitWorkflowClaimLostWarning(workflowId);
|
|
62
|
+
throw new EngineDeposedError(workflowId);
|
|
63
|
+
}
|
|
64
|
+
export async function commitFencedEngineWrite(internals, workflowId, operations, baseConditions, onLostRace) {
|
|
65
|
+
if (await fencedCommit(internals, workflowId, operations, baseConditions) === "lost-race")
|
|
46
66
|
throw onLostRace();
|
|
47
67
|
}
|
|
48
|
-
export async function commitFencedEngineWriteAllowingPreconditionFailure(internals, operations, baseConditions) {
|
|
49
|
-
return await fencedCommit(internals, operations, baseConditions) === "committed";
|
|
68
|
+
export async function commitFencedEngineWriteAllowingPreconditionFailure(internals, workflowId, operations, baseConditions) {
|
|
69
|
+
return await fencedCommit(internals, workflowId, operations, baseConditions) === "committed";
|
|
50
70
|
}
|
|
51
71
|
function epochBytesEqual(a, b) {
|
|
52
72
|
if (b === null || a.byteLength !== b.byteLength)
|
|
@@ -69,3 +89,16 @@ async function isDeposed(internals, heldEpochBytes) {
|
|
|
69
89
|
return !0;
|
|
70
90
|
return !epochBytesEqual(heldEpochBytes, currentEpochRaw);
|
|
71
91
|
}
|
|
92
|
+
async function isWorkflowDeposed(internals, workflowId, heldEpochBytes) {
|
|
93
|
+
let currentEpochRaw;
|
|
94
|
+
try {
|
|
95
|
+
currentEpochRaw = await internals.storage.get(KEYS.workflowOwnerEpoch(workflowId));
|
|
96
|
+
} catch {
|
|
97
|
+
return !0;
|
|
98
|
+
}
|
|
99
|
+
if (currentEpochRaw === null)
|
|
100
|
+
return !0;
|
|
101
|
+
if (decodeEpoch(currentEpochRaw) === null)
|
|
102
|
+
return !0;
|
|
103
|
+
return !epochBytesEqual(heldEpochBytes, currentEpochRaw);
|
|
104
|
+
}
|
|
@@ -16,7 +16,9 @@ import type { EngineInternals } from './internals.ts';
|
|
|
16
16
|
* the next checkpoint still flushes the staged op in the terminal batch, so the
|
|
17
17
|
* finalizer always sees the resource id. The staged op inherits the lease-epoch
|
|
18
18
|
* fence (#470) for free, since checkpoint and terminal commits route through
|
|
19
|
-
* `
|
|
19
|
+
* `commitSelfWorkflowStateOperations` (completion) or
|
|
20
|
+
* `commitExternalTerminalWorkflowStateOperations` (cancel/timeout/suspend —
|
|
21
|
+
* ADR 0002), both in `storage-io.ts`.
|
|
20
22
|
*
|
|
21
23
|
* Oversized payloads are rejected before staging (the same hostile-input guard
|
|
22
24
|
* activity results use). A call made once the workflow is already terminalizing
|
|
@@ -4,5 +4,53 @@ import type { EngineInternals } from './internals.ts';
|
|
|
4
4
|
export declare function createWorkflowHandleWithResultPromise(internals: EngineInternals, workflowId: string): WorkflowHandle;
|
|
5
5
|
export declare function createWorkflowResultWaiter(internals: EngineInternals, workflowId: string): WorkflowResultWaiter;
|
|
6
6
|
export declare function getWorkflowResultPromise(internals: EngineInternals, workflowId: string): Promise<unknown>;
|
|
7
|
-
|
|
7
|
+
/**
|
|
8
|
+
* Result promise for a parent generator parked on `ctx.startChild()`'s
|
|
9
|
+
* default `parentClosePolicy: 'await'` — the ONLY intended caller (see
|
|
10
|
+
* `child-workflow.ts`'s `executeChildWorkflow`, which must call this instead
|
|
11
|
+
* of the plain `childHandle.result()` / `handle.result()` path). Identical to
|
|
12
|
+
* {@link getWorkflowResultPromise} except the waiter this settles is marked
|
|
13
|
+
* generator-owned for `parentWorkflowId`: under `ownership: 'workflow-lease'`
|
|
14
|
+
* a settle attempt first confirms `parentWorkflowId` still holds the claim
|
|
15
|
+
* generation it parked under (`confirmWakeOwnership`, `'child-completion'`),
|
|
16
|
+
* fencing duplicate generator advancement (WFT-79 F1) — see
|
|
17
|
+
* {@link bootstrapWorkflowResultResolver}. Top-level, non-generator callers
|
|
18
|
+
* must keep using {@link getWorkflowResultPromise}, which stays unfenced by
|
|
19
|
+
* design: cross-engine `handle.result()` polling has no generator to
|
|
20
|
+
* duplicate and must remain settleable from durable state alone.
|
|
21
|
+
*
|
|
22
|
+
* If `internals.resultResolvers` already holds a waiter for `workflowId` —
|
|
23
|
+
* e.g. an observational `handle.result()` caller got there first — that
|
|
24
|
+
* SHARED waiter is untouched: {@link fenceResultOnParentGeneration} wraps
|
|
25
|
+
* only THIS caller's derived promise, so the observational caller's own
|
|
26
|
+
* promise settles normally from the shared waiter regardless of the parent's
|
|
27
|
+
* claim generation. See that function's doc for why fencing lives on the
|
|
28
|
+
* parent's view instead of the shared waiter itself.
|
|
29
|
+
*/
|
|
30
|
+
export declare function getGeneratorOwnedWorkflowResultPromise(internals: EngineInternals, workflowId: string, parentWorkflowId: string): Promise<unknown>;
|
|
31
|
+
/**
|
|
32
|
+
* Settle result waiters for workflows this engine does not own, whose terminal
|
|
33
|
+
* transition lands on another engine and so never touches this engine's
|
|
34
|
+
* in-memory resolver map.
|
|
35
|
+
*
|
|
36
|
+
* Exported for `backgroundTasks: 'manual'`, where no timer runs and the host
|
|
37
|
+
* drives every background step through an awaited `runMaintenance()`.
|
|
38
|
+
*/
|
|
39
|
+
export declare function pollPendingCrossEngineResultWaiters(internals: EngineInternals): Promise<void>;
|
|
40
|
+
/**
|
|
41
|
+
* Outcome of one {@link bootstrapWorkflowResultResolver} attempt, used by
|
|
42
|
+
* this file's callers to decide whether to keep polling:
|
|
43
|
+
*
|
|
44
|
+
* - `'settled'`: the waiter was resolved or rejected.
|
|
45
|
+
* - `'pending'`: no terminal result yet (still running, or a transient read
|
|
46
|
+
* failure under a guaranteed-retry ownership mode). Keep polling.
|
|
47
|
+
*
|
|
48
|
+
* A generator-owned caller's discarded parent generation needs no outcome
|
|
49
|
+
* here at all: `fenceResultOnParentGeneration` withholds settlement on that
|
|
50
|
+
* caller's own derived promise, independent of whether the SHARED waiter
|
|
51
|
+
* this function settles ever resolves.
|
|
52
|
+
*/
|
|
53
|
+
type ResultResolutionOutcome = 'settled' | 'pending';
|
|
54
|
+
export declare function bootstrapWorkflowResultResolver(internals: EngineInternals, workflowId: string, waiter: WorkflowResultWaiter): Promise<ResultResolutionOutcome>;
|
|
8
55
|
export declare function cacheHandle(internals: EngineInternals, workflowId: string, handle: WorkflowHandle): void;
|
|
56
|
+
export {};
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { EngineDisposedError } from "./errors.js";
|
|
2
2
|
import { WorkflowHandle } from "./handles.js";
|
|
3
|
-
import {
|
|
3
|
+
import { deriveWorkflowResultFromState, loadWorkflowState } from "./storage-io.js";
|
|
4
|
+
import { confirmWakeOwnership } from "./wake-ownership-guard.js";
|
|
4
5
|
export function createWorkflowHandleWithResultPromise(internals, workflowId) {
|
|
5
6
|
const handle = new WorkflowHandle(workflowId, internals.engine);
|
|
6
7
|
cacheHandle(internals, workflowId, handle);
|
|
@@ -22,32 +23,106 @@ export function getWorkflowResultPromise(internals, workflowId) {
|
|
|
22
23
|
return rejected;
|
|
23
24
|
}
|
|
24
25
|
const waiter = createWorkflowResultWaiter(internals, workflowId);
|
|
25
|
-
bootstrapWorkflowResultResolver(internals, workflowId, waiter);
|
|
26
|
+
bootstrapWorkflowResultResolver(internals, workflowId, waiter).then(() => scheduleCrossEngineResultPollIfPending(internals, workflowId, waiter));
|
|
26
27
|
return waiter.promise;
|
|
27
28
|
}
|
|
28
|
-
export
|
|
29
|
+
export function getGeneratorOwnedWorkflowResultPromise(internals, workflowId, parentWorkflowId) {
|
|
30
|
+
return fenceResultOnParentGeneration(internals, parentWorkflowId, getWorkflowResultPromise(internals, workflowId));
|
|
31
|
+
}
|
|
32
|
+
function fenceResultOnParentGeneration(internals, parentWorkflowId, promise) {
|
|
33
|
+
const gate = Promise.withResolvers();
|
|
34
|
+
promise.then(async (value) => {
|
|
35
|
+
if (await parentStillOwnsGeneration(internals, parentWorkflowId))
|
|
36
|
+
gate.resolve(value);
|
|
37
|
+
}, async (error) => {
|
|
38
|
+
if (await parentStillOwnsGeneration(internals, parentWorkflowId))
|
|
39
|
+
gate.reject(error);
|
|
40
|
+
});
|
|
41
|
+
return gate.promise;
|
|
42
|
+
}
|
|
43
|
+
async function parentStillOwnsGeneration(internals, parentWorkflowId) {
|
|
29
44
|
try {
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
45
|
+
return await confirmWakeOwnership(internals, parentWorkflowId, "child-completion") === "proceed";
|
|
46
|
+
} catch {
|
|
47
|
+
return !0;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
function scheduleCrossEngineResultPollIfPending(internals, workflowId, waiter) {
|
|
51
|
+
if (internals.workflowClaimRegistry === null)
|
|
52
|
+
return;
|
|
53
|
+
if (internals.resultResolvers.get(workflowId) !== waiter)
|
|
54
|
+
return;
|
|
55
|
+
if (internals.options.backgroundTaskMode !== "automatic")
|
|
56
|
+
return;
|
|
57
|
+
const handle = setTimeout(() => {
|
|
58
|
+
internals.pendingResultPollTimers.delete(handle);
|
|
59
|
+
bootstrapWorkflowResultResolver(internals, workflowId, waiter).then(() => scheduleCrossEngineResultPollIfPending(internals, workflowId, waiter));
|
|
60
|
+
}, internals.options.workflowClaimRenewIntervalMs);
|
|
61
|
+
internals.pendingResultPollTimers.add(handle);
|
|
62
|
+
handle.unref?.();
|
|
63
|
+
}
|
|
64
|
+
export async function pollPendingCrossEngineResultWaiters(internals) {
|
|
65
|
+
if (internals.workflowClaimRegistry === null)
|
|
66
|
+
return;
|
|
67
|
+
const pending = Array.from(internals.resultResolvers.entries());
|
|
68
|
+
for (const [workflowId, waiter] of pending) {
|
|
69
|
+
if (await deferToLocalTerminalDeliveryIfPending(internals, workflowId, waiter))
|
|
70
|
+
continue;
|
|
40
71
|
try {
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
72
|
+
await bootstrapWorkflowResultResolver(internals, workflowId, waiter);
|
|
73
|
+
} catch {}
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
function settleOrRetryOnTransientReadFailure(internals, workflowId, waiter, error) {
|
|
77
|
+
if (internals.workflowClaimRegistry === null) {
|
|
78
|
+
clearResultWaiter(internals, workflowId, waiter);
|
|
79
|
+
waiter.reject(error);
|
|
80
|
+
return "settled";
|
|
81
|
+
}
|
|
82
|
+
return "pending";
|
|
83
|
+
}
|
|
84
|
+
async function deferToLocalTerminalDeliveryIfPending(internals, workflowId, waiter) {
|
|
85
|
+
if (internals.workflowClaimRegistry?.currentEpoch(workflowId) == null)
|
|
86
|
+
return !1;
|
|
87
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
88
|
+
return internals.resultResolvers.get(workflowId) !== waiter;
|
|
89
|
+
}
|
|
90
|
+
async function prepareToSettleTerminalWaiter(internals, workflowId, waiter) {
|
|
91
|
+
if (await deferToLocalTerminalDeliveryIfPending(internals, workflowId, waiter))
|
|
92
|
+
return "settled";
|
|
93
|
+
return "proceed";
|
|
94
|
+
}
|
|
95
|
+
export async function bootstrapWorkflowResultResolver(internals, workflowId, waiter) {
|
|
96
|
+
let state;
|
|
97
|
+
try {
|
|
98
|
+
state = await loadWorkflowState(internals, workflowId);
|
|
99
|
+
} catch (error) {
|
|
100
|
+
return settleOrRetryOnTransientReadFailure(internals, workflowId, waiter, error);
|
|
101
|
+
}
|
|
102
|
+
if (linkToReplacementWaiter(internals, workflowId, waiter))
|
|
103
|
+
return "settled";
|
|
104
|
+
if (!state) {
|
|
105
|
+
const outcome = await prepareToSettleTerminalWaiter(internals, workflowId, waiter);
|
|
106
|
+
if (outcome !== "proceed")
|
|
107
|
+
return outcome;
|
|
108
|
+
internals.resultResolvers.delete(workflowId);
|
|
109
|
+
waiter.reject(Error(`Workflow "${workflowId}" not found in storage`));
|
|
110
|
+
return "settled";
|
|
111
|
+
}
|
|
112
|
+
if (state.status === "running" || state.status === "pending" || state.status === "suspended")
|
|
113
|
+
return "pending";
|
|
114
|
+
const outcome = await prepareToSettleTerminalWaiter(internals, workflowId, waiter);
|
|
115
|
+
if (outcome !== "proceed")
|
|
116
|
+
return outcome;
|
|
117
|
+
try {
|
|
118
|
+
const result = deriveWorkflowResultFromState(state);
|
|
119
|
+
clearResultWaiter(internals, workflowId, waiter);
|
|
120
|
+
waiter.resolve(result);
|
|
121
|
+
return "settled";
|
|
48
122
|
} catch (error) {
|
|
49
123
|
clearResultWaiter(internals, workflowId, waiter);
|
|
50
124
|
waiter.reject(error);
|
|
125
|
+
return "settled";
|
|
51
126
|
}
|
|
52
127
|
}
|
|
53
128
|
function linkToReplacementWaiter(internals, workflowId, waiter) {
|
|
@@ -27,7 +27,9 @@ export type { PendingAsyncActivity } from './async-activity-records.ts';
|
|
|
27
27
|
export type { PendingTimelineEntry, RegistrationEntry, ResolvedOptions, TrackedWaiterKeys, WorkflowResultWaiter, } from './engine-internal-types.ts';
|
|
28
28
|
export { ActivityResolutionError, BulkDeleteRequiresTerminalWorkflowsError, BulkOperationConfirmationError, EngineCreateNameMismatchError, EngineDisposalError, EngineDisposedError, IdempotencyKeyPurgedError, PersistedDataIncompatibleError, StartOrSignalConflictError, WorkflowAlreadyExistsError, WorkflowConcurrencyLimitExceededError, WorkflowNotFoundError, WorkflowNotRegisteredError, WorkflowSuspendNotSupportedError, WorkflowTeardownPendingError, WorkflowTypeNotRegisteredForRecoveryError, } from './errors.ts';
|
|
29
29
|
export { HANDLE_RESULT_PROMISE, WorkflowHandle } from './handles.ts';
|
|
30
|
-
export {
|
|
30
|
+
export { WeftWorkflowClaimLostWarning, WeftWorkflowWakeDiscardedWarning, WORKFLOW_CLAIM_LOST_WARNING_NAME, WORKFLOW_WAKE_DISCARDED_WARNING_NAME, } from './lease-deposition.ts';
|
|
31
|
+
export type { WorkflowWakeKind } from './lease-deposition.ts';
|
|
32
|
+
export { EngineLeaseAcquisitionTimeoutError, EngineLeaseCorruptedError, EngineLeaseNotHeldError, OwnershipModeMismatchError, WorkflowClaimUnavailableError, } from './lease-errors.ts';
|
|
31
33
|
export type { EngineLeaseHealth, LeaseLostReason } from './lease-health.ts';
|
|
32
34
|
export type { RecoverAllOptions, RecoveredWorkflowInfo } from './lifecycle.ts';
|
|
33
35
|
export { ScheduleHandle } from './schedule-handle.ts';
|
|
@@ -272,13 +274,19 @@ export declare class Engine<TWorkflows extends object = DefaultWorkflowRegistry,
|
|
|
272
274
|
aggregate(filter: ListFilter | undefined, options: AggregateOptions): Promise<AggregateResult>;
|
|
273
275
|
getRetentionOverview(): RetentionOverview;
|
|
274
276
|
/**
|
|
275
|
-
* Run one host-driven maintenance cycle. This
|
|
276
|
-
*
|
|
277
|
-
*
|
|
277
|
+
* Run one host-driven maintenance cycle. This runs the `ownership:
|
|
278
|
+
* 'workflow-lease'` construction gates (idempotent — a no-op once already
|
|
279
|
+
* bootstrapped) and one per-workflow claim-renewal pass, fires due durable
|
|
280
|
+
* timers, deletes expired update responses, applies configured retention,
|
|
281
|
+
* and re-evaluates alert rules without relying on process-local intervals.
|
|
278
282
|
*
|
|
279
283
|
* Use with `backgroundTasks: 'manual'` from a serverless alarm, Cron trigger,
|
|
280
284
|
* or another externally scheduled wake-up. Concurrent calls are safe, but a
|
|
281
|
-
* host should await each cycle before scheduling another.
|
|
285
|
+
* host should await each cycle before scheduling another. Under `ownership:
|
|
286
|
+
* 'workflow-lease'`, this is also the ONLY thing that renews this engine's
|
|
287
|
+
* per-workflow claims in manual mode — no interval is ever started for
|
|
288
|
+
* `backgroundTasks: 'manual'`, so a host that calls this less often than
|
|
289
|
+
* `workflowClaimRenewInterval` will see its own live claims lapse.
|
|
282
290
|
*/
|
|
283
291
|
runMaintenance(now?: number): Promise<void>;
|
|
284
292
|
purge(filter?: ListFilter): Promise<PurgeResult>;
|
|
@@ -37,6 +37,7 @@ import {
|
|
|
37
37
|
untagAll as untagAllWorkflows
|
|
38
38
|
} from "./bulk-operations.js";
|
|
39
39
|
import { createTimeOperationCallbacks as createTimeOperationCallbacksForEngine } from "./callback-creators-bundles.js";
|
|
40
|
+
import { createPendingUpdateCallbacks as createPendingUpdateCallbacksForEngine } from "./callback-creators-core.js";
|
|
40
41
|
import {
|
|
41
42
|
createBroadcastCallbacks as createBroadcastCallbacksForEngine,
|
|
42
43
|
createInlineParkingCallbacks as createInlineParkingCallbacksForEngine,
|
|
@@ -88,7 +89,8 @@ import { recordFinalizerState } from "./finalizer-state.js";
|
|
|
88
89
|
import { getFinalizerStatus as getFinalizerStatusFromInternals } from "./finalizer-status.js";
|
|
89
90
|
import {
|
|
90
91
|
createWorkflowHandleWithResultPromise as createWorkflowHandleWithResultPromiseFromInternals,
|
|
91
|
-
getWorkflowResultPromise as getWorkflowResultPromiseFromInternals
|
|
92
|
+
getWorkflowResultPromise as getWorkflowResultPromiseFromInternals,
|
|
93
|
+
pollPendingCrossEngineResultWaiters
|
|
92
94
|
} from "./handle-result.js";
|
|
93
95
|
import { HANDLE_RESULT_PROMISE, WorkflowHandle } from "./handles.js";
|
|
94
96
|
import { hasQueuedInlineWorkflowStart } from "./inline-launch-queue.js";
|
|
@@ -119,6 +121,12 @@ import { getStreamChunksFromInternals } from "./operations-stream.js";
|
|
|
119
121
|
import {
|
|
120
122
|
handleTimerFired as handleTimerFiredFromInternals
|
|
121
123
|
} from "./operations-time.js";
|
|
124
|
+
import {
|
|
125
|
+
bootstrapWorkflowLeaseOwnership,
|
|
126
|
+
buildOwnerSideSignalPollTarget
|
|
127
|
+
} from "./ownership-bootstrap.js";
|
|
128
|
+
import { bootstrapOwnershipGates } from "./ownership-mode-marker.js";
|
|
129
|
+
import { processPendingUpdatesForHandlers as processPendingUpdatesForHandlersFromInternals } from "./pending-updates.js";
|
|
122
130
|
import { assertCompatiblePersistedDataVersion } from "./persisted-data-version.js";
|
|
123
131
|
import { query as queryWorkflow } from "./queries.js";
|
|
124
132
|
import {
|
|
@@ -154,7 +162,11 @@ import {
|
|
|
154
162
|
createSecondInstanceDetectionTick,
|
|
155
163
|
createSecondInstanceDetector
|
|
156
164
|
} from "./second-instance-detector.js";
|
|
157
|
-
import {
|
|
165
|
+
import {
|
|
166
|
+
hasBufferedSignal as hasBufferedSignalFromInternals,
|
|
167
|
+
releaseSignalWaiter,
|
|
168
|
+
signal as signalWorkflow
|
|
169
|
+
} from "./signals.js";
|
|
158
170
|
import {
|
|
159
171
|
loadScheduleState,
|
|
160
172
|
loadWorkflowState,
|
|
@@ -173,12 +185,14 @@ import {
|
|
|
173
185
|
timeoutWorkflow as timeoutWorkflowFromTermination
|
|
174
186
|
} from "./termination.js";
|
|
175
187
|
import {
|
|
188
|
+
deliverCoordinatedUpdateToWaiterIfAvailable as deliverCoordinatedUpdateToWaiterIfAvailableFromInternals,
|
|
176
189
|
getUpdateResult as getUpdateResultFromInternals,
|
|
177
190
|
submitCoordinatedUpdate as submitCoordinatedUpdateFromInternals,
|
|
178
191
|
update as updateFromInternals
|
|
179
192
|
} from "./updates.js";
|
|
180
193
|
import { isTerminalWorkflowStatus } from "./validation.js";
|
|
181
194
|
import { coerceScheduleId } from "./validation/schedule.js";
|
|
195
|
+
import { confirmWakeOwnership } from "./wake-ownership-guard.js";
|
|
182
196
|
import {
|
|
183
197
|
replayWorkflowFeed,
|
|
184
198
|
snapshotWorkflowFeedTail,
|
|
@@ -209,10 +223,18 @@ export {
|
|
|
209
223
|
WorkflowTypeNotRegisteredForRecoveryError
|
|
210
224
|
} from "./errors.js";
|
|
211
225
|
export { HANDLE_RESULT_PROMISE, WorkflowHandle } from "./handles.js";
|
|
226
|
+
export {
|
|
227
|
+
WeftWorkflowClaimLostWarning,
|
|
228
|
+
WeftWorkflowWakeDiscardedWarning,
|
|
229
|
+
WORKFLOW_CLAIM_LOST_WARNING_NAME,
|
|
230
|
+
WORKFLOW_WAKE_DISCARDED_WARNING_NAME
|
|
231
|
+
} from "./lease-deposition.js";
|
|
212
232
|
export {
|
|
213
233
|
EngineLeaseAcquisitionTimeoutError,
|
|
214
234
|
EngineLeaseCorruptedError,
|
|
215
|
-
EngineLeaseNotHeldError
|
|
235
|
+
EngineLeaseNotHeldError,
|
|
236
|
+
OwnershipModeMismatchError,
|
|
237
|
+
WorkflowClaimUnavailableError
|
|
216
238
|
} from "./lease-errors.js";
|
|
217
239
|
export { ScheduleHandle } from "./schedule-handle.js";
|
|
218
240
|
export {
|
|
@@ -257,6 +279,7 @@ export class Engine extends EventTarget {
|
|
|
257
279
|
throw new EngineCreateNameMismatchError("workflow", name, definition.name);
|
|
258
280
|
engine.register(definition);
|
|
259
281
|
}
|
|
282
|
+
await engine.#bootstrapOwnershipIfConfigured();
|
|
260
283
|
await engine.#acquireLeaseIfConfigured();
|
|
261
284
|
if (options.recover !== !1)
|
|
262
285
|
await engine.recoverAll(options.acknowledgeUnknownWorkflowTypes !== void 0 ? { acknowledgeUnknownWorkflowTypes: options.acknowledgeUnknownWorkflowTypes } : {});
|
|
@@ -331,7 +354,7 @@ export class Engine extends EventTarget {
|
|
|
331
354
|
onTimerFired: (entry) => handleTimerFiredFromInternals(getInternals(this), entry, this.#createTimeOperationCallbacks()),
|
|
332
355
|
pollIntervalMs: resolvedOptions.schedulerPollIntervalMs,
|
|
333
356
|
getNow,
|
|
334
|
-
commitTimerCleanup: (operations) => commitFencedEngineWrite(getInternals(this), operations, [], () => Error("Timer cleanup lost its fenced CAS race"))
|
|
357
|
+
commitTimerCleanup: (operations) => commitFencedEngineWrite(getInternals(this), null, operations, [], () => Error("Timer cleanup lost its fenced CAS race"))
|
|
335
358
|
});
|
|
336
359
|
getInternals(this).strategy = strategyBundle.strategy;
|
|
337
360
|
getInternals(this).inlineStrategy = strategyBundle.inlineStrategy;
|
|
@@ -359,9 +382,11 @@ export class Engine extends EventTarget {
|
|
|
359
382
|
getInternals(this).parkedInlineWorkflows = new Set;
|
|
360
383
|
getInternals(this).terminalizingWorkflows = new Set;
|
|
361
384
|
getInternals(this).deliveredPendingUpdateIds = new Map;
|
|
385
|
+
getInternals(this).onRecoveredWorkflowHook = void 0;
|
|
362
386
|
getInternals(this).cancelHandlersByWorkflow = new Map;
|
|
363
387
|
getInternals(this).reviewTimerIds = new Map;
|
|
364
388
|
getInternals(this).pendingWebhooks = new Set;
|
|
389
|
+
getInternals(this).pendingResultPollTimers = new Set;
|
|
365
390
|
getInternals(this).pendingTimelineEntries = new Map;
|
|
366
391
|
getInternals(this).pendingAtomicWorkflowCommitSideEffects = new Map;
|
|
367
392
|
getInternals(this).cleanupIntervalDisposalTracker = null;
|
|
@@ -381,7 +406,11 @@ export class Engine extends EventTarget {
|
|
|
381
406
|
getInternals(this).secondInstanceDetectionInterval = null;
|
|
382
407
|
getInternals(this).secondInstanceDetector = null;
|
|
383
408
|
getInternals(this).leaseManager = null;
|
|
409
|
+
getInternals(this).workflowClaimRegistry = null;
|
|
410
|
+
getInternals(this).workflowClaimRenewalTask = null;
|
|
411
|
+
getInternals(this).workflowClaimMetrics = null;
|
|
384
412
|
getInternals(this).inFlightLeaseAcquire = null;
|
|
413
|
+
getInternals(this).inFlightOwnershipBootstrap = null;
|
|
385
414
|
getInternals(this).deposed = !1;
|
|
386
415
|
getInternals(this).tearDownAfterDeposition = () => {
|
|
387
416
|
this.#disposeAfterDeposition();
|
|
@@ -458,15 +487,81 @@ export class Engine extends EventTarget {
|
|
|
458
487
|
internals.inFlightLeaseAcquire = null;
|
|
459
488
|
}
|
|
460
489
|
}
|
|
490
|
+
async#bootstrapOwnershipIfConfigured() {
|
|
491
|
+
const internals = getInternals(this), ownershipMode = internals.options.ownershipMode;
|
|
492
|
+
if (ownershipMode === "none")
|
|
493
|
+
return;
|
|
494
|
+
if (internals.disposed)
|
|
495
|
+
throw new EngineDisposedError;
|
|
496
|
+
if (ownershipMode === "lease") {
|
|
497
|
+
await bootstrapOwnershipGates({
|
|
498
|
+
storage: internals.storage,
|
|
499
|
+
ownershipMode,
|
|
500
|
+
getNow: internals.options.getNow
|
|
501
|
+
});
|
|
502
|
+
return;
|
|
503
|
+
}
|
|
504
|
+
if (internals.inFlightOwnershipBootstrap !== null) {
|
|
505
|
+
await internals.inFlightOwnershipBootstrap;
|
|
506
|
+
return;
|
|
507
|
+
}
|
|
508
|
+
if (internals.workflowClaimRegistry !== null)
|
|
509
|
+
return;
|
|
510
|
+
const bootstrap = (async () => {
|
|
511
|
+
const result = await bootstrapWorkflowLeaseOwnership({
|
|
512
|
+
storage: internals.storage,
|
|
513
|
+
getNow: internals.options.getNow,
|
|
514
|
+
claimTtlMs: internals.options.workflowClaimTtlMs,
|
|
515
|
+
claimRenewIntervalMs: internals.options.workflowClaimRenewIntervalMs,
|
|
516
|
+
isWorkflowTypeRegistered: (workflowType) => internals.workflowDefinitionsByName.has(workflowType),
|
|
517
|
+
onWorkflowClaimReclaimed: async (workflowId) => {
|
|
518
|
+
internals.inlineStrategy?.evictContextForReclaim(workflowId);
|
|
519
|
+
internals.deliveredPendingUpdateIds.delete(workflowId);
|
|
520
|
+
await recoverPendingAsyncActivities(internals, workflowId);
|
|
521
|
+
if (internals.disposed)
|
|
522
|
+
return;
|
|
523
|
+
await resumeFromLifecycle(internals, workflowId, this.#createLifecycleCallbacks(), internals.onRecoveredWorkflowHook, { forceReplayFromStorage: !0 });
|
|
524
|
+
},
|
|
525
|
+
signalPollTarget: this.#buildOwnerSideSignalPollTarget(),
|
|
526
|
+
updatePollTarget: this.#buildOwnerSideUpdatePollTarget()
|
|
527
|
+
});
|
|
528
|
+
if (internals.disposed)
|
|
529
|
+
throw new EngineDisposedError;
|
|
530
|
+
internals.workflowClaimRegistry = result.registry;
|
|
531
|
+
internals.workflowClaimRenewalTask = result.renewalTask;
|
|
532
|
+
internals.workflowClaimMetrics = result.metrics;
|
|
533
|
+
if (internals.options.backgroundTaskMode === "automatic")
|
|
534
|
+
result.renewalTask.start();
|
|
535
|
+
})();
|
|
536
|
+
internals.inFlightOwnershipBootstrap = bootstrap;
|
|
537
|
+
try {
|
|
538
|
+
await bootstrap;
|
|
539
|
+
} finally {
|
|
540
|
+
if (internals.inFlightOwnershipBootstrap === bootstrap)
|
|
541
|
+
internals.inFlightOwnershipBootstrap = null;
|
|
542
|
+
}
|
|
543
|
+
}
|
|
544
|
+
#detachWorkflowClaimOwnership() {
|
|
545
|
+
const internals = getInternals(this), registry = internals.workflowClaimRegistry;
|
|
546
|
+
internals.workflowClaimRenewalTask?.stop();
|
|
547
|
+
internals.workflowClaimRenewalTask = null;
|
|
548
|
+
internals.workflowClaimRegistry = null;
|
|
549
|
+
internals.workflowClaimMetrics = null;
|
|
550
|
+
return { registry };
|
|
551
|
+
}
|
|
552
|
+
async#releaseWorkflowClaimsBestEffort(registry) {
|
|
553
|
+
await registry?.releaseAll().catch(() => {});
|
|
554
|
+
}
|
|
461
555
|
#disposeAfterDeposition() {
|
|
462
556
|
const internals = getInternals(this);
|
|
463
557
|
if (internals.disposed)
|
|
464
558
|
return;
|
|
465
|
-
const leaseManager = internals.leaseManager;
|
|
559
|
+
const leaseManager = internals.leaseManager, { registry: workflowClaimRegistry } = this.#detachWorkflowClaimOwnership();
|
|
466
560
|
disposeEngine(internals);
|
|
467
561
|
if (this.#synchronousDisposeResult === null)
|
|
468
562
|
this.#synchronousDisposeResult = (leaseManager?.release() ?? Promise.resolve(!0)).catch(() => !1);
|
|
469
563
|
this.#synchronousDisposeResult;
|
|
564
|
+
this.#releaseWorkflowClaimsBestEffort(workflowClaimRegistry);
|
|
470
565
|
}
|
|
471
566
|
#startSecondInstanceDetection() {
|
|
472
567
|
const internals = getInternals(this);
|
|
@@ -517,9 +612,55 @@ export class Engine extends EventTarget {
|
|
|
517
612
|
#createInlineParkingCallbacks() {
|
|
518
613
|
return createInlineParkingCallbacksForEngine(this);
|
|
519
614
|
}
|
|
615
|
+
#buildOwnerSideSignalPollTarget() {
|
|
616
|
+
const internals = getInternals(this);
|
|
617
|
+
return buildOwnerSideSignalPollTarget({
|
|
618
|
+
listParkedInlineWorkflowIds: () => internals.parkedInlineWorkflows,
|
|
619
|
+
isParkedInlineWorkflow: (workflowId) => internals.parkedInlineWorkflows.has(workflowId),
|
|
620
|
+
parkedSignalName: (workflowId) => {
|
|
621
|
+
const pending = internals.pendingTimelineEntries.get(workflowId);
|
|
622
|
+
return pending?.entry.operationType === "wait-signal" ? pending.entry.operationLabel : void 0;
|
|
623
|
+
},
|
|
624
|
+
listSignalWaiterEntries: function* signalWaiterEntries() {
|
|
625
|
+
for (const [workflowId, waiterKeys] of internals.signalWaitersByWorkflow) {
|
|
626
|
+
if (typeof waiterKeys === "string") {
|
|
627
|
+
yield [workflowId, waiterKeys];
|
|
628
|
+
continue;
|
|
629
|
+
}
|
|
630
|
+
for (const waiterKey of waiterKeys)
|
|
631
|
+
yield [workflowId, waiterKey];
|
|
632
|
+
}
|
|
633
|
+
},
|
|
634
|
+
hasBufferedSignal: (workflowId, signalName) => hasBufferedSignalFromInternals(internals, workflowId, signalName),
|
|
635
|
+
resumeParkedInlineWorkflow: (workflowId) => resumeParkedInlineWorkflowFromInternals(internals, workflowId, this.#createInlineParkingCallbacks()),
|
|
636
|
+
confirmSignalWakeOwnership: (workflowId) => confirmWakeOwnership(internals, workflowId, "signal"),
|
|
637
|
+
wakeSignalWaiter: (workflowId, waiterKey) => {
|
|
638
|
+
const waiter = internals.signalWaiters.get(waiterKey);
|
|
639
|
+
if (waiter === void 0)
|
|
640
|
+
return;
|
|
641
|
+
if (releaseSignalWaiter(internals, workflowId, waiterKey, waiter))
|
|
642
|
+
waiter();
|
|
643
|
+
}
|
|
644
|
+
});
|
|
645
|
+
}
|
|
520
646
|
#createUpdateCallbacks() {
|
|
521
647
|
return createUpdateCallbacksForEngine(this);
|
|
522
648
|
}
|
|
649
|
+
#buildOwnerSideUpdatePollTarget() {
|
|
650
|
+
const internals = getInternals(this);
|
|
651
|
+
return {
|
|
652
|
+
listHeldWorkflowIds: () => internals.workflowClaimRegistry?.listHeldWorkflowIds() ?? [],
|
|
653
|
+
hasPendingUpdates: async (workflowId) => {
|
|
654
|
+
return (await internals.updateCoordinator.getPendingUpdates(workflowId)).length > 0;
|
|
655
|
+
},
|
|
656
|
+
drainPendingUpdates: async (workflowId) => {
|
|
657
|
+
await processPendingUpdatesForHandlersFromInternals(internals, workflowId, createPendingUpdateCallbacksForEngine(this));
|
|
658
|
+
const stillPending = await internals.updateCoordinator.getPendingUpdates(workflowId), updateCallbacks = this.#createUpdateCallbacks();
|
|
659
|
+
for (const updateRequest of stillPending)
|
|
660
|
+
await deliverCoordinatedUpdateToWaiterIfAvailableFromInternals(internals, workflowId, updateRequest, !1, updateCallbacks);
|
|
661
|
+
}
|
|
662
|
+
};
|
|
663
|
+
}
|
|
523
664
|
#createTimeOperationCallbacks() {
|
|
524
665
|
return createTimeOperationCallbacksForEngine(this);
|
|
525
666
|
}
|
|
@@ -629,6 +770,9 @@ export class Engine extends EventTarget {
|
|
|
629
770
|
}
|
|
630
771
|
async runMaintenance(now = getInternals(this).options.getNow()) {
|
|
631
772
|
const internals = getInternals(this);
|
|
773
|
+
await this.#bootstrapOwnershipIfConfigured();
|
|
774
|
+
await internals.workflowClaimRenewalTask?.runOnce();
|
|
775
|
+
await pollPendingCrossEngineResultWaiters(internals);
|
|
632
776
|
await internals.scheduler.tick(now);
|
|
633
777
|
try {
|
|
634
778
|
await internals.updateCoordinator.cleanupExpiredResponses();
|
|
@@ -768,11 +912,13 @@ export class Engine extends EventTarget {
|
|
|
768
912
|
return resumeFromLifecycle(getInternals(this), workflowId, this.#createLifecycleCallbacks());
|
|
769
913
|
}
|
|
770
914
|
async recoverAll(options) {
|
|
915
|
+
await this.#bootstrapOwnershipIfConfigured();
|
|
771
916
|
await this.#acquireLeaseIfConfigured();
|
|
772
917
|
if (getInternals(this).disposed)
|
|
773
918
|
throw new EngineDisposedError;
|
|
774
919
|
await recoverPendingAsyncActivities(getInternals(this));
|
|
775
920
|
await recoverOrphanedScheduleTimers(getInternals(this));
|
|
921
|
+
getInternals(this).onRecoveredWorkflowHook = options?.onRecoveredWorkflow;
|
|
776
922
|
return recoverAllFromLifecycle(getInternals(this), this.#createLifecycleCallbacks(), options);
|
|
777
923
|
}
|
|
778
924
|
async completeAsyncActivity(token, result) {
|
|
@@ -934,10 +1080,12 @@ export class Engine extends EventTarget {
|
|
|
934
1080
|
const leaseManager = getInternals(this).leaseManager;
|
|
935
1081
|
if (leaseManager !== null && leaseManager.currentEpochBytes() !== null)
|
|
936
1082
|
process.emitWarning("engine ownership lease disposed synchronously; lease release is fire-and-forget, so exiting before the release completes can make the next instance wait for leaseTtl. Use await engine.shutdown(), await using, or await engine[Symbol.asyncDispose]() for prompt rolling-deploy handoff.", ENGINE_LEASE_SYNCHRONOUS_DISPOSE_WARNING_NAME);
|
|
1083
|
+
const { registry: workflowClaimRegistry } = this.#detachWorkflowClaimOwnership();
|
|
937
1084
|
disposeEngine(getInternals(this));
|
|
938
1085
|
if (this.#synchronousDisposeResult === null)
|
|
939
1086
|
this.#synchronousDisposeResult = leaseManager?.release() ?? Promise.resolve(!0);
|
|
940
1087
|
this.#synchronousDisposeResult;
|
|
1088
|
+
this.#releaseWorkflowClaimsBestEffort(workflowClaimRegistry);
|
|
941
1089
|
}
|
|
942
1090
|
async#disposeAsyncWithLeaseResult() {
|
|
943
1091
|
if (!getInternals(this).disposed) {
|
|
@@ -950,9 +1098,11 @@ export class Engine extends EventTarget {
|
|
|
950
1098
|
} catch (error) {
|
|
951
1099
|
drainFailure = { error };
|
|
952
1100
|
} finally {
|
|
1101
|
+
const { registry: workflowClaimRegistry } = this.#detachWorkflowClaimOwnership();
|
|
953
1102
|
disposeEngine(getInternals(this));
|
|
954
1103
|
await getInternals(this).inFlightLeaseAcquire?.catch(() => {});
|
|
955
1104
|
leaseReleased = await (this.#synchronousDisposeResult ?? leaseManager?.release() ?? Promise.resolve(!0));
|
|
1105
|
+
await this.#releaseWorkflowClaimsBestEffort(workflowClaimRegistry);
|
|
956
1106
|
}
|
|
957
1107
|
if (drainFailure !== null)
|
|
958
1108
|
throw new EngineDisposalError(drainFailure.error, leaseReleased);
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { WorkflowStartedEvent } from "../events.js";
|
|
2
2
|
import { startWorkflowExecution } from "./lifecycle.js";
|
|
3
3
|
import { loadWorkflowState } from "./storage-io.js";
|
|
4
|
+
import { confirmWakeOwnership } from "./wake-ownership-guard.js";
|
|
4
5
|
async function yieldQueuedShutdownAdvanceOpportunity() {
|
|
5
6
|
await new Promise((resolve) => {
|
|
6
7
|
setTimeout(resolve, 0);
|
|
@@ -83,11 +84,19 @@ export async function drainQueuedInlineWorkflowStarts(internals, callbacks, opti
|
|
|
83
84
|
}
|
|
84
85
|
}
|
|
85
86
|
}
|
|
87
|
+
async function isQueuedStartStillTheLiveRun(internals, start) {
|
|
88
|
+
const currentState = await loadWorkflowState(internals, start.workflowId);
|
|
89
|
+
return currentState !== null && currentState.status === "running" && (start.workflowExecutionToken === void 0 || currentState.workflowExecutionToken === start.workflowExecutionToken);
|
|
90
|
+
}
|
|
86
91
|
async function startQueuedInlineWorkflowExecution(internals, start, callbacks, options) {
|
|
87
92
|
try {
|
|
88
93
|
const state = await loadWorkflowState(internals, start.workflowId);
|
|
89
94
|
if (!state || state.status !== "running")
|
|
90
95
|
return;
|
|
96
|
+
if (await confirmWakeOwnership(internals, start.workflowId, "inline-macrotask-drive") === "discard")
|
|
97
|
+
return;
|
|
98
|
+
if (!await isQueuedStartStillTheLiveRun(internals, start))
|
|
99
|
+
return;
|
|
91
100
|
internals.queuedInlineWorkflowStartIds.delete(start.workflowId);
|
|
92
101
|
internals.engine.dispatchEvent(new WorkflowStartedEvent(start.workflowId, start.workflowType, start.input));
|
|
93
102
|
startWorkflowExecution(internals, start.workflowId, state.workflowExecutionToken, start.workflowType, start.input, start.checkpoint, start.nestingDepth, start.executionDeadline, start.executionStateOwnerId);
|