@lostgradient/weft 0.20.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/core/engine/activity-reconciliation.d.ts +2 -2
- package/dist/core/engine/activity-reconciliation.js +12 -8
- package/dist/core/engine/async-activity-completion.js +12 -4
- package/dist/core/engine/async-activity-records.d.ts +11 -1
- package/dist/core/engine/async-activity-records.js +4 -3
- package/dist/core/engine/attributes-tags.d.ts +11 -1
- package/dist/core/engine/attributes-tags.js +4 -3
- package/dist/core/engine/bulk-operations-purge.js +5 -2
- package/dist/core/engine/bulk-operations.js +17 -5
- package/dist/core/engine/callback-creators-bundles.js +1 -1
- package/dist/core/engine/callback-creators-core.js +4 -2
- package/dist/core/engine/checkpoint-io.js +1 -1
- package/dist/core/engine/child-workflow.js +2 -1
- package/dist/core/engine/completed-review-storage.js +1 -1
- package/dist/core/engine/condition-waiters.d.ts +45 -3
- package/dist/core/engine/condition-waiters.js +12 -3
- package/dist/core/engine/disposal.js +6 -0
- package/dist/core/engine/engine-internal-types.d.ts +12 -4
- package/dist/core/engine/fenced-write.d.ts +46 -15
- package/dist/core/engine/fenced-write.js +39 -6
- package/dist/core/engine/finalizer-state.d.ts +3 -1
- package/dist/core/engine/handle-result.d.ts +49 -1
- package/dist/core/engine/handle-result.js +95 -20
- package/dist/core/engine/index.d.ts +13 -5
- package/dist/core/engine/index.js +155 -5
- package/dist/core/engine/inline-launch-queue.js +9 -0
- package/dist/core/engine/internals.d.ts +73 -0
- package/dist/core/engine/invoke-update-handler.d.ts +19 -0
- package/dist/core/engine/invoke-update-handler.js +7 -0
- package/dist/core/engine/lease-deposition.d.ts +132 -0
- package/dist/core/engine/lease-deposition.js +31 -0
- package/dist/core/engine/lease-errors.d.ts +80 -9
- package/dist/core/engine/lease-errors.js +28 -2
- package/dist/core/engine/lifecycle/resume.js +3 -1
- package/dist/core/engine/lifecycle/standalone-claim-acquire.d.ts +33 -0
- package/dist/core/engine/lifecycle/standalone-claim-acquire.js +25 -0
- package/dist/core/engine/lifecycle/start-commit.js +20 -6
- package/dist/core/engine/lifecycle/transition.d.ts +15 -1
- package/dist/core/engine/lifecycle/transition.js +6 -3
- package/dist/core/engine/operations-activity.js +1 -1
- package/dist/core/engine/operations-time.js +21 -7
- package/dist/core/engine/owner-side-signal-poll.d.ts +132 -0
- package/dist/core/engine/owner-side-signal-poll.js +22 -0
- package/dist/core/engine/owner-side-update-poll.d.ts +116 -0
- package/dist/core/engine/owner-side-update-poll.js +22 -0
- package/dist/core/engine/ownership-bootstrap.d.ts +280 -0
- package/dist/core/engine/ownership-bootstrap.js +89 -0
- package/dist/core/engine/ownership-mode-marker.d.ts +82 -0
- package/dist/core/engine/ownership-mode-marker.js +52 -0
- package/dist/core/engine/ownership-options.d.ts +22 -6
- package/dist/core/engine/ownership-options.js +42 -12
- package/dist/core/engine/pending-updates.js +35 -24
- package/dist/core/engine/queries.d.ts +70 -0
- package/dist/core/engine/queries.js +58 -4
- package/dist/core/engine/schedules.js +1 -1
- package/dist/core/engine/signals.d.ts +11 -1
- package/dist/core/engine/signals.js +11 -7
- package/dist/core/engine/sleep-timer-acknowledgements.d.ts +68 -0
- package/dist/core/engine/sleep-timer-acknowledgements.js +19 -0
- package/dist/core/engine/storage-io.d.ts +58 -10
- package/dist/core/engine/storage-io.js +27 -9
- package/dist/core/engine/termination/cleanup.d.ts +5 -4
- package/dist/core/engine/termination/complete.js +23 -4
- package/dist/core/engine/termination/finalizer-claim.js +4 -4
- package/dist/core/engine/termination/state-commit-callbacks.d.ts +15 -0
- package/dist/core/engine/termination/state-commit-callbacks.js +0 -0
- package/dist/core/engine/termination/suspend.js +1 -1
- package/dist/core/engine/update-validation.d.ts +28 -0
- package/dist/core/engine/update-validation.js +37 -0
- package/dist/core/engine/updates.d.ts +17 -16
- package/dist/core/engine/updates.js +24 -47
- package/dist/core/engine/wake-ownership-check.d.ts +86 -0
- package/dist/core/engine/wake-ownership-check.js +19 -0
- package/dist/core/engine/wake-ownership-guard.d.ts +46 -0
- package/dist/core/engine/wake-ownership-guard.js +23 -0
- package/dist/core/engine/workflow-claim-codec.d.ts +64 -0
- package/dist/core/engine/workflow-claim-codec.js +44 -0
- package/dist/core/engine/workflow-claim-cooldown.d.ts +60 -0
- package/dist/core/engine/workflow-claim-cooldown.js +17 -0
- package/dist/core/engine/workflow-claim-fold.d.ts +82 -0
- package/dist/core/engine/workflow-claim-fold.js +53 -0
- package/dist/core/engine/workflow-claim-metrics.d.ts +131 -0
- package/dist/core/engine/workflow-claim-metrics.js +62 -0
- package/dist/core/engine/workflow-claim-reclaim-scan.d.ts +96 -0
- package/dist/core/engine/workflow-claim-reclaim-scan.js +66 -0
- package/dist/core/engine/workflow-claim-reclaim-target.d.ts +107 -0
- package/dist/core/engine/workflow-claim-reclaim-target.js +155 -0
- package/dist/core/engine/workflow-claim-registry.d.ts +227 -0
- package/dist/core/engine/workflow-claim-registry.js +198 -0
- package/dist/core/engine/workflow-claim-renewal-interval.d.ts +50 -0
- package/dist/core/engine/workflow-claim-renewal-interval.js +132 -0
- package/dist/core/engine/workflow-claim-renewal-subpasses.d.ts +265 -0
- package/dist/core/engine/workflow-claim-renewal-subpasses.js +104 -0
- package/dist/core/engine/workflow-claim-renewal-task.d.ts +159 -0
- package/dist/core/engine/workflow-claim-renewal-task.js +65 -0
- package/dist/core/engine/workflow-claim-transitions.d.ts +186 -0
- package/dist/core/engine/workflow-claim-transitions.js +120 -0
- package/dist/core/inline-execution-strategy.d.ts +20 -0
- package/dist/core/inline-execution-strategy.js +15 -4
- package/dist/core/scheduler/scheduler-class.js +3 -3
- package/dist/core/scheduler/timer-sources.d.ts +13 -0
- package/dist/core/scheduler/timer-sources.js +1 -1
- package/dist/core/types/options.d.ts +17 -1
- package/dist/core/weft-error.d.ts +1 -1
- package/dist/core/weft-error.js +2 -1
- package/dist/http.js +2 -2
- package/dist/index.d.ts +2 -1
- package/dist/index.js +8 -0
- package/dist/indexeddb.js +1 -1
- package/dist/json-schema.js +3 -3
- package/dist/observability/index.js +2 -2
- package/dist/storage/auto.js +1 -1
- package/dist/storage/bun-sql.js +113 -0
- package/dist/storage/compressed-storage.js +1 -1
- package/dist/storage/index.d.ts +38 -35
- package/dist/storage/interface.d.ts +87 -125
- package/dist/storage/interface.js +1 -1
- package/dist/storage/key-encoding.d.ts +51 -0
- package/dist/storage/key-encoding.js +13 -0
- package/dist/storage/key-prefixes.d.ts +1 -1
- package/dist/storage/key-prefixes.js +3 -0
- package/dist/storage/lmdb.js +1 -1
- package/dist/storage/memory.js +1 -1
- package/dist/storage/neon.js +2 -2
- package/dist/storage/node-sqlite.js +113 -0
- package/dist/storage/ownership-keys.d.ts +55 -0
- package/dist/storage/ownership-keys.js +6 -0
- package/dist/storage/postgres.js +2 -2
- package/dist/storage/resolve.js +1 -1
- package/dist/storage/scoped-storage.js +1 -1
- package/dist/storage/testing.js +1 -1
- package/dist/storage/turso.js +2 -2
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/web-extension.js +1 -1
- package/dist/worker/protocol.js +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Weft
|
|
2
2
|
|
|
3
|
-
A Bun-native durable execution engine. Current release: `0.
|
|
3
|
+
A Bun-native durable execution engine. Current release: `0.21.0`.
|
|
4
4
|
|
|
5
5
|
Install the library from npm as `@lostgradient/weft`:
|
|
6
6
|
|
|
@@ -96,7 +96,7 @@ const result = await handle.result();
|
|
|
96
96
|
|
|
97
97
|
That's the core loop: `workflow({ name })` is a **chained builder** that co-locates the workflow's side-effecting steps inside `.activities({...})`, and `.execute(fn)` seals it all together and returns a `WorkflowDefinition`. Inside the generator, `ctx.run('formatGreeting', input)` autocompletes from the workflow's own activity table, typechecks the input, and infers the output. Every `yield*` is a checkpoint boundary; `handle.result()` waits for the output. Checkpoints are written to `./weft.db`, so running workflows survive process crashes.
|
|
98
98
|
|
|
99
|
-
`Engine.create()` does the registration dance for you: it constructs the engine and registers each workflow in the `workflows` map, pulling in all the activities each workflow declares. It then **recovers by default** — `engine.recoverAll()` runs after registration, so any workflows still running from a previous process pick up where they left off. That's the point of durable storage, so you don't have to ask for it. Pass `recover: false` to opt out (handy for tests, for `ScopedStorage`-isolated engines, or when you want to inspect a store before migrating it). Durability is separate: each step is persisted before it commits no matter what `recover` is set to — `recover` only decides whether _this_ engine resumes that persisted work on boot. If a host owns recovery but still needs durable timers, use `startScheduler: true`; otherwise `recover: false` also leaves the real-time scheduler stopped.
|
|
99
|
+
`Engine.create()` does the registration dance for you: it constructs the engine and registers each workflow in the `workflows` map, pulling in all the activities each workflow declares. It then **recovers by default** — `engine.recoverAll()` runs after registration, so any workflows still running from a previous process pick up where they left off. That's the point of durable storage, so you don't have to ask for it. Pass `recover: false` to opt out (handy for tests, for `ScopedStorage`-isolated engines, or when you want to inspect a store before migrating it). Durability is separate: each step is persisted before it commits no matter what `recover` is set to — `recover` only decides whether _this_ engine resumes that persisted work on boot. If a host owns recovery but still needs durable timers, use `startScheduler: true`; otherwise `recover: false` also leaves the real-time scheduler stopped. By default (`ownership: 'none'`) run a single engine per durable store: pointing two at the same store is uncoordinated and can double-resume a workflow. To share one store across engines, select `ownership: 'workflow-lease'`, which claims each workflow for exactly one engine before its generator runs.
|
|
100
100
|
|
|
101
101
|
Passing an explicitly empty workflow map is the same default-registry boot shape as omitting `workflows`: `Engine.create({ workflows: {} })` recovers after registration and returns an engine whose TypeScript type is compatible with default-registry consumers such as `serve({ engine })`. Use a non-empty `workflows` map when you want TypeScript to narrow `engine.start(...)` to the registered names.
|
|
102
102
|
|
|
@@ -95,14 +95,14 @@ export declare function buildActivityReconciliationReference(workflowId: string,
|
|
|
95
95
|
export declare function resolveActivityIdempotencyKey(activity: ActivityReconciliationMetadata | undefined, operation: ActivityOperation): string | undefined;
|
|
96
96
|
export declare function readActivityReconciliationRecord(storage: Storage, key: string): Promise<ActivityReconciliationRecord | null>;
|
|
97
97
|
export declare function validateActivityResultForReconciliation(result: unknown, maxPayloadBytes: number | null): Uint8Array;
|
|
98
|
-
export declare function claimActivityReconciliationStart(
|
|
98
|
+
export declare function claimActivityReconciliationStart(internals: EngineInternals, workflowId: string, reference: ActivityReconciliationReference, record: ActivityReconciliationRecord): Promise<boolean>;
|
|
99
99
|
export declare function createCompletedActivityReconciliationRecord(startedRecord: ActivityReconciliationRecord, result: unknown, now: number): ActivityReconciliationRecord;
|
|
100
100
|
export declare function resolveStartedActivityReconciliationRecord(internals: EngineInternals, workflowId: string, operation: ActivityOperation, reference: ActivityReconciliationReference, activity: ActivityReconciliationMetadata | undefined, idempotencyKey: string, attempt: number): Promise<ActivityReconciliationRecord | {
|
|
101
101
|
completedResult: unknown;
|
|
102
102
|
}>;
|
|
103
103
|
export declare function writeActivityReconciliationTransition(storage: Storage, reference: ActivityReconciliationReference, expectedRecord: ActivityReconciliationRecord, nextRecord: ActivityReconciliationRecord): Promise<void>;
|
|
104
104
|
export declare function stageActivityReconciliationTransitionWithAtomicWorkflowCommit(internals: EngineInternals, workflowId: string, reference: ActivityReconciliationReference, expectedRecord: ActivityReconciliationRecord, nextRecord: ActivityReconciliationRecord): void;
|
|
105
|
-
export declare function commitActivityReconciliationTransitionWithFencedWrite(internals: EngineInternals, reference: ActivityReconciliationReference, expectedRecord: ActivityReconciliationRecord, nextRecord: ActivityReconciliationRecord): Promise<void>;
|
|
105
|
+
export declare function commitActivityReconciliationTransitionWithFencedWrite(internals: EngineInternals, workflowId: string, reference: ActivityReconciliationReference, expectedRecord: ActivityReconciliationRecord, nextRecord: ActivityReconciliationRecord): Promise<void>;
|
|
106
106
|
export declare function normalizePreDispatchVerificationResult(result: ActivityVerificationResult): 'not-completed' | 'completed-result-unavailable' | 'indeterminate' | {
|
|
107
107
|
result: unknown;
|
|
108
108
|
};
|
|
@@ -7,7 +7,10 @@ import { decode, encode } from "../codec.js";
|
|
|
7
7
|
import { assertPayloadWithinLimit } from "../payload-size.js";
|
|
8
8
|
import { WeftError } from "../weft-error.js";
|
|
9
9
|
import { stageAtomicWorkflowCommitSideEffects } from "./checkpoint-side-effects.js";
|
|
10
|
-
import {
|
|
10
|
+
import {
|
|
11
|
+
commitFencedEngineWrite,
|
|
12
|
+
commitFencedEngineWriteAllowingPreconditionFailure
|
|
13
|
+
} from "./fenced-write.js";
|
|
11
14
|
|
|
12
15
|
export class ActivityReconciliationCapabilityError extends WeftError {
|
|
13
16
|
constructor() {
|
|
@@ -50,11 +53,12 @@ export function validateActivityResultForReconciliation(result, maxPayloadBytes)
|
|
|
50
53
|
assertPayloadWithinLimit(result, maxPayloadBytes, "activity result");
|
|
51
54
|
return encode(result);
|
|
52
55
|
}
|
|
53
|
-
export async function claimActivityReconciliationStart(
|
|
56
|
+
export async function claimActivityReconciliationStart(internals, workflowId, reference, record) {
|
|
57
|
+
const storage = internals.storage;
|
|
54
58
|
if (!storage.capabilities().conditionalBatch)
|
|
55
59
|
throw new ActivityReconciliationCapabilityError;
|
|
56
60
|
requireStorageCapability(storage, "conditionalBatch", "activity result reconciliation");
|
|
57
|
-
return
|
|
61
|
+
return commitFencedEngineWriteAllowingPreconditionFailure(internals, workflowId, [{ type: "put", key: reference.key, value: encode(record) }], [{ key: reference.key, expectedValue: null }]);
|
|
58
62
|
}
|
|
59
63
|
export function createCompletedActivityReconciliationRecord(startedRecord, result, now) {
|
|
60
64
|
return {
|
|
@@ -84,7 +88,7 @@ export async function resolveStartedActivityReconciliationRecord(internals, work
|
|
|
84
88
|
ownerId: crypto.randomUUID(),
|
|
85
89
|
updatedAt: internals.options.getNow()
|
|
86
90
|
};
|
|
87
|
-
await commitActivityReconciliationTransitionWithFencedWrite(internals, reference, record, nextRecord);
|
|
91
|
+
await commitActivityReconciliationTransitionWithFencedWrite(internals, workflowId, reference, record, nextRecord);
|
|
88
92
|
return nextRecord;
|
|
89
93
|
}
|
|
90
94
|
if (normalized === "completed-result-unavailable")
|
|
@@ -93,7 +97,7 @@ export async function resolveStartedActivityReconciliationRecord(internals, work
|
|
|
93
97
|
throw new ActivityReconciliationIndeterminateError(`Activity "${operation.activityName}" reconciliation is indeterminate.`);
|
|
94
98
|
validateActivityResultForReconciliation(normalized.result, internals.options.payloadSizePolicy.maxBytes);
|
|
95
99
|
const completedRecord = createCompletedActivityReconciliationRecord(record, normalized.result, internals.options.getNow());
|
|
96
|
-
await commitActivityReconciliationTransitionWithFencedWrite(internals, reference, record, completedRecord);
|
|
100
|
+
await commitActivityReconciliationTransitionWithFencedWrite(internals, workflowId, reference, record, completedRecord);
|
|
97
101
|
return { completedResult: normalized.result };
|
|
98
102
|
}
|
|
99
103
|
export async function writeActivityReconciliationTransition(storage, reference, expectedRecord, nextRecord) {
|
|
@@ -104,9 +108,9 @@ export async function writeActivityReconciliationTransition(storage, reference,
|
|
|
104
108
|
export function stageActivityReconciliationTransitionWithAtomicWorkflowCommit(internals, workflowId, reference, expectedRecord, nextRecord) {
|
|
105
109
|
stageAtomicWorkflowCommitSideEffects(internals, workflowId, buildActivityReconciliationTransitionSideEffects(reference, expectedRecord, nextRecord));
|
|
106
110
|
}
|
|
107
|
-
export async function commitActivityReconciliationTransitionWithFencedWrite(internals, reference, expectedRecord, nextRecord) {
|
|
111
|
+
export async function commitActivityReconciliationTransitionWithFencedWrite(internals, workflowId, reference, expectedRecord, nextRecord) {
|
|
108
112
|
const sideEffects = buildActivityReconciliationTransitionSideEffects(reference, expectedRecord, nextRecord);
|
|
109
|
-
await commitFencedEngineWrite(internals, sideEffects.operations, sideEffects.conditions, () => new ActivityReconciliationConflictError("Activity reconciliation completion lost compare-and-set ownership."));
|
|
113
|
+
await commitFencedEngineWrite(internals, workflowId, sideEffects.operations, sideEffects.conditions, () => new ActivityReconciliationConflictError("Activity reconciliation completion lost compare-and-set ownership."));
|
|
110
114
|
}
|
|
111
115
|
function buildActivityReconciliationTransitionSideEffects(reference, expectedRecord, nextRecord) {
|
|
112
116
|
return {
|
|
@@ -171,7 +175,7 @@ function createStartedRecord(workflowId, operation, reference, attempt, now) {
|
|
|
171
175
|
}
|
|
172
176
|
async function claimStartedRecord(internals, workflowId, operation, reference, attempt) {
|
|
173
177
|
const startedRecord = createStartedRecord(workflowId, operation, reference, attempt, internals.options.getNow());
|
|
174
|
-
if (await claimActivityReconciliationStart(internals
|
|
178
|
+
if (await claimActivityReconciliationStart(internals, workflowId, reference, startedRecord))
|
|
175
179
|
return { didClaim: !0, record: startedRecord };
|
|
176
180
|
const record = await readActivityReconciliationRecord(internals.storage, reference.key);
|
|
177
181
|
if (record === null)
|
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
} from "./async-activity-records.js";
|
|
11
11
|
import { stageAtomicWorkflowCommitSideEffects } from "./checkpoint-side-effects.js";
|
|
12
12
|
import { commitFencedEngineWrite } from "./fenced-write.js";
|
|
13
|
+
import { confirmWakeOwnership } from "./wake-ownership-guard.js";
|
|
13
14
|
|
|
14
15
|
export class AsyncActivityDeferral extends Error {
|
|
15
16
|
token;
|
|
@@ -30,7 +31,7 @@ export class AsyncActivityTokenNotFoundError extends WeftError {
|
|
|
30
31
|
export async function parkDeferredAsyncActivity(internals, deferral, details, callbacks) {
|
|
31
32
|
const queuedResolution = takePendingAsyncActivityResolution(internals, details.workflowId, deferral.token);
|
|
32
33
|
if (queuedResolution !== void 0) {
|
|
33
|
-
deliverPendingAsyncActivityResolution(internals, details.workflowId, queuedResolution, callbacks);
|
|
34
|
+
await deliverPendingAsyncActivityResolution(internals, details.workflowId, queuedResolution, callbacks);
|
|
34
35
|
return new Promise(() => {});
|
|
35
36
|
}
|
|
36
37
|
await registerPendingAsyncActivity(internals, {
|
|
@@ -44,16 +45,23 @@ async function consumePendingAsyncActivity(internals, token, outcome) {
|
|
|
44
45
|
const pending = internals.pendingAsyncActivities.get(token);
|
|
45
46
|
if (!pending)
|
|
46
47
|
throw new AsyncActivityTokenNotFoundError(token);
|
|
48
|
+
const registry = internals.workflowClaimRegistry;
|
|
49
|
+
if (registry !== null && registry.currentEpoch(pending.workflowId) === null)
|
|
50
|
+
throw new AsyncActivityTokenNotFoundError(token);
|
|
47
51
|
internals.pendingAsyncActivities.delete(token);
|
|
48
52
|
try {
|
|
49
|
-
await commitFencedEngineWrite(internals, buildAsyncActivityAcknowledgementOperations(pending, outcome), [], () => Error(`Async activity acknowledgement for token "${token}" lost its precondition.`));
|
|
53
|
+
await commitFencedEngineWrite(internals, pending.workflowId, buildAsyncActivityAcknowledgementOperations(pending, outcome), [], () => Error(`Async activity acknowledgement for token "${token}" lost its precondition.`));
|
|
50
54
|
} catch (error) {
|
|
51
55
|
internals.pendingAsyncActivities.set(token, pending);
|
|
52
56
|
throw error;
|
|
53
57
|
}
|
|
54
58
|
return pending;
|
|
55
59
|
}
|
|
56
|
-
function deliverPendingAsyncActivityResolution(internals, workflowId, resolution, callbacks) {
|
|
60
|
+
async function deliverPendingAsyncActivityResolution(internals, workflowId, resolution, callbacks) {
|
|
61
|
+
if (internals.workflowClaimRegistry !== null) {
|
|
62
|
+
if (await confirmWakeOwnership(internals, workflowId, "async-activity") === "discard")
|
|
63
|
+
return;
|
|
64
|
+
}
|
|
57
65
|
stageAtomicWorkflowCommitSideEffects(internals, workflowId, {
|
|
58
66
|
conditions: [],
|
|
59
67
|
operations: [
|
|
@@ -75,7 +83,7 @@ async function resolvePendingAsyncActivity(internals, token, outcome, callbacks,
|
|
|
75
83
|
queuePendingAsyncActivityResolution(internals, pending.workflowId, resolution);
|
|
76
84
|
return;
|
|
77
85
|
}
|
|
78
|
-
deliverPendingAsyncActivityResolution(internals, pending.workflowId, resolution, callbacks);
|
|
86
|
+
await deliverPendingAsyncActivityResolution(internals, pending.workflowId, resolution, callbacks);
|
|
79
87
|
}
|
|
80
88
|
export async function completeAsyncActivity(internals, token, result, callbacks) {
|
|
81
89
|
assertPayloadWithinLimit(result, internals.options.payloadSizePolicy.maxBytes, "activity result");
|
|
@@ -104,8 +104,18 @@ export declare function registerPendingAsyncActivity(internals: EngineInternals,
|
|
|
104
104
|
* before the recovered workflow has replayed far enough to re-register it —
|
|
105
105
|
* and so an acknowledged-but-not-yet-checkpointed resolution is redelivered
|
|
106
106
|
* when replay re-parks on the same deterministic token.
|
|
107
|
+
*
|
|
108
|
+
* Pass `workflowId` to reload just one workflow's records. ADR 0002's
|
|
109
|
+
* reclaim-driven resume needs this: a resolution acknowledged by an expired
|
|
110
|
+
* owner is discarded by that engine's `confirmWakeOwnership` check, leaving
|
|
111
|
+
* only the durable record. The engine that takes the workflow over finished
|
|
112
|
+
* its startup scan long ago, so without a per-workflow reload it never
|
|
113
|
+
* discovers a resolution written after that scan — the workflow re-parks and
|
|
114
|
+
* waits forever even though the completion caller was told it succeeded. The
|
|
115
|
+
* scan is bounded to {@link asyncActivityWorkflowPrefix}, so a takeover pays
|
|
116
|
+
* for one workflow rather than a store-wide sweep.
|
|
107
117
|
*/
|
|
108
|
-
export declare function recoverPendingAsyncActivities(internals: EngineInternals): Promise<void>;
|
|
118
|
+
export declare function recoverPendingAsyncActivities(internals: EngineInternals, workflowId?: string): Promise<void>;
|
|
109
119
|
/**
|
|
110
120
|
* True when a resolution cannot be delivered yet because inline replay has not
|
|
111
121
|
* adopted the workflow generator (the post-recovery window).
|
|
@@ -148,12 +148,13 @@ export function buildAsyncActivityAcknowledgementOperations(pending, outcome) {
|
|
|
148
148
|
export async function registerPendingAsyncActivity(internals, pending) {
|
|
149
149
|
const alreadyRegistered = internals.pendingAsyncActivities.has(pending.token);
|
|
150
150
|
internals.pendingAsyncActivities.set(pending.token, pending);
|
|
151
|
-
await commitFencedEngineWrite(internals, [buildPersistPendingAsyncActivityOperation(pending)], [], () => Error(`Async activity registration for token "${pending.token}" lost its precondition.`));
|
|
151
|
+
await commitFencedEngineWrite(internals, pending.workflowId, [buildPersistPendingAsyncActivityOperation(pending)], [], () => Error(`Async activity registration for token "${pending.token}" lost its precondition.`));
|
|
152
152
|
if (!alreadyRegistered)
|
|
153
153
|
internals.engine.dispatchEvent(new ActivityAsyncPendingEvent(pending.token, pending.operationId, pending.workflowId, pending.activityName, pending.attempt));
|
|
154
154
|
}
|
|
155
|
-
export async function recoverPendingAsyncActivities(internals) {
|
|
156
|
-
|
|
155
|
+
export async function recoverPendingAsyncActivities(internals, workflowId) {
|
|
156
|
+
const scanPrefix = workflowId === void 0 ? ASYNC_ACTIVITY_KEY_PREFIX : asyncActivityWorkflowPrefix(workflowId);
|
|
157
|
+
for await (const [, bytes] of internals.storage.scan(scanPrefix)) {
|
|
157
158
|
const decoded = decode(bytes);
|
|
158
159
|
if (isPersistedAsyncActivity(decoded)) {
|
|
159
160
|
internals.pendingAsyncActivities.set(decoded.token, {
|
|
@@ -1,6 +1,16 @@
|
|
|
1
1
|
import type { BatchOperation } from '../../storage/interface.ts';
|
|
2
2
|
import type { BulkTagResult, ListFilter, SearchAttributeValue, WorkflowState, WorkflowStatus } from '../types.ts';
|
|
3
3
|
import type { EngineInternals } from './internals.ts';
|
|
4
|
+
/**
|
|
5
|
+
* `updateWorkflowState`'s two callers commit very different transitions per
|
|
6
|
+
* ADR 0002: `failWorkflow` is a SELF-transition (this engine finishing its
|
|
7
|
+
* own workflow) while `terminateWorkflow` (cancel/timeout) is an EXTERNAL
|
|
8
|
+
* terminal transition (any engine may commit it, and it rotates the claim
|
|
9
|
+
* epoch under `ownership: 'workflow-lease'`). `category` is a REQUIRED
|
|
10
|
+
* positional parameter rather than an optional options field so a caller
|
|
11
|
+
* cannot silently default into the wrong fencing shape.
|
|
12
|
+
*/
|
|
13
|
+
export type WorkflowStateTransitionCategory = 'self' | 'external-terminal';
|
|
4
14
|
type WorkflowStateUpdateOptions = {
|
|
5
15
|
allowedStatuses?: readonly WorkflowStatus[];
|
|
6
16
|
buildAdditionalOperations?: (previousState: WorkflowState, updatedAt: number) => BatchOperation[];
|
|
@@ -21,7 +31,7 @@ export declare function writeRetainedTerminalSearchAttributes(internals: EngineI
|
|
|
21
31
|
/** Build secondary-index updates for terminal workflow state transitions. */
|
|
22
32
|
export declare function buildTerminalWorkflowIndexOperations(previousState: WorkflowState, nextState: WorkflowState): BatchOperation[];
|
|
23
33
|
/** Apply a serialized workflow-state update and return the previous state metadata. */
|
|
24
|
-
export declare function updateWorkflowState(internals: EngineInternals, workflowId: string, updates: Partial<WorkflowState>, options?: WorkflowStateUpdateOptions): Promise<WorkflowStateUpdateResult | null>;
|
|
34
|
+
export declare function updateWorkflowState(internals: EngineInternals, workflowId: string, updates: Partial<WorkflowState>, category: WorkflowStateTransitionCategory, options?: WorkflowStateUpdateOptions): Promise<WorkflowStateUpdateResult | null>;
|
|
25
35
|
/** Add or remove tags for a single workflow under the serialized state-write lock. */
|
|
26
36
|
export declare function mutateWorkflowTags(internals: EngineInternals, workflowId: string, tags: string[], mode: 'add' | 'remove'): Promise<boolean>;
|
|
27
37
|
/** Add or remove tags for every workflow selected by a scoped bulk filter. */
|
|
@@ -10,7 +10,8 @@ import { assertWorkflowTagCount, coerceStartWorkflowTags } from "../start-workfl
|
|
|
10
10
|
import { buildWorkflowTagIndexOperations, normalizeWorkflowTags } from "../workflow-tags.js";
|
|
11
11
|
import { WorkflowNotFoundError } from "./errors.js";
|
|
12
12
|
import {
|
|
13
|
-
|
|
13
|
+
commitExternalTerminalWorkflowStateOperations,
|
|
14
|
+
commitSelfWorkflowStateOperations,
|
|
14
15
|
runSerializedWorkflowStateWrite
|
|
15
16
|
} from "./storage-io.js";
|
|
16
17
|
import {
|
|
@@ -63,7 +64,7 @@ export function buildTerminalWorkflowIndexOperations(previousState, nextState) {
|
|
|
63
64
|
});
|
|
64
65
|
return operations;
|
|
65
66
|
}
|
|
66
|
-
export async function updateWorkflowState(internals, workflowId, updates, options = {}) {
|
|
67
|
+
export async function updateWorkflowState(internals, workflowId, updates, category, options = {}) {
|
|
67
68
|
return await runSerializedWorkflowStateWrite(internals, workflowId, async () => {
|
|
68
69
|
const bytes = await internals.storage.get(KEYS.workflow(workflowId));
|
|
69
70
|
if (!bytes)
|
|
@@ -76,7 +77,7 @@ export async function updateWorkflowState(internals, workflowId, updates, option
|
|
|
76
77
|
...updates,
|
|
77
78
|
updatedAt
|
|
78
79
|
}, additionalOperations = options.buildAdditionalOperations?.(state, updatedAt) ?? [];
|
|
79
|
-
await
|
|
80
|
+
await (category === "self" ? commitSelfWorkflowStateOperations : commitExternalTerminalWorkflowStateOperations)(internals, state, [
|
|
80
81
|
...buildTerminalWorkflowIndexOperations(state, updated),
|
|
81
82
|
{ type: "put", key: KEYS.workflow(workflowId), value: encode(updated) },
|
|
82
83
|
...buildWorkflowVisibilityIndexTransition(workflowId, state, updated).batchOps,
|
|
@@ -14,6 +14,7 @@ import { commitFencedEngineWrite } from "./fenced-write.js";
|
|
|
14
14
|
import { streamWorkflowStates } from "./listing.js";
|
|
15
15
|
import { decodeScheduleRunMetadata } from "./schedule-run-metadata.js";
|
|
16
16
|
import { createTerminalCleanupTimerId } from "./state-utilities.js";
|
|
17
|
+
import { buildExternalTerminalRotationFragment } from "./storage-io.js";
|
|
17
18
|
import {
|
|
18
19
|
decodeWorkflowState,
|
|
19
20
|
isTerminalWorkflowStatus,
|
|
@@ -132,8 +133,10 @@ function getWorkflowRetentionDeadline(internals, state) {
|
|
|
132
133
|
return state.updatedAt + retentionMs;
|
|
133
134
|
}
|
|
134
135
|
export async function purgeWorkflow(internals, state, cleanupWaiters) {
|
|
135
|
-
const deleteOperations = await collectWorkflowPurgeDeleteOperations(internals, state);
|
|
136
|
-
await commitFencedEngineWrite(internals,
|
|
136
|
+
const deleteOperations = await collectWorkflowPurgeDeleteOperations(internals, state), rotation = await buildExternalTerminalRotationFragment(internals, state.id), operations = [...deleteOperations, ...rotation.operations];
|
|
137
|
+
await commitFencedEngineWrite(internals, null, operations, rotation.conditions, () => {
|
|
138
|
+
return Error(`Purge commit for workflow "${state.id}" lost its precondition.`);
|
|
139
|
+
});
|
|
137
140
|
clearPurgedWorkflowInMemoryState(internals, state.id, cleanupWaiters);
|
|
138
141
|
}
|
|
139
142
|
export async function collectWorkflowPurgeDeleteOperations(internals, state) {
|
|
@@ -31,6 +31,11 @@ import { BULK_OPERATION_BATCH_SIZE } from "./listing.js";
|
|
|
31
31
|
import { createTerminalCleanupTimerId } from "./state-utilities.js";
|
|
32
32
|
import { loadWorkflowState, runSerializedWorkflowStateWrite } from "./storage-io.js";
|
|
33
33
|
import { decodeWorkflowState, isTerminalWorkflowStatus } from "./validation.js";
|
|
34
|
+
import {
|
|
35
|
+
commitWithWorkflowClaimFold,
|
|
36
|
+
prepareWorkflowClaimFold,
|
|
37
|
+
throwWorkflowClaimUnavailable
|
|
38
|
+
} from "./workflow-claim-fold.js";
|
|
34
39
|
import { buildWorkflowConcurrencyStartOperations } from "./workflow-concurrency.js";
|
|
35
40
|
import { buildWorkflowVisibilityIndexTransition } from "./workflow-indexes.js";
|
|
36
41
|
export { purgeInternal, TERMINAL_CLEANUP_DELAY_MS } from "./bulk-operations-purge.js";
|
|
@@ -171,17 +176,24 @@ async function reactivateFailedWorkflowFromCheckpointSerialized(internals, workf
|
|
|
171
176
|
}),
|
|
172
177
|
...concurrencyStartOperations?.operations ?? []
|
|
173
178
|
], conditions = concurrencyStartOperations?.conditions ?? [];
|
|
174
|
-
if (await commitFailedWorkflowReactivation(internals, operations, conditions))
|
|
179
|
+
if (await commitFailedWorkflowReactivation(internals, workflowId, operations, conditions))
|
|
175
180
|
return {
|
|
176
181
|
terminalCleanupTimerId: currentState.terminalCleanupToken === void 0 ? void 0 : createTerminalCleanupTimerId(!1, currentState.terminalCleanupToken)
|
|
177
182
|
};
|
|
178
183
|
}
|
|
179
184
|
throw Error(`Workflow concurrency admission for "${lastConcurrencyStateKey ?? workflowId}" changed too many times while retrying failed workflow "${workflowId}"`);
|
|
180
185
|
}
|
|
181
|
-
async function commitFailedWorkflowReactivation(internals, operations, conditions) {
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
186
|
+
async function commitFailedWorkflowReactivation(internals, workflowId, operations, conditions) {
|
|
187
|
+
const claimFold = await prepareWorkflowClaimFold(internals, workflowId);
|
|
188
|
+
if (claimFold === void 0) {
|
|
189
|
+
if (conditions.length > 0)
|
|
190
|
+
requireStorageCapability(internals.storage, "conditionalBatch", "retry failed workflow");
|
|
191
|
+
return commitFencedEngineWriteAllowingPreconditionFailure(internals, workflowId, operations, conditions);
|
|
192
|
+
}
|
|
193
|
+
const result = await commitWithWorkflowClaimFold(internals, claimFold, operations, conditions, "retry failed workflow claim acquisition");
|
|
194
|
+
if (result.status === "committed")
|
|
195
|
+
return !0;
|
|
196
|
+
return result.claimConflict ? throwWorkflowClaimUnavailable(internals, workflowId) : !1;
|
|
185
197
|
}
|
|
186
198
|
function buildReactivatedWorkflowState(internals, state) {
|
|
187
199
|
const reactivatedState = {
|
|
@@ -154,7 +154,7 @@ export function createUpdateCallbacks(engine) {
|
|
|
154
154
|
async function persistCoordinatedUpdateResponse(engine, workflowId, updateName, updateId, idempotencyKey, value) {
|
|
155
155
|
const internals = getInternals(engine), responseOperations = internals.updateCoordinator.buildResponseOperations(updateId, workflowId, value, void 0, idempotencyKey);
|
|
156
156
|
try {
|
|
157
|
-
await commitFencedEngineWrite(internals, responseOperations, [], () => Error(`Coordinated update response for workflow "${workflowId}" lost its CAS race.`));
|
|
157
|
+
await commitFencedEngineWrite(internals, workflowId, responseOperations, [], () => Error(`Coordinated update response for workflow "${workflowId}" lost its CAS race.`));
|
|
158
158
|
engine.dispatchEvent(new UpdateCompletedEvent(updateId, workflowId, updateName, value));
|
|
159
159
|
broadcastFromInternals(internals, { type: "update:completed", workflowId, updateId }, createBroadcastCallbacks(engine));
|
|
160
160
|
} catch (error) {
|
|
@@ -18,7 +18,8 @@ import {
|
|
|
18
18
|
import { resolveWorkflowTypeTarget } from "./registration.js";
|
|
19
19
|
import { cleanupReviews } from "./reviews.js";
|
|
20
20
|
import {
|
|
21
|
-
|
|
21
|
+
commitExternalTerminalWorkflowStateOperations,
|
|
22
|
+
commitSelfWorkflowStateOperations,
|
|
22
23
|
loadWorkflowState,
|
|
23
24
|
runSerializedWorkflowStateWrite
|
|
24
25
|
} from "./storage-io.js";
|
|
@@ -94,7 +95,8 @@ export function createTerminationCallbacksWith(engine, handleScheduledWorkflowTe
|
|
|
94
95
|
handleScheduledWorkflowTerminal,
|
|
95
96
|
loadWorkflowState: (workflowId) => loadWorkflowState(getInternals(engine), workflowId),
|
|
96
97
|
runSerializedWorkflowStateWrite: (workflowId, writeOperation) => runSerializedWorkflowStateWrite(getInternals(engine), workflowId, writeOperation),
|
|
97
|
-
|
|
98
|
+
commitSelfWorkflowStateOperations: (state, operations, options) => commitSelfWorkflowStateOperations(getInternals(engine), state, operations, options),
|
|
99
|
+
commitExternalTerminalWorkflowStateOperations: (state, operations, options) => commitExternalTerminalWorkflowStateOperations(getInternals(engine), state, operations, options),
|
|
98
100
|
cleanupReviews: (workflowId) => cleanupReviews(getInternals(engine), workflowId)
|
|
99
101
|
};
|
|
100
102
|
}
|
|
@@ -122,7 +122,7 @@ async function commitCheckpoint(internals, workflowId, operation, commit, callba
|
|
|
122
122
|
if (pendingSideEffects !== void 0)
|
|
123
123
|
commit.operations.push(...pendingSideEffects.operations);
|
|
124
124
|
const storageSupportsConditionalBatch = internals.storage.capabilities().conditionalBatch, sideEffectConditions = checkpointSideEffectConditions(pendingSideEffects, storageSupportsConditionalBatch), conditions = buildCheckpointCommitConditions(workflowId, commit, storageSupportsConditionalBatch, sideEffectConditions);
|
|
125
|
-
await commitFencedEngineWrite(internals, commit.operations, conditions, () => {
|
|
125
|
+
await commitFencedEngineWrite(internals, workflowId, commit.operations, conditions, () => {
|
|
126
126
|
return Error(`Checkpoint commit for workflow "${workflowId}" lost its CAS race against a newer checkpoint.`);
|
|
127
127
|
});
|
|
128
128
|
if (pendingSideEffects !== void 0)
|
|
@@ -5,6 +5,7 @@ import {
|
|
|
5
5
|
registerChildCancellationHandler
|
|
6
6
|
} from "./child-workflow-cancellation.js";
|
|
7
7
|
import { WorkflowAlreadyExistsError } from "./errors.js";
|
|
8
|
+
import { getGeneratorOwnedWorkflowResultPromise } from "./handle-result.js";
|
|
8
9
|
import { encodedValuesEqual } from "./state-utilities.js";
|
|
9
10
|
export async function processChildWorkflowOperation(internals, workflowId, operation, callbacks) {
|
|
10
11
|
return callbacks.runOperationWithResult(workflowId, operation, () => executeChildWorkflow(internals, workflowId, operation, assertChildWorkflowNestingDepth(internals, workflowId), callbacks));
|
|
@@ -83,7 +84,7 @@ export async function executeChildWorkflow(internals, workflowId, operation, cur
|
|
|
83
84
|
registerChildCancellationHandler(internals, workflowId, childHandle.id, callbacks);
|
|
84
85
|
return createChildWorkflowHandleReference(childHandle.id);
|
|
85
86
|
}
|
|
86
|
-
return childHandle.
|
|
87
|
+
return getGeneratorOwnedWorkflowResultPromise(internals, childHandle.id, workflowId);
|
|
87
88
|
}, composedInterceptor = callbacks.getComposedWorkflowInterceptor();
|
|
88
89
|
if (!composedInterceptor)
|
|
89
90
|
return executeChild();
|
|
@@ -35,7 +35,7 @@ export async function listCompletedReviewsFromStorage(storage, filter) {
|
|
|
35
35
|
}
|
|
36
36
|
export async function persistCompletedReviewRecord(internals, reviewKey, reviewData, decisionResult) {
|
|
37
37
|
const completedReview = toCompletedReviewEntry(reviewData, decisionResult);
|
|
38
|
-
await commitFencedEngineWrite(internals, [
|
|
38
|
+
await commitFencedEngineWrite(internals, reviewData.workflowId, [
|
|
39
39
|
{
|
|
40
40
|
type: "put",
|
|
41
41
|
key: completedReviewStorageKey(reviewData.workflowId, reviewData.reviewId),
|
|
@@ -1,14 +1,56 @@
|
|
|
1
1
|
import type { EngineInternals } from './internals.ts';
|
|
2
|
+
import { type WakeOwnershipDecision } from './wake-ownership-guard.ts';
|
|
2
3
|
/**
|
|
3
4
|
* Re-drive every in-process `ctx.waitUntil` waiter registered for a workflow.
|
|
4
|
-
* Called after an inline `onUpdate` handler runs (`tryInlineUpdateHandler`)
|
|
5
|
-
* since the handler may have mutated the workflow-local state a predicate
|
|
6
|
-
*
|
|
5
|
+
* Called after an inline `onUpdate` handler runs (`tryInlineUpdateHandler`) —
|
|
6
|
+
* since the handler may have mutated the workflow-local state a predicate
|
|
7
|
+
* reads. Wakes the parked `processWaitConditionOperation`, which re-evaluates
|
|
7
8
|
* its predicate against the (possibly mutated) state.
|
|
8
9
|
*
|
|
9
10
|
* Weft signals are pull-only (`ctx.waitForSignal`) and run no state-mutating
|
|
10
11
|
* handler, so signal delivery is intentionally NOT a re-drive trigger — `onUpdate`
|
|
11
12
|
* is the push path. Lives in its own module so `updates.ts` can call it without
|
|
12
13
|
* an import cycle through `operations-coordination.ts`.
|
|
14
|
+
*
|
|
15
|
+
* **`wakeOwnershipCheck` under `ownership: 'workflow-lease'`.** This is a
|
|
16
|
+
* claim-requiring wake site (ADR 0002's `wait-condition` kind): resolving
|
|
17
|
+
* `conditionWaiters` drives a parked generator turn. This function's only
|
|
18
|
+
* remaining callers — `updates.ts` and `pending-updates.ts` — invoke it from
|
|
19
|
+
* an `onUpdate` handler with no associated durable timer to protect, so they
|
|
20
|
+
* call it synchronously with no `await` and must stay that way: this
|
|
21
|
+
* function's signature stays `void`, never `Promise<void>`, so neither
|
|
22
|
+
* becomes a floating-promise lint violation. Under `'none'`/`'lease'`
|
|
23
|
+
* (`workflowClaimRegistry === null`) the resolver is looked up and called
|
|
24
|
+
* synchronously — byte-identical to before this check existed. Under
|
|
25
|
+
* `'workflow-lease'` the ownership re-read runs as a fire-and-forget async
|
|
26
|
+
* check, and the resolver is looked up fresh (never captured before the
|
|
27
|
+
* await) once that check settles, so a newer waiter registered in the
|
|
28
|
+
* meantime is never resolved by a stale reference.
|
|
29
|
+
*
|
|
30
|
+
* The durable-timer-driven wake (a `wait-condition` deadline timer firing)
|
|
31
|
+
* does NOT use this function — see {@link notifyConditionWaitersForTimerFire}
|
|
32
|
+
* below. That caller feeds directly into the `Scheduler`'s "was this fire
|
|
33
|
+
* processed" decision, so it cannot be fire-and-forget the way an
|
|
34
|
+
* `onUpdate`-driven poke safely can.
|
|
13
35
|
*/
|
|
14
36
|
export declare function notifyConditionWaiters(internals: EngineInternals, workflowId: string): void;
|
|
37
|
+
/**
|
|
38
|
+
* Timer-driven counterpart to {@link notifyConditionWaiters}, used only by
|
|
39
|
+
* `operations-time.ts`'s `resolveConditionTimer` when a durable
|
|
40
|
+
* `ctx.waitUntil()` deadline timer fires under `ownership: 'workflow-lease'`.
|
|
41
|
+
*
|
|
42
|
+
* Unlike the `onUpdate`-driven path above, this caller feeds directly into
|
|
43
|
+
* the `Scheduler`'s `#processSelectedTimer`: once the returned promise
|
|
44
|
+
* settles, the caller decides whether the Scheduler may treat this fire as
|
|
45
|
+
* "processed" and durably delete the timer key. A fire-and-forget check here
|
|
46
|
+
* (mirroring `notifyConditionWaiters`'s `'workflow-lease'` branch) would let
|
|
47
|
+
* the Scheduler delete the timer before the ownership decision — and, on a
|
|
48
|
+
* `'proceed'`, the wake dispatch itself — has actually happened, stranding a
|
|
49
|
+
* parked workflow with no deterministic timeout record if the process exits
|
|
50
|
+
* or disposes in that window. So this returns the raw {@link
|
|
51
|
+
* WakeOwnershipDecision} instead of swallowing it: the caller uses a
|
|
52
|
+
* `'discard'` to decide retain-vs-collect via
|
|
53
|
+
* `sleep-timer-acknowledgements.ts`'s `resolveDiscardedTimerDisposition`,
|
|
54
|
+
* the same disposal policy the `sleep` wake kind shares this hazard with.
|
|
55
|
+
*/
|
|
56
|
+
export declare function notifyConditionWaitersForTimerFire(internals: EngineInternals, workflowId: string): Promise<WakeOwnershipDecision>;
|
|
@@ -1,5 +1,14 @@
|
|
|
1
|
+
import { confirmWakeOwnership } from "./wake-ownership-guard.js";
|
|
1
2
|
export function notifyConditionWaiters(internals, workflowId) {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
if (internals.workflowClaimRegistry === null) {
|
|
4
|
+
internals.conditionWaiters.get(workflowId)?.();
|
|
5
|
+
return;
|
|
6
|
+
}
|
|
7
|
+
confirmWakeOwnership(internals, workflowId, "wait-condition").then((decision) => decision === "proceed" ? internals.conditionWaiters.get(workflowId)?.() : void 0);
|
|
8
|
+
}
|
|
9
|
+
export async function notifyConditionWaitersForTimerFire(internals, workflowId) {
|
|
10
|
+
const decision = await confirmWakeOwnership(internals, workflowId, "wait-condition");
|
|
11
|
+
if (decision === "proceed")
|
|
12
|
+
internals.conditionWaiters.get(workflowId)?.();
|
|
13
|
+
return decision;
|
|
5
14
|
}
|
|
@@ -8,6 +8,11 @@ function settleSleepResolverReadyWaitersForTesting(internals) {
|
|
|
8
8
|
notifyReady();
|
|
9
9
|
internals.sleepResolverReadyWaitersForTesting?.clear();
|
|
10
10
|
}
|
|
11
|
+
function clearPendingResultPollTimers(internals) {
|
|
12
|
+
for (const timer of internals.pendingResultPollTimers)
|
|
13
|
+
clearTimeout(timer);
|
|
14
|
+
internals.pendingResultPollTimers.clear();
|
|
15
|
+
}
|
|
11
16
|
export function disposeEngine(internals) {
|
|
12
17
|
internals.disposed = !0;
|
|
13
18
|
internals.alertManager?.[Symbol.dispose]();
|
|
@@ -52,6 +57,7 @@ export function disposeEngine(internals) {
|
|
|
52
57
|
for (const controller of internals.pendingWebhooks)
|
|
53
58
|
controller.abort();
|
|
54
59
|
internals.pendingWebhooks.clear();
|
|
60
|
+
clearPendingResultPollTimers(internals);
|
|
55
61
|
internals.sleepResolvers.clear();
|
|
56
62
|
internals.sleepResolversByWorkflow.clear();
|
|
57
63
|
settleSleepResolverReadyWaitersForTesting(internals);
|
|
@@ -46,17 +46,25 @@ export interface ResolvedOptions {
|
|
|
46
46
|
/** Heartbeat interval (ms) for the second-instance detector when enabled. */
|
|
47
47
|
secondInstanceHeartbeatIntervalMs: number;
|
|
48
48
|
/**
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
* lease
|
|
49
|
+
* Ownership posture. `'none'` recovers immediately (infra owns mutual
|
|
50
|
+
* exclusion); `'lease'` acquires a single store-wide storage lease before
|
|
51
|
+
* recovery; `'workflow-lease'` fences execution per workflow instead
|
|
52
|
+
* (ADR 0002), letting multiple engines share one durable store. The lease
|
|
53
|
+
* tuning fields below are only meaningful when this is `'lease'`; the
|
|
54
|
+
* workflow-claim tuning fields are only meaningful when this is
|
|
55
|
+
* `'workflow-lease'`.
|
|
52
56
|
*/
|
|
53
|
-
ownershipMode: 'none' | 'lease';
|
|
57
|
+
ownershipMode: 'none' | 'lease' | 'workflow-lease';
|
|
54
58
|
/** Lease time-to-live (ms) when `ownershipMode` is `'lease'`. */
|
|
55
59
|
leaseTtlMs: number;
|
|
56
60
|
/** Lease renewal interval (ms) when `ownershipMode` is `'lease'`. */
|
|
57
61
|
leaseRenewIntervalMs: number;
|
|
58
62
|
/** Boot-time lease acquisition wait window (ms) when `ownershipMode` is `'lease'`. */
|
|
59
63
|
leaseWaitTimeoutMs: number;
|
|
64
|
+
/** Per-workflow claim time-to-live (ms) when `ownershipMode` is `'workflow-lease'`. */
|
|
65
|
+
workflowClaimTtlMs: number;
|
|
66
|
+
/** Per-workflow claim renewal interval (ms) when `ownershipMode` is `'workflow-lease'`. */
|
|
67
|
+
workflowClaimRenewIntervalMs: number;
|
|
60
68
|
getNow: () => number;
|
|
61
69
|
/**
|
|
62
70
|
* Re-provides the non-serialized per-run `services` value on recovery; `null`
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Step-2 epoch fencing for `ownership: 'lease'` (issue #470)
|
|
2
|
+
* Step-2 epoch fencing for `ownership: 'lease'` (issue #470), extended by ADR
|
|
3
|
+
* 0002 to per-workflow fencing under `ownership: 'workflow-lease'`.
|
|
3
4
|
*
|
|
4
5
|
* Step 1 made a rolling deploy a clean lease handoff but added no correctness
|
|
5
6
|
* backstop: a deposed zombie instance (GC pause, partition) that emerged after
|
|
@@ -15,6 +16,20 @@
|
|
|
15
16
|
* from silently spinning on swallowed write losses, surface the deposition to the
|
|
16
17
|
* operator, and release resources. They are not what makes the system safe.
|
|
17
18
|
*
|
|
19
|
+
* ADR 0002 adds a SECOND, independent fencing token: `wf-owner-epoch:<workflowId>`.
|
|
20
|
+
* Every caller of {@link commitFencedEngineWrite} / {@link
|
|
21
|
+
* commitFencedEngineWriteAllowingPreconditionFailure} now states, via the
|
|
22
|
+
* required `workflowId` parameter, whether the write is scoped to one
|
|
23
|
+
* workflow's execution (pass its id) or is engine-scoped/cross-workflow (pass
|
|
24
|
+
* `null`) — an optional parameter would let a call site silently forget the
|
|
25
|
+
* fence, which is exactly the correctness hole this mechanism exists to close.
|
|
26
|
+
* Under `ownership: 'workflow-lease'` with a non-null `workflowId`, the write
|
|
27
|
+
* is fenced on THAT workflow's claim epoch instead of the global lease epoch;
|
|
28
|
+
* losing that fence deposes only that one workflow (warn + throw), never the
|
|
29
|
+
* whole engine. Under `'lease'` and `'none'`, and under `'workflow-lease'`
|
|
30
|
+
* with `workflowId: null`, behavior is byte-for-byte unchanged from Step 2 —
|
|
31
|
+
* `workflowId` is only consulted in the one new branch.
|
|
32
|
+
*
|
|
18
33
|
* This module is allow-listed for import only from `src/core/engine/**`.
|
|
19
34
|
*/
|
|
20
35
|
import type { BatchOperation, ConditionalBatchCondition } from '../../storage/interface.ts';
|
|
@@ -31,37 +46,53 @@ import type { EngineInternals } from './internals.ts';
|
|
|
31
46
|
* `pauseSchedule`, `resumeSchedule`, `cancelSchedule`, and `updateSchedule`) would
|
|
32
47
|
* reach `resolveFenceEpochOrHalt` with no held epoch and be misreported as a
|
|
33
48
|
* deposition (warn + teardown) rather than the true "lease not held yet"
|
|
34
|
-
* condition. A no-op under `ownership: 'none'
|
|
49
|
+
* condition. A no-op under `ownership: 'none'` and under `ownership:
|
|
50
|
+
* 'workflow-lease'` — the analogous "claim not held yet" guard for a
|
|
51
|
+
* workflow-scoped write is a later stage's concern (folding `acquire()` into
|
|
52
|
+
* the enabling write), not this global-lease-only assertion.
|
|
35
53
|
*/
|
|
36
54
|
export declare function assertLeaseHeldForEngineWork(internals: EngineInternals): void;
|
|
37
55
|
/**
|
|
38
|
-
* Commit an engine-generator-owned durable write, fenced on the
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
56
|
+
* Commit an engine-generator-owned durable write, fenced on the applicable
|
|
57
|
+
* epoch: the global lease epoch under `ownership: 'lease'`, this workflow's
|
|
58
|
+
* claim epoch under `ownership: 'workflow-lease'` when `workflowId` is
|
|
59
|
+
* non-null, or — under `'none'` and every other `workflowId`/mode
|
|
60
|
+
* combination — byte-for-byte unfenced. A lost CAS race throws the
|
|
61
|
+
* caller-supplied `onLostRace` error so existing retry semantics apply; a
|
|
62
|
+
* deposition halts (the engine under global `'lease'`, or just this one
|
|
63
|
+
* workflow under `'workflow-lease'`) and throws {@link EngineDeposedError}.
|
|
64
|
+
* The helper owns the batch-vs-conditionalBatch decision — pass plain
|
|
65
|
+
* operations plus whatever base conditions you already need.
|
|
44
66
|
*
|
|
45
|
-
* @param internals - the engine internals (ownership mode, lease manager,
|
|
67
|
+
* @param internals - the engine internals (ownership mode, lease manager,
|
|
68
|
+
* workflow claim registry, deposed flag)
|
|
69
|
+
* @param workflowId - the workflow this write is scoped to, fenced on that
|
|
70
|
+
* workflow's claim epoch under `ownership: 'workflow-lease'`; or `null` for
|
|
71
|
+
* an engine-scoped/cross-workflow write, which is never fenced on a
|
|
72
|
+
* per-workflow claim regardless of ownership mode. Required — there is no
|
|
73
|
+
* default — so every call site states its scope explicitly.
|
|
46
74
|
* @param operations - the durable operations to commit atomically
|
|
47
75
|
* @param baseConditions - CAS conditions the caller already requires (may be empty)
|
|
48
76
|
* @param onLostRace - builds the error thrown on a same-epoch lost CAS race
|
|
49
77
|
*/
|
|
50
|
-
export declare function commitFencedEngineWrite(internals: EngineInternals, operations: BatchOperation[], baseConditions: ConditionalBatchCondition[], onLostRace: () => Error): Promise<void>;
|
|
78
|
+
export declare function commitFencedEngineWrite(internals: EngineInternals, workflowId: string | null, operations: BatchOperation[], baseConditions: ConditionalBatchCondition[], onLostRace: () => Error): Promise<void>;
|
|
51
79
|
/**
|
|
52
80
|
* Like {@link commitFencedEngineWrite}, but the caller treats a base-precondition
|
|
53
81
|
* failure as a legitimate outcome rather than an error — used by the idempotent
|
|
54
82
|
* start path, where a `false` means a concurrent same-key caller already wrote the
|
|
55
83
|
* record (resolve to the existing run) rather than "retry". Returns `true` when the
|
|
56
84
|
* batch committed and `false` when a base condition failed. Deposition is still a
|
|
57
|
-
* hard halt: if the epoch condition is the one that failed, this drives
|
|
58
|
-
*
|
|
59
|
-
*
|
|
60
|
-
* and silently
|
|
85
|
+
* hard halt: if the epoch condition is the one that failed, this drives the
|
|
86
|
+
* applicable deposition path and throws {@link EngineDeposedError} — a deposed
|
|
87
|
+
* engine (or a deposed single workflow, under `workflow-lease`) must never report a
|
|
88
|
+
* precondition-failure the caller would read as "already exists" and silently
|
|
89
|
+
* move on.
|
|
61
90
|
*
|
|
62
91
|
* @param internals - the engine internals
|
|
92
|
+
* @param workflowId - see {@link commitFencedEngineWrite} — the workflow this
|
|
93
|
+
* write is scoped to, or `null` for an engine-scoped write
|
|
63
94
|
* @param operations - the durable operations to commit atomically
|
|
64
95
|
* @param baseConditions - the caller's required CAS conditions (non-empty in
|
|
65
96
|
* practice — this path is for conditional starts, which always carry one)
|
|
66
97
|
*/
|
|
67
|
-
export declare function commitFencedEngineWriteAllowingPreconditionFailure(internals: EngineInternals, operations: BatchOperation[], baseConditions: ConditionalBatchCondition[]): Promise<boolean>;
|
|
98
|
+
export declare function commitFencedEngineWriteAllowingPreconditionFailure(internals: EngineInternals, workflowId: string | null, operations: BatchOperation[], baseConditions: ConditionalBatchCondition[]): Promise<boolean>;
|