@lostgradient/weft 0.24.0 → 0.24.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/cli/conformance.js +5 -2
- package/dist/cli/generated/operation-catalog.snapshot.json +95 -1
- package/dist/cli/generated/operation-client.generated.d.ts +28 -2
- package/dist/cli/generated/operation-client.generated.js +2 -0
- package/dist/cli/parse-schedule-arguments.js +15 -3
- package/dist/cli/schedule.js +2 -1
- package/dist/cli/types.d.ts +2 -1
- package/dist/client/http-client-requests.js +6 -9
- package/dist/client/start-body.js +1 -0
- package/dist/core/catalog/index.d.ts +2 -2
- package/dist/core/catalog/index.js +9 -2
- package/dist/core/catalog/reference-counts.d.ts +29 -14
- package/dist/core/catalog/removal.d.ts +63 -10
- package/dist/core/catalog/removal.js +18 -2
- package/dist/core/catalog/storage-io.d.ts +23 -0
- package/dist/core/catalog/storage-io.js +1 -1
- package/dist/core/engine/activity-resolution.js +17 -17
- package/dist/core/engine/bulk-operations-purge.js +14 -20
- package/dist/core/engine/bulk-operations-retry.d.ts +9 -0
- package/dist/core/engine/bulk-operations-retry.js +177 -0
- package/dist/core/engine/bulk-operations-shared.d.ts +18 -0
- package/dist/core/engine/bulk-operations-shared.js +14 -0
- package/dist/core/engine/bulk-operations.js +8 -167
- package/dist/core/engine/callback-creators-bundles.js +4 -2
- package/dist/core/engine/callback-creators-core.js +9 -1
- package/dist/core/engine/callback-creators-schedule.js +2 -2
- package/dist/core/engine/catalog-readiness.js +2 -0
- package/dist/core/engine/catalog-removal.d.ts +107 -3
- package/dist/core/engine/catalog-removal.js +66 -13
- package/dist/core/engine/catalog-tombstone-recovery.d.ts +56 -0
- package/dist/core/engine/catalog-tombstone-recovery.js +39 -0
- package/dist/core/engine/constraints.js +5 -1
- package/dist/core/engine/construction.d.ts +1 -1
- package/dist/core/engine/decode-revision.d.ts +32 -0
- package/dist/core/engine/decode-revision.js +9 -0
- package/dist/core/engine/disposal.d.ts +6 -2
- package/dist/core/engine/disposal.js +24 -10
- package/dist/core/engine/dynamic-source-errors.d.ts +81 -0
- package/dist/core/engine/dynamic-source-errors.js +25 -0
- package/dist/core/engine/dynamic-source-execution.d.ts +173 -0
- package/dist/core/engine/dynamic-source-execution.js +108 -0
- package/dist/core/engine/engine-internal-types.d.ts +16 -0
- package/dist/core/engine/engine-workflows-namespace.d.ts +10 -0
- package/dist/core/engine/engine-workflows-namespace.js +3 -1
- package/dist/core/engine/index.d.ts +18 -10
- package/dist/core/engine/index.js +21 -10
- package/dist/core/engine/inline-launch-queue.js +1 -1
- package/dist/core/engine/internals.d.ts +26 -30
- package/dist/core/engine/lifecycle/checkpoint-launch.d.ts +29 -0
- package/dist/core/engine/lifecycle/checkpoint-launch.js +75 -0
- package/dist/core/engine/lifecycle/delayed-start-registration.d.ts +30 -0
- package/dist/core/engine/lifecycle/delayed-start-registration.js +11 -0
- package/dist/core/engine/lifecycle/fork-helpers.d.ts +33 -1
- package/dist/core/engine/lifecycle/fork-helpers.js +3 -1
- package/dist/core/engine/lifecycle/recovery-revision-groups.d.ts +75 -0
- package/dist/core/engine/lifecycle/recovery-revision-groups.js +64 -0
- package/dist/core/engine/lifecycle/resume-generation-guard.d.ts +58 -0
- package/dist/core/engine/lifecycle/resume-generation-guard.js +11 -0
- package/dist/core/engine/lifecycle/resume.js +18 -9
- package/dist/core/engine/lifecycle/shared.d.ts +43 -0
- package/dist/core/engine/lifecycle/standalone-claim-acquire.d.ts +12 -0
- package/dist/core/engine/lifecycle/standalone-claim-acquire.js +6 -0
- package/dist/core/engine/lifecycle/start-commit-errors.d.ts +12 -0
- package/dist/core/engine/lifecycle/start-commit-errors.js +6 -0
- package/dist/core/engine/lifecycle/start-commit.d.ts +22 -7
- package/dist/core/engine/lifecycle/start-commit.js +49 -15
- package/dist/core/engine/lifecycle/start-exec.d.ts +18 -2
- package/dist/core/engine/lifecycle/start-exec.js +7 -5
- package/dist/core/engine/lifecycle/start-or-signal-create.js +1 -3
- package/dist/core/engine/lifecycle/start-or-signal.js +1 -1
- package/dist/core/engine/lifecycle/start-precondition-attribution.d.ts +49 -0
- package/dist/core/engine/lifecycle/start-precondition-attribution.js +13 -0
- package/dist/core/engine/lifecycle/start-revision-resolution.d.ts +34 -0
- package/dist/core/engine/lifecycle/start-revision-resolution.js +11 -0
- package/dist/core/engine/lifecycle/start-state.d.ts +16 -0
- package/dist/core/engine/lifecycle/start-state.js +60 -0
- package/dist/core/engine/lifecycle/start-terminal-conflict-purge.d.ts +66 -3
- package/dist/core/engine/lifecycle/start-terminal-conflict-purge.js +13 -3
- package/dist/core/engine/lifecycle/start.d.ts +16 -6
- package/dist/core/engine/lifecycle/start.js +38 -68
- package/dist/core/engine/lifecycle/transition.d.ts +3 -4
- package/dist/core/engine/lifecycle/transition.js +30 -80
- package/dist/core/engine/lifecycle.d.ts +4 -2
- package/dist/core/engine/lifecycle.js +6 -6
- package/dist/core/engine/listing.js +15 -1
- package/dist/core/engine/memo-durable-activity.js +4 -1
- package/dist/core/engine/nonterminal-revision-count.d.ts +25 -0
- package/dist/core/engine/nonterminal-revision-count.js +16 -0
- package/dist/core/engine/operations-time.d.ts +5 -23
- package/dist/core/engine/operations-time.js +13 -11
- package/dist/core/engine/ownership-bootstrap.d.ts +1 -1
- package/dist/core/engine/pinned-schedule-revision-count.d.ts +26 -0
- package/dist/core/engine/pinned-schedule-revision-count.js +19 -0
- package/dist/core/engine/pinned-schedule-revision.d.ts +76 -0
- package/dist/core/engine/pinned-schedule-revision.js +53 -0
- package/dist/core/engine/registration.d.ts +30 -0
- package/dist/core/engine/registration.js +17 -2
- package/dist/core/engine/retention.js +9 -2
- package/dist/core/engine/revision-errors.d.ts +71 -0
- package/dist/core/engine/revision-errors.js +13 -0
- package/dist/core/engine/schedule-revision-fire.d.ts +17 -0
- package/dist/core/engine/schedule-revision-fire.js +3 -0
- package/dist/core/engine/schedule-run.js +2 -1
- package/dist/core/engine/schedules.d.ts +2 -1
- package/dist/core/engine/schedules.js +28 -9
- package/dist/core/engine/source-diagnostics.d.ts +76 -0
- package/dist/core/engine/source-diagnostics.js +105 -0
- package/dist/core/engine/source-registration.js +4 -2
- package/dist/core/engine/source-resolution.d.ts +25 -9
- package/dist/core/engine/source-resolution.js +49 -14
- package/dist/core/engine/source-runtime-state.d.ts +92 -0
- package/dist/core/engine/source-runtime-state.js +36 -0
- package/dist/core/engine/storage-io.d.ts +19 -2
- package/dist/core/engine/storage-io.js +21 -2
- package/dist/core/engine/termination/finalizer-registration.d.ts +18 -0
- package/dist/core/engine/termination/finalizer-registration.js +21 -0
- package/dist/core/engine/termination/finalizer.d.ts +16 -19
- package/dist/core/engine/termination/finalizer.js +3 -3
- package/dist/core/engine/time-operation-callbacks.d.ts +32 -0
- package/dist/core/engine/time-operation-callbacks.js +0 -0
- package/dist/core/engine/validation/schedule-cadence.d.ts +16 -0
- package/dist/core/engine/validation/schedule-cadence.js +20 -0
- package/dist/core/engine/validation/schedule-options.d.ts +4 -1
- package/dist/core/engine/validation/schedule-options.js +14 -0
- package/dist/core/engine/validation/schedule-revision.d.ts +24 -0
- package/dist/core/engine/validation/schedule-revision.js +21 -0
- package/dist/core/engine/validation/schedule-warnings.d.ts +11 -0
- package/dist/core/engine/validation/schedule-warnings.js +5 -0
- package/dist/core/engine/validation/schedule.d.ts +4 -3
- package/dist/core/engine/validation/schedule.js +15 -26
- package/dist/core/engine/validation.js +3 -0
- package/dist/core/engine/workflow-claim-reclaim-target.d.ts +1 -1
- package/dist/core/engine/workflow-claim-reclaim-target.js +16 -2
- package/dist/core/engine/workflow-retention-deadline.d.ts +16 -0
- package/dist/core/engine/workflow-retention-deadline.js +33 -0
- package/dist/core/events/event-map.d.ts +5 -0
- package/dist/core/events/index.d.ts +1 -0
- package/dist/core/events/index.js +1 -0
- package/dist/core/events/workflow-source-events.d.ts +102 -0
- package/dist/core/events/workflow-source-events.js +57 -0
- package/dist/core/execution-strategy.d.ts +7 -0
- package/dist/core/inline-execution-strategy.context-options.d.ts +1 -1
- package/dist/core/inline-execution-strategy.js +1 -1
- package/dist/core/types/checkpoint.d.ts +14 -0
- package/dist/core/types/schedules.d.ts +52 -1
- package/dist/core/types/state.d.ts +16 -0
- package/dist/core/weft-error.d.ts +1 -1
- package/dist/core/weft-error.js +4 -1
- package/dist/core/worker-execution-strategy.d.ts +1 -0
- package/dist/core/worker-execution-strategy.js +13 -1
- package/dist/core/worker-fault-handling.d.ts +10 -0
- package/dist/core/worker-fault-handling.js +1 -0
- package/dist/core/worker-inbound-message.d.ts +1 -0
- package/dist/core/worker-inbound-message.js +1 -0
- package/dist/core/worker-protocol-guard.js +4 -0
- package/dist/core/worker-protocol.d.ts +7 -0
- package/dist/core/worker-protocol.js +2 -1
- package/dist/core/worker-turn-watchdog.d.ts +8 -1
- package/dist/core/worker-turn-watchdog.js +3 -2
- package/dist/diagnostics/format.js +5 -0
- package/dist/diagnostics/types.d.ts +24 -0
- package/dist/diagnostics/version-check.js +8 -2
- package/dist/http.js +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.js +7 -0
- package/dist/indexeddb.js +1 -1
- package/dist/json-schema.js +3 -3
- package/dist/server/fault-to-json-rpc.js +2 -1
- package/dist/server/index.d.ts +11 -0
- package/dist/server/operation-fault.d.ts +2 -0
- package/dist/server/operation-fault.js +2 -1
- package/dist/server/operations/create-schedule.d.ts +2 -0
- package/dist/server/operations/create-schedule.js +10 -6
- package/dist/server/operations/get-catalog-diagnostics.d.ts +28 -0
- package/dist/server/operations/get-catalog-diagnostics.js +10 -2
- package/dist/server/operations/get-task-detail-schema.d.ts +8 -0
- package/dist/server/operations/get-task-detail-schema.js +1 -0
- package/dist/server/operations/get-task-detail.d.ts +8 -0
- package/dist/server/operations/get-task-detail.js +1 -0
- package/dist/server/operations/get-task-diagnostics.d.ts +4 -4
- package/dist/server/operations/preload-workflow-revision.d.ts +28 -0
- package/dist/server/operations/preload-workflow-revision.js +60 -0
- package/dist/server/operations/schedule-faults.d.ts +1 -0
- package/dist/server/operations/schedule-faults.js +32 -4
- package/dist/server/operations/schedule-rest-body.d.ts +1 -0
- package/dist/server/operations/schedule-rest-body.js +2 -1
- package/dist/server/operations/static-registrations.js +6 -0
- package/dist/server/operations/update-schedule.d.ts +2 -0
- package/dist/server/operations/update-schedule.js +4 -2
- package/dist/server/operations/workflow-catalog-operation-helpers.js +27 -0
- package/dist/server/runtime/task-dispatch-envelope.d.ts +14 -0
- package/dist/server/runtime/task-dispatch-envelope.js +32 -0
- package/dist/server/runtime/task-dispatch-revision.d.ts +29 -0
- package/dist/server/runtime/task-dispatch-revision.js +13 -0
- package/dist/server/runtime/task-dispatch.js +12 -37
- package/dist/server/runtime/task-ledger-recovery.js +1 -1
- package/dist/server/runtime/task-ledger-runtime.d.ts +14 -2
- package/dist/server/runtime/task-ledger-runtime.js +5 -3
- package/dist/server/runtime/task-polling.js +9 -3
- package/dist/server/runtime/task-reconciliation.js +2 -1
- package/dist/server/runtime/websocket-worker.js +9 -0
- package/dist/server/task-ledger-codec.d.ts +9 -0
- package/dist/server/task-ledger-codec.js +4 -1
- package/dist/server/task-ledger-transition-helpers.js +1 -0
- package/dist/server/task-ledger-types.d.ts +9 -0
- package/dist/server/task-ledger.d.ts +1 -1
- package/dist/server/task-ledger.js +2 -1
- package/dist/server/task-queue-types.d.ts +2 -0
- package/dist/storage/bun-sql.js +3 -1
- package/dist/storage/catalog-keys.d.ts +19 -0
- package/dist/storage/catalog-keys.js +3 -1
- package/dist/storage/compressed-storage.js +1 -1
- package/dist/storage/index.d.ts +3 -1
- package/dist/storage/interface.d.ts +2 -0
- package/dist/storage/interface.js +1 -1
- package/dist/storage/key-prefixes.d.ts +1 -1
- package/dist/storage/key-prefixes.js +1 -0
- package/dist/storage/lmdb.d.ts +30 -2
- package/dist/storage/lmdb.js +1 -1
- package/dist/storage/memory.js +1 -1
- package/dist/storage/neon.js +2 -2
- package/dist/storage/node-sqlite.js +3 -1
- package/dist/storage/postgres.js +2 -2
- package/dist/storage/resolve.js +1 -1
- package/dist/storage/scoped-storage.js +1 -1
- package/dist/storage/testing.js +1 -1
- package/dist/storage/turso.js +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/web-extension.js +1 -1
- package/dist/worker/protocol-messages.d.ts +17 -0
- package/dist/worker/protocol-schemas.d.ts +32 -0
- package/dist/worker/protocol-schemas.js +7 -3
- package/dist/worker/protocol-task-result.js +35 -3
- package/dist/worker/protocol.js +1 -1
- package/dist/worker/registry/types.d.ts +2 -0
- package/dist/worker/registry.d.ts +1 -1
- package/dist/worker/registry.js +3 -1
- package/dist/workers/workflow-runner.d.ts +9 -0
- package/dist/workers/workflow-runner.js +9 -2
- package/dist/workers/workflow-worker-entry.js +9 -4
- package/package.json +2 -2
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `resumeWorkflowFromStorage()`'s generation-identity re-check (WFT-19
|
|
3
|
+
* review round 7, chatgpt-codex-connector + stevekinney) — split out of
|
|
4
|
+
* `resume.ts`, which has no headroom under the repository's 500-line
|
|
5
|
+
* implementation-file ceiling for this doc-heavy addition.
|
|
6
|
+
*
|
|
7
|
+
* @module core/engine/lifecycle/resume-generation-guard
|
|
8
|
+
*/
|
|
9
|
+
import type { WorkflowState } from '../../types.ts';
|
|
10
|
+
/**
|
|
11
|
+
* The generation a resume's `registration`/`resumeCheckpoint`/`resolvedRevision`
|
|
12
|
+
* were all resolved against — captured from the SAME `WorkflowState` decoded
|
|
13
|
+
* at the very top of `resumeWorkflowFromStorage()`, before
|
|
14
|
+
* `acquireStandaloneClaimBeforeResume`, the checkpoint load, and registration
|
|
15
|
+
* resolution all ran.
|
|
16
|
+
*
|
|
17
|
+
* `performSerializedResume()` re-reads workflow state fresh inside its
|
|
18
|
+
* serialized section but (before this guard existed) only re-checked
|
|
19
|
+
* `status` — never identity. An `onTerminalConflict: 'start-new'`
|
|
20
|
+
* replacement landing on this SAME `workflowId` between the top-of-function
|
|
21
|
+
* read and the serialized section's fresh read (e.g. a concurrent
|
|
22
|
+
* cancel/timeout terminalizes this run, then a fresh `start()` call replaces
|
|
23
|
+
* it) purges the old record and writes a brand-new one — new
|
|
24
|
+
* `type`/`revision` possibly, and ALWAYS a fresh `workflowExecutionToken`
|
|
25
|
+
* (`crypto.randomUUID()`, minted by every `start()` — see
|
|
26
|
+
* `start-state.ts`'s `buildInitialIdentitySlice`). Without
|
|
27
|
+
* {@link assertSameGeneration}, the stale `registration`/
|
|
28
|
+
* `resumeCheckpoint` a resume closure already built would replay against the
|
|
29
|
+
* replacement's fresh state: `commitSerializedResumeState` populates
|
|
30
|
+
* `internals.checkpoints` from the OLD checkpoint, and
|
|
31
|
+
* `relaunchInlineWorkflowAfterResume` drives the OLD handler — silently
|
|
32
|
+
* corrupting the replacement's run.
|
|
33
|
+
*
|
|
34
|
+
* A DIFFERENT engine's replacement is additionally caught earlier, by
|
|
35
|
+
* `acquireStandaloneClaimBeforeResume`'s durable `wakeOwnershipCheck`
|
|
36
|
+
* (workflow-lease mode only) — but that check compares only holder
|
|
37
|
+
* identity/epoch, never workflow identity, so a SAME-engine replacement
|
|
38
|
+
* (whose fold-acquire updates this engine's own already-held claim in
|
|
39
|
+
* place, leaving the epoch check trivially satisfied) is not caught there.
|
|
40
|
+
* This guard closes that gap for every ownership mode, not just
|
|
41
|
+
* workflow-lease.
|
|
42
|
+
*/
|
|
43
|
+
export type ResumeGeneration = {
|
|
44
|
+
type: string;
|
|
45
|
+
revision: string | undefined;
|
|
46
|
+
workflowExecutionToken: string | undefined;
|
|
47
|
+
};
|
|
48
|
+
/** Captures {@link ResumeGeneration} off the `WorkflowState` a resume decoded at its top, before any `await`. */
|
|
49
|
+
export declare function deriveResumeGeneration(state: WorkflowState): ResumeGeneration;
|
|
50
|
+
/**
|
|
51
|
+
* Throws when `latestState` — freshly re-read inside `performSerializedResume()`'s
|
|
52
|
+
* serialized section — no longer belongs to the same generation `expected`
|
|
53
|
+
* names. See {@link ResumeGeneration}'s doc for the exact race this
|
|
54
|
+
* closes. All three fields are compared — `workflowExecutionToken` alone
|
|
55
|
+
* would miss a legacy record that carries `undefined` on both sides of the
|
|
56
|
+
* race.
|
|
57
|
+
*/
|
|
58
|
+
export declare function assertSameGeneration(workflowId: string, latestState: WorkflowState, expected: ResumeGeneration): void;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export function deriveResumeGeneration(state) {
|
|
2
|
+
return {
|
|
3
|
+
type: state.type,
|
|
4
|
+
revision: state.revision,
|
|
5
|
+
workflowExecutionToken: state.workflowExecutionToken
|
|
6
|
+
};
|
|
7
|
+
}
|
|
8
|
+
export function assertSameGeneration(workflowId, latestState, expected) {
|
|
9
|
+
if (latestState.workflowExecutionToken !== expected.workflowExecutionToken || latestState.type !== expected.type || latestState.revision !== expected.revision)
|
|
10
|
+
throw Error(`Cannot resume workflow "${workflowId}": its persisted state changed generation (a replacement run landed at this same id) between resume's registration lookup ` + "and its serialized commit \u2014 retry against the new generation instead of replaying " + "stale state.");
|
|
11
|
+
}
|
|
@@ -8,6 +8,7 @@ import { buildTimerBatchOperations } from "../../scheduler.js";
|
|
|
8
8
|
import { createCancelHandlerRegistration, resetCancelHandlers } from "../cancel-handlers.js";
|
|
9
9
|
import { rememberCommittedCheckpointBytes } from "../checkpoint-commit-snapshots.js";
|
|
10
10
|
import { rehydrateChildCancellationHandlers } from "../child-workflow-cancellation.js";
|
|
11
|
+
import { resolveExecutableRegistrationOrRenamedNotFound } from "../dynamic-source-execution.js";
|
|
11
12
|
import { commitFencedEngineWrite } from "../fenced-write.js";
|
|
12
13
|
import { getWorkflowExecutionStartedAt } from "../handles.js";
|
|
13
14
|
import { loadWorkflowState } from "../storage-io.js";
|
|
@@ -19,6 +20,7 @@ import {
|
|
|
19
20
|
reprovideRecoveredServices,
|
|
20
21
|
workflowServicesResolverInfoFromState
|
|
21
22
|
} from "./recovered-services.js";
|
|
23
|
+
import { assertSameGeneration, deriveResumeGeneration } from "./resume-generation-guard.js";
|
|
22
24
|
import {
|
|
23
25
|
enforceHistoryPolicyBeforeReplay,
|
|
24
26
|
loadTerminalCleanupTrackedState,
|
|
@@ -79,7 +81,11 @@ function commitSerializedResumeState(internals, args) {
|
|
|
79
81
|
internals.parkedInlineWorkflows.delete(workflowId);
|
|
80
82
|
}
|
|
81
83
|
async function relaunchInlineWorkflowAfterResume(internals, latestState, args) {
|
|
82
|
-
const { workflowId, resumeCheckpoint, registration, callbacks } = args, accumulatedResults = new Map(resumeCheckpoint.accumulatedResults), workflowAbort = new AbortController;
|
|
84
|
+
const { workflowId, resumeCheckpoint, registration, resolvedRevision, callbacks } = args, accumulatedResults = new Map(resumeCheckpoint.accumulatedResults), workflowAbort = new AbortController;
|
|
85
|
+
internals.workflowTypeByWorkflowId.set(workflowId, {
|
|
86
|
+
type: latestState.type,
|
|
87
|
+
revision: resolvedRevision
|
|
88
|
+
});
|
|
83
89
|
resetCancelHandlers(internals, workflowId);
|
|
84
90
|
await rehydrateChildCancellationHandlers(internals, workflowId, callbacks);
|
|
85
91
|
const context = new Context({
|
|
@@ -115,7 +121,11 @@ async function relaunchInlineWorkflowAfterResume(internals, latestState, args) {
|
|
|
115
121
|
inlineStrategy.continueWorkflow(workflowId, void 0);
|
|
116
122
|
}
|
|
117
123
|
async function relaunchWorkerWorkflowAfterResume(internals, latestState, args) {
|
|
118
|
-
const { workflowId, resumeCheckpoint, workflowStartHeaders, callbacks } = args;
|
|
124
|
+
const { workflowId, resumeCheckpoint, workflowStartHeaders, resolvedRevision, callbacks } = args;
|
|
125
|
+
internals.workflowTypeByWorkflowId.set(workflowId, {
|
|
126
|
+
type: latestState.type,
|
|
127
|
+
revision: resolvedRevision
|
|
128
|
+
});
|
|
119
129
|
resetCancelHandlers(internals, workflowId);
|
|
120
130
|
await rehydrateChildCancellationHandlers(internals, workflowId, callbacks);
|
|
121
131
|
const serialized = serializeCheckpoint(resumeCheckpoint);
|
|
@@ -146,6 +156,7 @@ async function performSerializedResume(internals, args) {
|
|
|
146
156
|
throw Error(`Workflow "${workflowId}" not found in storage`);
|
|
147
157
|
if (latestState.status !== "running" && latestState.status !== "suspended")
|
|
148
158
|
throw Error(`Cannot resume workflow "${workflowId}": status is "${latestState.status}", expected "running" or "suspended"`);
|
|
159
|
+
assertSameGeneration(workflowId, latestState, args.expectedGeneration);
|
|
149
160
|
await reactivateSuspendedWorkflowState(internals, latestState);
|
|
150
161
|
commitSerializedResumeState(internals, args);
|
|
151
162
|
if (internals.inlineStrategy) {
|
|
@@ -179,18 +190,14 @@ export async function resumeWorkflowFromStorage(internals, workflowId, dispatchR
|
|
|
179
190
|
if (state.status !== "running" && state.status !== "suspended")
|
|
180
191
|
throw Error(`Cannot resume workflow "${workflowId}": status is "${state.status}", expected "running" or "suspended"`);
|
|
181
192
|
await acquireStandaloneClaimBeforeResume(internals, workflowId);
|
|
193
|
+
await loadTerminalCleanupTrackedState(internals, workflowId, callbacks);
|
|
182
194
|
const checkpointBytes = await internals.storage.get(KEYS.checkpoint(workflowId));
|
|
183
195
|
if (!checkpointBytes)
|
|
184
196
|
throw Error(`Checkpoint not found for workflow "${workflowId}"`);
|
|
185
|
-
const checkpoint = deserializeCheckpoint(checkpointBytes), registration =
|
|
186
|
-
if (!registration)
|
|
187
|
-
throw Error(`No workflow registered with name "${state.type}" (needed to resume "${workflowId}")`);
|
|
188
|
-
const preparedResumeState = await prepareResumeState(internals, workflowId, state, checkpoint, checkpointBytes, registration, callbacks), resumeCheckpoint = preparedResumeState.checkpoint, registeredVersionTuple = preparedResumeState.versionTuple, restoredHead = await new EventLog(internals.storage, workflowId).loadHead();
|
|
197
|
+
const checkpoint = deserializeCheckpoint(checkpointBytes), { entry: registration, revision: resolvedRevision } = await resolveExecutableRegistrationOrRenamedNotFound((type) => callbacks.resolveExecutableRegistrationForRevision(type, state.revision), state.type, () => Error(`No workflow registered with name "${state.type}" (needed to resume "${workflowId}")`)), preparedResumeState = await prepareResumeState(internals, workflowId, state, checkpoint, checkpointBytes, registration, callbacks), resumeCheckpoint = preparedResumeState.checkpoint, registeredVersionTuple = preparedResumeState.versionTuple, restoredHead = await new EventLog(internals.storage, workflowId).loadHead();
|
|
189
198
|
if (await enforceHistoryPolicyBeforeReplay(internals, workflowId, restoredHead, callbacks))
|
|
190
199
|
return callbacks.getHandle(workflowId);
|
|
191
|
-
const workflowStartHeaders = await loadWorkflowStartHeaders(internals, workflowId, callbacks);
|
|
192
|
-
await loadTerminalCleanupTrackedState(internals, workflowId, callbacks);
|
|
193
|
-
const { handle, shouldStop } = await prepareRecoveredWorkflowOrFail(internals, state, callbacks, onRecoveredWorkflow);
|
|
200
|
+
const workflowStartHeaders = await loadWorkflowStartHeaders(internals, workflowId, callbacks), { handle, shouldStop } = await prepareRecoveredWorkflowOrFail(internals, state, callbacks, onRecoveredWorkflow);
|
|
194
201
|
if (shouldStop)
|
|
195
202
|
return handle;
|
|
196
203
|
await callbacks.runSerializedWorkflowStateWrite(workflowId, () => performSerializedResume(internals, {
|
|
@@ -201,6 +208,8 @@ export async function resumeWorkflowFromStorage(internals, workflowId, dispatchR
|
|
|
201
208
|
restoredHead,
|
|
202
209
|
workflowStartHeaders,
|
|
203
210
|
registration,
|
|
211
|
+
resolvedRevision,
|
|
212
|
+
expectedGeneration: deriveResumeGeneration(state),
|
|
204
213
|
callbacks
|
|
205
214
|
}));
|
|
206
215
|
if (dispatchResumedEvent)
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { ComposedWorkflowInterceptor } from '../../interceptor.ts';
|
|
2
2
|
import type { WorkflowServicesResolverLaunchOptions, WorkflowServicesResolverScheduleInfo } from '../../types.ts';
|
|
3
|
+
import type { ExecutableRegistration } from '../dynamic-source-execution.ts';
|
|
3
4
|
import type { QueuedInlineWorkflowExecutionStart } from '../engine-internal-types.ts';
|
|
4
5
|
import { type WorkflowHandle } from '../handles.ts';
|
|
5
6
|
import type { EngineInternals } from '../internals.ts';
|
|
@@ -125,6 +126,48 @@ export type LifecycleCallbacks = {
|
|
|
125
126
|
* {@link RecoverAllOptions.versionMismatchPolicy}.
|
|
126
127
|
*/
|
|
127
128
|
failWorkflowForVersionMismatch: (workflowId: string, error: Error) => Promise<void>;
|
|
129
|
+
/**
|
|
130
|
+
* Turn a workflow `type` into an executable registration, awaiting
|
|
131
|
+
* dynamic-source resolution when `type` is not eagerly registered
|
|
132
|
+
* (WFT-15/16). See `dynamic-source-execution.ts`'s own doc for the full
|
|
133
|
+
* contract.
|
|
134
|
+
*
|
|
135
|
+
* `onRevisionChosen`, when given, fires synchronously the moment a lazy
|
|
136
|
+
* `type`'s target revision is picked — BEFORE the (potentially slow)
|
|
137
|
+
* loader is awaited — so a caller like `startWorkflow` can reserve an
|
|
138
|
+
* `inFlightStarts` slot for that exact revision immediately, closing the
|
|
139
|
+
* window where a concurrent `removeWorkflowRevision()` could observe zero
|
|
140
|
+
* references against a revision this call's own source load is about to
|
|
141
|
+
* durably (re)install. Never fires for an eager type (no revision to
|
|
142
|
+
* choose) or when resolution fails before a revision is picked (the
|
|
143
|
+
* ambiguous-revision case).
|
|
144
|
+
*/
|
|
145
|
+
resolveExecutableRegistration: (type: string, onRevisionChosen?: (revision: string) => void) => Promise<ExecutableRegistration>;
|
|
146
|
+
/**
|
|
147
|
+
* Force a recovered workflow to a terminal `failed` state because its
|
|
148
|
+
* dynamic workflow source could not be resolved during the recovery
|
|
149
|
+
* preload barrier — only this run fails; `recoverAll()` continues
|
|
150
|
+
* recovering sibling types.
|
|
151
|
+
*/
|
|
152
|
+
failWorkflowForUnavailableDynamicSource: (workflowId: string, error: Error) => Promise<void>;
|
|
153
|
+
/**
|
|
154
|
+
* Resolve workflow `type` against its EXACT pinned revision (WFT-17's
|
|
155
|
+
* `WorkflowState.revision`, `undefined` for a legacy pre-pinning record)
|
|
156
|
+
* instead of whichever revision the catalog currently considers active.
|
|
157
|
+
* Used by every resume path (`resumeWorkflowFromStorage`) so a pinned run
|
|
158
|
+
* never falls back to the active pointer. See
|
|
159
|
+
* `dynamic-source-execution.ts`'s `resolveExecutableRegistrationForRevision()`
|
|
160
|
+
* for the full classification contract.
|
|
161
|
+
*/
|
|
162
|
+
resolveExecutableRegistrationForRevision: (type: string, revision: string | undefined) => Promise<ExecutableRegistration>;
|
|
163
|
+
/**
|
|
164
|
+
* Force a recovered workflow to a terminal `failed` state because its
|
|
165
|
+
* pinned revision could not be resolved during the recovery preload
|
|
166
|
+
* barrier (WFT-18) — a {@link import('../revision-errors.ts').WorkflowRevisionUnavailableError}.
|
|
167
|
+
* Only this run fails; `recoverAll()` continues recovering sibling
|
|
168
|
+
* `(type, revision)` groups, including other revisions of the same type.
|
|
169
|
+
*/
|
|
170
|
+
failWorkflowForRevisionUnavailable: (workflowId: string, error: Error) => Promise<void>;
|
|
128
171
|
};
|
|
129
172
|
/**
|
|
130
173
|
* Pre-replay history circuit breaker. Called at every restore-from-checkpoint
|
|
@@ -31,3 +31,15 @@
|
|
|
31
31
|
*/
|
|
32
32
|
import type { EngineInternals } from '../internals.ts';
|
|
33
33
|
export declare function acquireStandaloneClaimBeforeResume(internals: EngineInternals, workflowId: string): Promise<void>;
|
|
34
|
+
/**
|
|
35
|
+
* Standalone-acquire `workflowId`'s claim (see {@link acquireStandaloneClaimBeforeResume},
|
|
36
|
+
* a no-op once already held) and hydrate its terminal-cleanup tracking from
|
|
37
|
+
* the durable marker — the two preconditions a `'self'`-fenced `failWorkflow()`
|
|
38
|
+
* write needs. Used by `operations-time.ts`'s `startDelayedWorkflow` on every
|
|
39
|
+
* failure exit (dynamic-source resolve, invalid execution-timeout) BEFORE
|
|
40
|
+
* failing: a delayed-start's pending→running write is the only other place
|
|
41
|
+
* this claim would otherwise get acquired (folded atomically, per ADR 0002),
|
|
42
|
+
* so a failure committed ahead of that write needs this standalone path
|
|
43
|
+
* instead, exactly like a recovered `running`/`suspended` workflow does.
|
|
44
|
+
*/
|
|
45
|
+
export declare function ensureDelayedStartClaimAndCleanupBeforeFailure(internals: EngineInternals, workflowId: string): Promise<void>;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { KEYS, storageHas } from "../../../storage/interface.js";
|
|
1
2
|
import { WorkflowClaimUnavailableError } from "../lease-errors.js";
|
|
2
3
|
import { wakeOwnershipCheck } from "../wake-ownership-check.js";
|
|
3
4
|
export async function acquireStandaloneClaimBeforeResume(internals, workflowId) {
|
|
@@ -23,3 +24,8 @@ export async function acquireStandaloneClaimBeforeResume(internals, workflowId)
|
|
|
23
24
|
if (result.status === "lost-race")
|
|
24
25
|
throw new WorkflowClaimUnavailableError(workflowId, result.heldBy);
|
|
25
26
|
}
|
|
27
|
+
export async function ensureDelayedStartClaimAndCleanupBeforeFailure(internals, workflowId) {
|
|
28
|
+
await acquireStandaloneClaimBeforeResume(internals, workflowId);
|
|
29
|
+
if (await storageHas(internals.storage, KEYS.terminalCleanupNeeded(workflowId)))
|
|
30
|
+
internals.workflowsNeedingTerminalCleanup.add(workflowId);
|
|
31
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Internal sentinel: the idempotent create batch lost its compare-and-swap to a
|
|
3
|
+
* concurrent caller holding the same idempotency key. Never surfaced to users —
|
|
4
|
+
* `start` / `startOrSignal` catch it and resolve to the winning run's handle.
|
|
5
|
+
*
|
|
6
|
+
* Lives in its own module so both `start-commit.ts` and the attribution helper it
|
|
7
|
+
* delegates to (`start-precondition-attribution.ts`) can raise it without an import
|
|
8
|
+
* cycle between them.
|
|
9
|
+
*/
|
|
10
|
+
export declare class StartIdempotencyRaceLostError extends Error {
|
|
11
|
+
constructor();
|
|
12
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type BatchOperation, type ConditionalBatchCondition } from '../../../storage/interface.ts';
|
|
2
2
|
import type { Checkpoint, StartOptions, TimerEntry, WorkflowState } from '../../types.ts';
|
|
3
3
|
import type { EngineInternals } from '../internals.ts';
|
|
4
4
|
import type { WorkflowConcurrencyStartOperations } from '../workflow-concurrency.ts';
|
|
@@ -17,13 +17,19 @@ export type BuildIdempotentStartOperations = (workflowId: string) => {
|
|
|
17
17
|
conditions: ConditionalBatchCondition[];
|
|
18
18
|
};
|
|
19
19
|
/**
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
20
|
+
* The `conditionalBatch` precondition half of {@link buildCatalogEntryStartPrecondition}
|
|
21
|
+
* — a bare `type`/`revision` pair rather than a full `WorkflowState`, so it
|
|
22
|
+
* is also reusable by a checkpoint-backed failed-run retry's reactivation
|
|
23
|
+
* commit (`bulk-operations-retry.ts`, WFT-17/18 Codex review on PR #958),
|
|
24
|
+
* which has no `WorkflowState & { revision: string }` narrowing of its own
|
|
25
|
+
* and — unlike a fresh start — carries no `inFlightStartsByRevision`
|
|
26
|
+
* reservation to close the SAME-process half of this race, so it needs this
|
|
27
|
+
* fence unconditionally rather than only under `ownershipMode !== 'none'`.
|
|
28
|
+
* Exported (rather than kept local to this module) specifically for that
|
|
29
|
+
* reuse; returns the bare {@link ConditionalBatchCondition} so callers never
|
|
30
|
+
* need this module's own non-exported {@link TaggedStartCondition} shape.
|
|
23
31
|
*/
|
|
24
|
-
export declare
|
|
25
|
-
constructor();
|
|
26
|
-
}
|
|
32
|
+
export declare function buildCatalogEntryRevisionCondition(internals: EngineInternals, type: string, revision: string): Promise<ConditionalBatchCondition>;
|
|
27
33
|
/** Everything {@link buildAndCommitStartBatch} needs to assemble the start batch. */
|
|
28
34
|
export type StartBatchContext = {
|
|
29
35
|
internals: EngineInternals;
|
|
@@ -44,6 +50,15 @@ export type StartBatchContext = {
|
|
|
44
50
|
* {@link buildStartBatchOperations}). Undefined for an ordinary start.
|
|
45
51
|
*/
|
|
46
52
|
purgeDeleteOperations: BatchOperation[] | undefined;
|
|
53
|
+
/**
|
|
54
|
+
* Compare-and-swap precondition making the caller-supplied-id duplicate check
|
|
55
|
+
* atomic with this commit (WFT-152). Built by `resolveTerminalConflictForRestart`
|
|
56
|
+
* from the exact bytes its duplicate-id read observed, and carried here as the
|
|
57
|
+
* `duplicateIdCondition` of the `StartDuplicateIdDecision` it returns. Undefined
|
|
58
|
+
* for a generated id, which cannot collide and so keeps the unconditioned hot
|
|
59
|
+
* path.
|
|
60
|
+
*/
|
|
61
|
+
duplicateIdCondition: ConditionalBatchCondition | undefined;
|
|
47
62
|
};
|
|
48
63
|
/**
|
|
49
64
|
* Assemble the start batch — folding in the id-dependent idempotency mapping and
|
|
@@ -1,23 +1,39 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
KEYS,
|
|
3
|
+
requireStorageCapability,
|
|
4
|
+
storageValuesEqual
|
|
5
|
+
} from "../../../storage/interface.js";
|
|
2
6
|
import { AtomicStateConflictError } from "../../atomic-state.js";
|
|
7
|
+
import { WorkflowAlreadyExistsError } from "../errors.js";
|
|
3
8
|
import {
|
|
4
9
|
commitFencedEngineWrite,
|
|
5
10
|
commitFencedEngineWriteAllowingPreconditionFailure
|
|
6
11
|
} from "../fenced-write.js";
|
|
12
|
+
import { WorkflowRevisionUnavailableError } from "../revision-errors.js";
|
|
7
13
|
import {
|
|
8
14
|
commitWithWorkflowClaimFold,
|
|
9
15
|
prepareWorkflowClaimFold,
|
|
10
16
|
throwWorkflowClaimUnavailable
|
|
11
17
|
} from "../workflow-claim-fold.js";
|
|
12
18
|
import { buildStartBatchOperations } from "./start-batch.js";
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
constructor() {
|
|
16
|
-
super("start idempotency compare-and-swap lost to a concurrent caller");
|
|
17
|
-
this.name = "StartIdempotencyRaceLostError";
|
|
18
|
-
}
|
|
19
|
-
}
|
|
19
|
+
import { StartIdempotencyRaceLostError } from "./start-commit-errors.js";
|
|
20
|
+
import { attributeLostStartPreconditionOrRetry } from "./start-precondition-attribution.js";
|
|
20
21
|
const WORKFLOW_CONCURRENCY_ADMISSION_MAX_ATTEMPTS = 5;
|
|
22
|
+
function needsCatalogEntryStartPrecondition(internals, state) {
|
|
23
|
+
return internals.options.ownershipMode !== "none" && state.revision !== void 0;
|
|
24
|
+
}
|
|
25
|
+
async function buildCatalogEntryStartPrecondition(internals, state) {
|
|
26
|
+
return {
|
|
27
|
+
source: "catalog-entry",
|
|
28
|
+
condition: await buildCatalogEntryRevisionCondition(internals, state.type, state.revision)
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
export async function buildCatalogEntryRevisionCondition(internals, type, revision) {
|
|
32
|
+
const key = KEYS.catalogEntry(type, revision), entryBytes = await internals.storage.get(key);
|
|
33
|
+
if (entryBytes === null)
|
|
34
|
+
throw new WorkflowRevisionUnavailableError(type, revision, "not-installed");
|
|
35
|
+
return { key, expectedValue: entryBytes };
|
|
36
|
+
}
|
|
21
37
|
async function persistStartBatch(internals, workflowId, startOperations, conditions, claimFold, isDelayedStart) {
|
|
22
38
|
if (claimFold) {
|
|
23
39
|
const result = await commitWithWorkflowClaimFold(internals, claimFold, startOperations, conditions.map((entry) => entry.condition), "workflow claim acquisition");
|
|
@@ -33,9 +49,19 @@ async function persistStartBatch(internals, workflowId, startOperations, conditi
|
|
|
33
49
|
requireStorageCapability(internals.storage, "conditionalBatch", "start preconditions");
|
|
34
50
|
return await commitFencedEngineWriteAllowingPreconditionFailure(internals, fenceWorkflowId, startOperations, conditions.map((entry) => entry.condition)) ? "committed" : "precondition-lost";
|
|
35
51
|
}
|
|
36
|
-
async function
|
|
52
|
+
async function hasStartConditionConflict(internals, conditions, source) {
|
|
53
|
+
for (const entry of conditions) {
|
|
54
|
+
if (entry.source !== source)
|
|
55
|
+
continue;
|
|
56
|
+
const currentValue = await internals.storage.get(entry.condition.key);
|
|
57
|
+
if (!storageValuesEqual(currentValue, entry.condition.expectedValue))
|
|
58
|
+
return !0;
|
|
59
|
+
}
|
|
60
|
+
return !1;
|
|
61
|
+
}
|
|
62
|
+
async function hasCatalogEntryConflict(internals, conditions) {
|
|
37
63
|
for (const entry of conditions) {
|
|
38
|
-
if (entry.source !== "
|
|
64
|
+
if (entry.source !== "catalog-entry")
|
|
39
65
|
continue;
|
|
40
66
|
const currentValue = await internals.storage.get(entry.condition.key);
|
|
41
67
|
if (!storageValuesEqual(currentValue, entry.condition.expectedValue))
|
|
@@ -49,6 +75,9 @@ function tagStartPreconditions(conditions) {
|
|
|
49
75
|
condition
|
|
50
76
|
}));
|
|
51
77
|
}
|
|
78
|
+
function tagDuplicateIdCondition(condition) {
|
|
79
|
+
return condition === void 0 ? [] : [{ source: "duplicate-id", condition }];
|
|
80
|
+
}
|
|
52
81
|
function tagWorkflowConcurrencyConditions(conditions) {
|
|
53
82
|
return conditions.map((condition) => ({
|
|
54
83
|
source: "workflow-concurrency",
|
|
@@ -66,18 +95,23 @@ export async function buildAndCommitStartBatch(context, buildIdempotentStartOper
|
|
|
66
95
|
for (let attempt = 0;attempt < maxAttempts; attempt += 1) {
|
|
67
96
|
const idempotent = buildIdempotentStartOperations?.(workflowId), workflowConcurrency = await context.buildWorkflowConcurrencyStartOperations?.();
|
|
68
97
|
lastWorkflowConcurrencyStateKey = workflowConcurrency?.stateKey ?? lastWorkflowConcurrencyStateKey;
|
|
69
|
-
const startOperations = buildStartBatchOperations(internals, workflowId, state, checkpoint, registration, options, state.executionDeadline, context.delayedStartTimer, context.persistedWorkflowStartHeaders, mergeAdditionalStartOperations(context.additionalStartOperations, mergeAdditionalStartOperations(idempotent?.operations, workflowConcurrency?.operations)), context.callbacks, context.purgeDeleteOperations), conditions = [
|
|
98
|
+
const catalogEntryPrecondition = needsCatalogEntryStartPrecondition(internals, state) ? await buildCatalogEntryStartPrecondition(internals, state) : void 0, startOperations = buildStartBatchOperations(internals, workflowId, state, checkpoint, registration, options, state.executionDeadline, context.delayedStartTimer, context.persistedWorkflowStartHeaders, mergeAdditionalStartOperations(context.additionalStartOperations, mergeAdditionalStartOperations(idempotent?.operations, workflowConcurrency?.operations)), context.callbacks, context.purgeDeleteOperations), conditions = [
|
|
70
99
|
...tagStartPreconditions(idempotent?.conditions),
|
|
71
|
-
...
|
|
100
|
+
...tagDuplicateIdCondition(context.duplicateIdCondition),
|
|
101
|
+
...tagWorkflowConcurrencyConditions(workflowConcurrency?.conditions ?? []),
|
|
102
|
+
...catalogEntryPrecondition === void 0 ? [] : [catalogEntryPrecondition]
|
|
72
103
|
], isDelayedStart = context.delayedStartTimer !== void 0, claimFold = isDelayedStart ? void 0 : await prepareWorkflowClaimFold(internals, workflowId), outcome = await persistStartBatch(internals, workflowId, startOperations, conditions, claimFold, isDelayedStart);
|
|
73
104
|
if (outcome === "committed")
|
|
74
105
|
return;
|
|
75
|
-
if (await
|
|
106
|
+
if (outcome !== "claim-lost" && await hasStartConditionConflict(internals, conditions, "duplicate-id"))
|
|
107
|
+
throw new WorkflowAlreadyExistsError(workflowId);
|
|
108
|
+
if (await hasStartConditionConflict(internals, conditions, "start-precondition"))
|
|
76
109
|
throw new StartIdempotencyRaceLostError;
|
|
110
|
+
if (await hasCatalogEntryConflict(internals, conditions))
|
|
111
|
+
throw new WorkflowRevisionUnavailableError(state.type, state.revision, "not-installed");
|
|
77
112
|
if (outcome === "claim-lost")
|
|
78
113
|
return throwWorkflowClaimUnavailable(internals, workflowId);
|
|
79
|
-
|
|
80
|
-
throw new StartIdempotencyRaceLostError;
|
|
114
|
+
await attributeLostStartPreconditionOrRetry(workflowId, context.duplicateIdCondition !== void 0, workflowConcurrency !== void 0, () => hasStartConditionConflict(internals, conditions, "workflow-concurrency"));
|
|
81
115
|
}
|
|
82
116
|
throw new AtomicStateConflictError(lastWorkflowConcurrencyStateKey ?? "workflow concurrency admission", WORKFLOW_CONCURRENCY_ADMISSION_MAX_ATTEMPTS);
|
|
83
117
|
}
|
|
@@ -2,8 +2,24 @@ import type { Checkpoint, StartOptions, WorkflowState } from '../../types.ts';
|
|
|
2
2
|
import type { EngineInternals } from '../internals.ts';
|
|
3
3
|
import { type LifecycleCallbacks, type RegistrationEntry } from './shared.ts';
|
|
4
4
|
export declare function runWorkflowStartInterceptor(_internals: EngineInternals, workflowId: string, workflowType: string, input: unknown, parentHeaders: Map<string, string> | undefined, callbacks: LifecycleCallbacks): Map<string, string> | undefined;
|
|
5
|
-
export declare function startWorkflowExecution(internals: EngineInternals, workflowId: string, workflowExecutionToken: string | undefined, workflowType: string,
|
|
6
|
-
|
|
5
|
+
export declare function startWorkflowExecution(internals: EngineInternals, workflowId: string, workflowExecutionToken: string | undefined, workflowType: string,
|
|
6
|
+
/**
|
|
7
|
+
* The starting run's persisted `WorkflowState.revision` (WFT-19/WFT-20):
|
|
8
|
+
* populates the per-instance identity cache readers use for revision-scoped
|
|
9
|
+
* routing, and is echoed to the strategy's `startWorkflow` for worker-protocol
|
|
10
|
+
* revision validation.
|
|
11
|
+
*/
|
|
12
|
+
revision: string | undefined, input: unknown, checkpoint: Checkpoint, nestingDepth: number, executionDeadline: number | undefined, executionStateOwnerId: string, _callbacks?: LifecycleCallbacks): void;
|
|
13
|
+
export declare function beginWorkflowExecution(internals: EngineInternals, workflowId: string, workflowExecutionToken: string | undefined, workflowType: string,
|
|
14
|
+
/**
|
|
15
|
+
* The starting run's persisted `WorkflowState.revision` (WFT-19/WFT-20).
|
|
16
|
+
* Threaded into both the queued-inline path below (which also re-reads
|
|
17
|
+
* `revision` off the RELOADED `WorkflowState` at flush time —
|
|
18
|
+
* `inline-launch-queue.ts` — exactly like it already does for
|
|
19
|
+
* `workflowExecutionToken`) and the non-inline `startWorkflowExecution`
|
|
20
|
+
* branch, which populates the per-instance identity cache with it directly.
|
|
21
|
+
*/
|
|
22
|
+
revision: string | undefined, input: unknown, checkpoint: Checkpoint, executionDeadline: number | undefined, executionStateOwnerId: string, _registration: RegistrationEntry, callbacks: LifecycleCallbacks, onStarted?: () => void): void;
|
|
7
23
|
/**
|
|
8
24
|
* `defer: false` is an inline-only liveness gate: it awaits the moment the
|
|
9
25
|
* generator is first driven. A worker-mode start queues to the Worker transport
|
|
@@ -20,13 +20,14 @@ export function runWorkflowStartInterceptor(_internals, workflowId, workflowType
|
|
|
20
20
|
});
|
|
21
21
|
return capturedHeaders;
|
|
22
22
|
}
|
|
23
|
-
export function startWorkflowExecution(internals, workflowId, workflowExecutionToken, workflowType, input, checkpoint, nestingDepth, executionDeadline, executionStateOwnerId, _callbacks) {
|
|
23
|
+
export function startWorkflowExecution(internals, workflowId, workflowExecutionToken, workflowType, revision, input, checkpoint, nestingDepth, executionDeadline, executionStateOwnerId, _callbacks) {
|
|
24
24
|
if (nestingDepth !== 0)
|
|
25
25
|
internals.workflowNestingDepths.set(workflowId, nestingDepth);
|
|
26
|
-
internals.workflowTypeByWorkflowId.set(workflowId, workflowType);
|
|
26
|
+
internals.workflowTypeByWorkflowId.set(workflowId, { type: workflowType, revision });
|
|
27
27
|
internals.strategy.startWorkflow({
|
|
28
28
|
workflowId,
|
|
29
29
|
...workflowExecutionToken !== void 0 && { workflowExecutionToken },
|
|
30
|
+
...revision !== void 0 && { revision },
|
|
30
31
|
workflowType,
|
|
31
32
|
input,
|
|
32
33
|
checkpoint: serializeCheckpoint(checkpoint),
|
|
@@ -40,7 +41,7 @@ export function startWorkflowExecution(internals, workflowId, workflowExecutionT
|
|
|
40
41
|
}
|
|
41
42
|
});
|
|
42
43
|
}
|
|
43
|
-
export function beginWorkflowExecution(internals, workflowId, workflowExecutionToken, workflowType, input, checkpoint, executionDeadline, executionStateOwnerId, _registration, callbacks, onStarted) {
|
|
44
|
+
export function beginWorkflowExecution(internals, workflowId, workflowExecutionToken, workflowType, revision, input, checkpoint, executionDeadline, executionStateOwnerId, _registration, callbacks, onStarted) {
|
|
44
45
|
const nestingDepth = internals.pendingNestingDepth ?? 0;
|
|
45
46
|
internals.pendingNestingDepth = void 0;
|
|
46
47
|
if (internals.inlineStrategy !== null) {
|
|
@@ -48,6 +49,7 @@ export function beginWorkflowExecution(internals, workflowId, workflowExecutionT
|
|
|
48
49
|
workflowId,
|
|
49
50
|
...workflowExecutionToken !== void 0 && { workflowExecutionToken },
|
|
50
51
|
workflowType,
|
|
52
|
+
revision,
|
|
51
53
|
input,
|
|
52
54
|
checkpoint,
|
|
53
55
|
nestingDepth,
|
|
@@ -58,7 +60,7 @@ export function beginWorkflowExecution(internals, workflowId, workflowExecutionT
|
|
|
58
60
|
return;
|
|
59
61
|
}
|
|
60
62
|
callbacks.dispatchEvent(new WorkflowStartedEvent(workflowId, workflowType, input));
|
|
61
|
-
startWorkflowExecution(internals, workflowId, workflowExecutionToken, workflowType, input, checkpoint, nestingDepth, executionDeadline, executionStateOwnerId, callbacks);
|
|
63
|
+
startWorkflowExecution(internals, workflowId, workflowExecutionToken, workflowType, revision, input, checkpoint, nestingDepth, executionDeadline, executionStateOwnerId, callbacks);
|
|
62
64
|
onStarted?.();
|
|
63
65
|
}
|
|
64
66
|
export function assertDeferSupported(internals, options, isDelayedStart) {
|
|
@@ -73,7 +75,7 @@ export async function beginExecutionAwaitingLiveness(internals, params, workflow
|
|
|
73
75
|
if (params.isDelayed)
|
|
74
76
|
return;
|
|
75
77
|
const liveness = params.options?.defer === !1 ? Promise.withResolvers() : void 0;
|
|
76
|
-
beginWorkflowExecution(internals, workflowId, params.state.workflowExecutionToken, params.type, params.input, params.checkpoint, params.state.executionDeadline, params.state.executionStateOwnerId ?? workflowId, params.registration, callbacks, liveness ? () => liveness.resolve() : void 0);
|
|
78
|
+
beginWorkflowExecution(internals, workflowId, params.state.workflowExecutionToken, params.type, params.state.revision, params.input, params.checkpoint, params.state.executionDeadline, params.state.executionStateOwnerId ?? workflowId, params.registration, callbacks, liveness ? () => liveness.resolve() : void 0);
|
|
77
79
|
if (liveness)
|
|
78
80
|
await liveness.promise;
|
|
79
81
|
}
|
|
@@ -2,9 +2,7 @@ import { KEYS } from "../../../storage/interface.js";
|
|
|
2
2
|
import { encode } from "../../codec.js";
|
|
3
3
|
import { WorkflowAlreadyExistsError } from "../errors.js";
|
|
4
4
|
import { buildCreateBatchSignalOperations } from "../signals.js";
|
|
5
|
-
import {
|
|
6
|
-
StartIdempotencyRaceLostError
|
|
7
|
-
} from "./start-commit.js";
|
|
5
|
+
import { StartIdempotencyRaceLostError } from "./start-commit-errors.js";
|
|
8
6
|
import {
|
|
9
7
|
requireWinnerId,
|
|
10
8
|
resolveCallerIdWinnerOrRetry,
|
|
@@ -6,7 +6,7 @@ import {
|
|
|
6
6
|
StartWorkflowValidationError
|
|
7
7
|
} from "../../start-workflow-validation.js";
|
|
8
8
|
import { IdempotencyKeyPurgedError, StartOrSignalConflictError } from "../errors.js";
|
|
9
|
-
import { StartIdempotencyRaceLostError } from "./start-commit.js";
|
|
9
|
+
import { StartIdempotencyRaceLostError } from "./start-commit-errors.js";
|
|
10
10
|
import {
|
|
11
11
|
createWithSignalOrFallback,
|
|
12
12
|
idempotentStartOperationsFor
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Decide what a lost start compare-and-swap MEANS, once every positively-detectable
|
|
3
|
+
* cause has already been ruled out by the caller — an idempotency precondition, a
|
|
4
|
+
* catalog-entry removal, and a lost workflow claim are each re-checked and raise
|
|
5
|
+
* their own error before this runs.
|
|
6
|
+
*
|
|
7
|
+
* Throws when the loss is terminal; returns normally when the caller should retry
|
|
8
|
+
* workflow-concurrency admission.
|
|
9
|
+
*
|
|
10
|
+
* WFT-152: a caller-supplied id that lost its duplicate-id condition means another
|
|
11
|
+
* engine sharing this store committed a create for the same id after this start's
|
|
12
|
+
* duplicate-id read. It raises the SAME {@link WorkflowAlreadyExistsError} the
|
|
13
|
+
* in-engine `pendingStarts` guard raises for the identical collision, so a
|
|
14
|
+
* cross-engine duplicate id is indistinguishable from an in-engine one at the call
|
|
15
|
+
* site.
|
|
16
|
+
*
|
|
17
|
+
* Attribution is by ELIMINATION, never by re-reading the duplicate-id key. A
|
|
18
|
+
* re-read is unsound: the winning run can complete and be purged (or swept by
|
|
19
|
+
* retention) between the failed compare-and-swap and the check, restoring the
|
|
20
|
+
* workflow record to the very value the condition expected, so the conflict reads
|
|
21
|
+
* back as "no conflict". With no workflow-concurrency conditions in the batch,
|
|
22
|
+
* nothing else is left for the lost outcome to mean.
|
|
23
|
+
*
|
|
24
|
+
* When concurrency conditions ARE present, retry only on positive evidence that the
|
|
25
|
+
* retryable one is what missed — hence `hasWorkflowConcurrencyConflict`, invoked
|
|
26
|
+
* lazily so the extra storage read happens only on that path.
|
|
27
|
+
*
|
|
28
|
+
* That evidence is NOT proof the duplicate id was fine, and deliberately is not
|
|
29
|
+
* treated as such. The concurrency precondition is a monotonic atomic-state VERSION
|
|
30
|
+
* key (`buildWorkflowConcurrencyStartOperations` conditions on `snapshot.version`
|
|
31
|
+
* and writes `version + 1`; releasing the slot increments again rather than
|
|
32
|
+
* restoring), so once a same-id winner acquires and releases, that condition stays
|
|
33
|
+
* mismatched forever and reports a conflict regardless of what else missed. What
|
|
34
|
+
* makes retrying safe here is the caller's own earlier check: it re-reads the
|
|
35
|
+
* duplicate-id key positively and raises `WorkflowAlreadyExistsError` before this
|
|
36
|
+
* runs, so reaching this point means the workflow record currently matches the
|
|
37
|
+
* expectation — the id is free right now, and a retry re-conditions on that same
|
|
38
|
+
* still-matching value.
|
|
39
|
+
*
|
|
40
|
+
* The residual is the pre-compare-and-swap purge ABA (WFT-153): a winner purged
|
|
41
|
+
* between the read and the commit makes an absent record look never-used, which no
|
|
42
|
+
* value-comparing condition can detect. That is tracked separately and is the reason
|
|
43
|
+
* this function does not claim exclusive attribution.
|
|
44
|
+
*
|
|
45
|
+
* With no concurrency evidence at all, fail closed: something missed, nothing
|
|
46
|
+
* retryable explains it, and a spurious `WorkflowAlreadyExistsError` is public,
|
|
47
|
+
* non-destructive, and retryable by the caller.
|
|
48
|
+
*/
|
|
49
|
+
export declare function attributeLostStartPreconditionOrRetry(workflowId: string, hasDuplicateIdCondition: boolean, hasWorkflowConcurrency: boolean, hasWorkflowConcurrencyConflict: () => Promise<boolean>): Promise<void>;
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { WorkflowAlreadyExistsError } from "../errors.js";
|
|
2
|
+
import { StartIdempotencyRaceLostError } from "./start-commit-errors.js";
|
|
3
|
+
export async function attributeLostStartPreconditionOrRetry(workflowId, hasDuplicateIdCondition, hasWorkflowConcurrency, hasWorkflowConcurrencyConflict) {
|
|
4
|
+
if (!hasDuplicateIdCondition) {
|
|
5
|
+
if (!hasWorkflowConcurrency)
|
|
6
|
+
throw new StartIdempotencyRaceLostError;
|
|
7
|
+
return;
|
|
8
|
+
}
|
|
9
|
+
if (!hasWorkflowConcurrency)
|
|
10
|
+
throw new WorkflowAlreadyExistsError(workflowId);
|
|
11
|
+
if (!await hasWorkflowConcurrencyConflict())
|
|
12
|
+
throw new WorkflowAlreadyExistsError(workflowId);
|
|
13
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Resolve the exact revision a fresh `startWorkflow()` call — or a schedule's
|
|
3
|
+
* pin capture (WFT-20, `pinned-schedule-revision.ts`) — is about to commit
|
|
4
|
+
* to. Extracted out of `lifecycle/start.ts` so both that module and the
|
|
5
|
+
* schedule pin-capture path import the SAME logic instead of duplicating the
|
|
6
|
+
* `registeredCatalogRevisions`-fallback + `ensureWorkflowCatalogReady()`
|
|
7
|
+
* recheck rule; `start.ts` imports these back unchanged.
|
|
8
|
+
*
|
|
9
|
+
* @module core/engine/lifecycle/start-revision-resolution
|
|
10
|
+
*/
|
|
11
|
+
import type { EngineInternals } from '../internals.ts';
|
|
12
|
+
/**
|
|
13
|
+
* The exact executable artifact a start is about to run: the resolved
|
|
14
|
+
* dynamic-source candidate revision, or — for an eager registration, which
|
|
15
|
+
* never populates `resolvedRevision` — this process's own
|
|
16
|
+
* `registeredCatalogRevisions` entry for `type` (the revision of the code
|
|
17
|
+
* actually loaded here, NOT `inFlightRevision`, which for an eager type
|
|
18
|
+
* falls back to the catalog's cached ACTIVE pointer and can name a revision
|
|
19
|
+
* this process never loaded under a multi-engine deployment). Synchronous,
|
|
20
|
+
* on purpose: every top-level engine.* method already awaits
|
|
21
|
+
* `ensureWorkflowCatalogReady()` before reaching a call site that needs this,
|
|
22
|
+
* so this map is populated by the time the overwhelmingly common case gets
|
|
23
|
+
* here. `undefined` means "genuinely not cached yet"; the caller falls back
|
|
24
|
+
* to {@link resolveStartRevisionUncached}.
|
|
25
|
+
*/
|
|
26
|
+
export declare function resolveCachedStartRevision(internals: EngineInternals, type: string, resolvedRevision: string | undefined): string | undefined;
|
|
27
|
+
/**
|
|
28
|
+
* The rare fallback {@link resolveCachedStartRevision} defers to: a fired
|
|
29
|
+
* schedule occurrence or a delayed-start timer calls `startWorkflow`
|
|
30
|
+
* directly from background scheduler code, with no top-level
|
|
31
|
+
* `ensureWorkflowCatalogReady()` gate already awaited. Re-checks catalog
|
|
32
|
+
* readiness once, then re-reads the cache.
|
|
33
|
+
*/
|
|
34
|
+
export declare function resolveStartRevisionUncached(internals: EngineInternals, type: string): Promise<string>;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { ensureWorkflowCatalogReady } from "../catalog-readiness.js";
|
|
2
|
+
export function resolveCachedStartRevision(internals, type, resolvedRevision) {
|
|
3
|
+
return resolvedRevision ?? internals.registeredCatalogRevisions.get(type);
|
|
4
|
+
}
|
|
5
|
+
export async function resolveStartRevisionUncached(internals, type) {
|
|
6
|
+
await ensureWorkflowCatalogReady(internals.engine);
|
|
7
|
+
const afterReadiness = internals.registeredCatalogRevisions.get(type);
|
|
8
|
+
if (afterReadiness !== void 0)
|
|
9
|
+
return afterReadiness;
|
|
10
|
+
throw Error(`Cannot start workflow "${type}": no catalog revision is registered for this eagerly-registered type, even after re-checking catalog readiness. This should be unreachable.`);
|
|
11
|
+
}
|