@lostgradient/weft 0.24.0 → 0.24.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/cli/conformance.js +5 -2
- package/dist/cli/generated/operation-catalog.snapshot.json +95 -1
- package/dist/cli/generated/operation-client.generated.d.ts +28 -2
- package/dist/cli/generated/operation-client.generated.js +2 -0
- package/dist/cli/parse-schedule-arguments.js +15 -3
- package/dist/cli/schedule.js +2 -1
- package/dist/cli/types.d.ts +2 -1
- package/dist/client/http-client-requests.js +6 -9
- package/dist/client/start-body.js +1 -0
- package/dist/core/catalog/index.d.ts +2 -2
- package/dist/core/catalog/index.js +9 -2
- package/dist/core/catalog/reference-counts.d.ts +29 -14
- package/dist/core/catalog/removal.d.ts +63 -10
- package/dist/core/catalog/removal.js +18 -2
- package/dist/core/catalog/storage-io.d.ts +23 -0
- package/dist/core/catalog/storage-io.js +1 -1
- package/dist/core/engine/activity-resolution.js +17 -17
- package/dist/core/engine/bulk-operations-purge.d.ts +1 -1
- package/dist/core/engine/bulk-operations-purge.js +17 -22
- package/dist/core/engine/bulk-operations-retry.d.ts +24 -0
- package/dist/core/engine/bulk-operations-retry.js +190 -0
- package/dist/core/engine/bulk-operations-shared.d.ts +18 -0
- package/dist/core/engine/bulk-operations-shared.js +14 -0
- package/dist/core/engine/bulk-operations.d.ts +3 -2
- package/dist/core/engine/bulk-operations.js +10 -169
- package/dist/core/engine/callback-creators-bundles.js +5 -3
- package/dist/core/engine/callback-creators-core.js +9 -1
- package/dist/core/engine/callback-creators-schedule.js +2 -2
- package/dist/core/engine/catalog-readiness.js +2 -0
- package/dist/core/engine/catalog-removal.d.ts +107 -3
- package/dist/core/engine/catalog-removal.js +66 -13
- package/dist/core/engine/catalog-tombstone-recovery.d.ts +56 -0
- package/dist/core/engine/catalog-tombstone-recovery.js +39 -0
- package/dist/core/engine/child-workflow.d.ts +13 -1
- package/dist/core/engine/child-workflow.js +22 -4
- package/dist/core/engine/constraints.js +5 -1
- package/dist/core/engine/construction.d.ts +1 -1
- package/dist/core/engine/decode-revision.d.ts +32 -0
- package/dist/core/engine/decode-revision.js +9 -0
- package/dist/core/engine/disposal.d.ts +6 -2
- package/dist/core/engine/disposal.js +24 -10
- package/dist/core/engine/dynamic-source-errors.d.ts +81 -0
- package/dist/core/engine/dynamic-source-errors.js +25 -0
- package/dist/core/engine/dynamic-source-execution.d.ts +173 -0
- package/dist/core/engine/dynamic-source-execution.js +108 -0
- package/dist/core/engine/engine-internal-types.d.ts +16 -0
- package/dist/core/engine/engine-workflows-namespace.d.ts +10 -0
- package/dist/core/engine/engine-workflows-namespace.js +3 -1
- package/dist/core/engine/generation-codec.d.ts +22 -0
- package/dist/core/engine/generation-codec.js +2 -0
- package/dist/core/engine/index.d.ts +18 -10
- package/dist/core/engine/index.js +22 -11
- package/dist/core/engine/inline-launch-queue.js +1 -1
- package/dist/core/engine/internals.d.ts +26 -30
- package/dist/core/engine/lifecycle/checkpoint-launch.d.ts +29 -0
- package/dist/core/engine/lifecycle/checkpoint-launch.js +75 -0
- package/dist/core/engine/lifecycle/delayed-start-registration.d.ts +30 -0
- package/dist/core/engine/lifecycle/delayed-start-registration.js +11 -0
- package/dist/core/engine/lifecycle/fork-helpers.d.ts +33 -1
- package/dist/core/engine/lifecycle/fork-helpers.js +3 -1
- package/dist/core/engine/lifecycle/recovery-isolation.d.ts +20 -0
- package/dist/core/engine/lifecycle/recovery-isolation.js +26 -0
- package/dist/core/engine/lifecycle/recovery-revision-groups.d.ts +75 -0
- package/dist/core/engine/lifecycle/recovery-revision-groups.js +64 -0
- package/dist/core/engine/lifecycle/resume-body.d.ts +44 -0
- package/dist/core/engine/lifecycle/resume-body.js +211 -0
- package/dist/core/engine/lifecycle/resume-generation-guard.d.ts +58 -0
- package/dist/core/engine/lifecycle/resume-generation-guard.js +11 -0
- package/dist/core/engine/lifecycle/resume.d.ts +46 -3
- package/dist/core/engine/lifecycle/resume.js +21 -201
- package/dist/core/engine/lifecycle/shared.d.ts +43 -0
- package/dist/core/engine/lifecycle/standalone-claim-acquire.d.ts +80 -2
- package/dist/core/engine/lifecycle/standalone-claim-acquire.js +20 -4
- package/dist/core/engine/lifecycle/start-commit-errors.d.ts +12 -0
- package/dist/core/engine/lifecycle/start-commit-errors.js +6 -0
- package/dist/core/engine/lifecycle/start-commit.d.ts +24 -7
- package/dist/core/engine/lifecycle/start-commit.js +57 -15
- package/dist/core/engine/lifecycle/start-exec.d.ts +18 -2
- package/dist/core/engine/lifecycle/start-exec.js +7 -5
- package/dist/core/engine/lifecycle/start-or-signal-create.js +1 -3
- package/dist/core/engine/lifecycle/start-or-signal-resolution.d.ts +8 -2
- package/dist/core/engine/lifecycle/start-or-signal-resolution.js +11 -3
- package/dist/core/engine/lifecycle/start-or-signal.js +1 -1
- package/dist/core/engine/lifecycle/start-precondition-attribution.d.ts +55 -0
- package/dist/core/engine/lifecycle/start-precondition-attribution.js +13 -0
- package/dist/core/engine/lifecycle/start-revision-resolution.d.ts +34 -0
- package/dist/core/engine/lifecycle/start-revision-resolution.js +11 -0
- package/dist/core/engine/lifecycle/start-schedule-timing.d.ts +11 -0
- package/dist/core/engine/lifecycle/start-schedule-timing.js +21 -0
- package/dist/core/engine/lifecycle/start-state.d.ts +16 -0
- package/dist/core/engine/lifecycle/start-state.js +60 -0
- package/dist/core/engine/lifecycle/start-terminal-conflict-purge.d.ts +146 -4
- package/dist/core/engine/lifecycle/start-terminal-conflict-purge.js +33 -4
- package/dist/core/engine/lifecycle/start.d.ts +54 -7
- package/dist/core/engine/lifecycle/start.js +53 -90
- package/dist/core/engine/lifecycle/transition.d.ts +12 -4
- package/dist/core/engine/lifecycle/transition.js +40 -95
- package/dist/core/engine/lifecycle.d.ts +5 -2
- package/dist/core/engine/lifecycle.js +7 -6
- package/dist/core/engine/listing.js +15 -1
- package/dist/core/engine/memo-durable-activity.js +4 -1
- package/dist/core/engine/nonterminal-revision-count.d.ts +25 -0
- package/dist/core/engine/nonterminal-revision-count.js +16 -0
- package/dist/core/engine/operations-time.d.ts +5 -23
- package/dist/core/engine/operations-time.js +13 -11
- package/dist/core/engine/ownership-bootstrap.d.ts +1 -1
- package/dist/core/engine/pinned-schedule-revision-count.d.ts +26 -0
- package/dist/core/engine/pinned-schedule-revision-count.js +19 -0
- package/dist/core/engine/pinned-schedule-revision.d.ts +76 -0
- package/dist/core/engine/pinned-schedule-revision.js +53 -0
- package/dist/core/engine/registration.d.ts +30 -0
- package/dist/core/engine/registration.js +17 -2
- package/dist/core/engine/retention.js +9 -2
- package/dist/core/engine/revision-errors.d.ts +71 -0
- package/dist/core/engine/revision-errors.js +13 -0
- package/dist/core/engine/schedule-overlap.js +2 -1
- package/dist/core/engine/schedule-revision-fire.d.ts +17 -0
- package/dist/core/engine/schedule-revision-fire.js +3 -0
- package/dist/core/engine/schedule-run.d.ts +10 -0
- package/dist/core/engine/schedule-run.js +2 -1
- package/dist/core/engine/schedules.d.ts +11 -1
- package/dist/core/engine/schedules.js +28 -9
- package/dist/core/engine/source-diagnostics.d.ts +76 -0
- package/dist/core/engine/source-diagnostics.js +105 -0
- package/dist/core/engine/source-registration.js +4 -2
- package/dist/core/engine/source-resolution.d.ts +25 -9
- package/dist/core/engine/source-resolution.js +49 -14
- package/dist/core/engine/source-runtime-state.d.ts +92 -0
- package/dist/core/engine/source-runtime-state.js +36 -0
- package/dist/core/engine/storage-io.d.ts +19 -2
- package/dist/core/engine/storage-io.js +21 -2
- package/dist/core/engine/termination/complete.js +1 -1
- package/dist/core/engine/termination/finalizer-registration.d.ts +18 -0
- package/dist/core/engine/termination/finalizer-registration.js +21 -0
- package/dist/core/engine/termination/finalizer.d.ts +16 -19
- package/dist/core/engine/termination/finalizer.js +3 -3
- package/dist/core/engine/termination/suspend.js +3 -0
- package/dist/core/engine/time-operation-callbacks.d.ts +32 -0
- package/dist/core/engine/time-operation-callbacks.js +0 -0
- package/dist/core/engine/validation/schedule-cadence.d.ts +16 -0
- package/dist/core/engine/validation/schedule-cadence.js +20 -0
- package/dist/core/engine/validation/schedule-options.d.ts +4 -1
- package/dist/core/engine/validation/schedule-options.js +14 -0
- package/dist/core/engine/validation/schedule-revision.d.ts +24 -0
- package/dist/core/engine/validation/schedule-revision.js +21 -0
- package/dist/core/engine/validation/schedule-warnings.d.ts +11 -0
- package/dist/core/engine/validation/schedule-warnings.js +5 -0
- package/dist/core/engine/validation/schedule.d.ts +29 -4
- package/dist/core/engine/validation/schedule.js +19 -36
- package/dist/core/engine/validation.js +13 -17
- package/dist/core/engine/workflow-claim-reclaim-target.d.ts +1 -1
- package/dist/core/engine/workflow-claim-reclaim-target.js +22 -8
- package/dist/core/engine/workflow-claim-registry.d.ts +79 -75
- package/dist/core/engine/workflow-claim-registry.js +15 -2
- package/dist/core/engine/workflow-generation-fence.d.ts +74 -0
- package/dist/core/engine/workflow-generation-fence.js +19 -0
- package/dist/core/engine/workflow-retention-deadline.d.ts +16 -0
- package/dist/core/engine/workflow-retention-deadline.js +33 -0
- package/dist/core/events/event-map.d.ts +5 -0
- package/dist/core/events/index.d.ts +1 -0
- package/dist/core/events/index.js +1 -0
- package/dist/core/events/workflow-source-events.d.ts +102 -0
- package/dist/core/events/workflow-source-events.js +57 -0
- package/dist/core/execution-strategy.d.ts +7 -0
- package/dist/core/inline-execution-strategy.context-options.d.ts +1 -1
- package/dist/core/inline-execution-strategy.js +1 -1
- package/dist/core/start-workflow-validation.d.ts +15 -0
- package/dist/core/start-workflow-validation.js +11 -1
- package/dist/core/types/checkpoint.d.ts +14 -0
- package/dist/core/types/schedules.d.ts +52 -1
- package/dist/core/types/state.d.ts +16 -0
- package/dist/core/weft-error.d.ts +1 -1
- package/dist/core/weft-error.js +4 -1
- package/dist/core/worker-execution-strategy.d.ts +1 -0
- package/dist/core/worker-execution-strategy.js +13 -1
- package/dist/core/worker-fault-handling.d.ts +10 -0
- package/dist/core/worker-fault-handling.js +1 -0
- package/dist/core/worker-inbound-message.d.ts +1 -0
- package/dist/core/worker-inbound-message.js +1 -0
- package/dist/core/worker-protocol-guard.js +4 -0
- package/dist/core/worker-protocol.d.ts +7 -0
- package/dist/core/worker-protocol.js +2 -1
- package/dist/core/worker-turn-watchdog.d.ts +8 -1
- package/dist/core/worker-turn-watchdog.js +3 -2
- package/dist/core/workflow-identifiers.d.ts +37 -0
- package/dist/core/workflow-identifiers.js +19 -1
- package/dist/diagnostics/format.js +5 -0
- package/dist/diagnostics/types.d.ts +24 -0
- package/dist/diagnostics/version-check.js +8 -2
- package/dist/http.js +2 -2
- package/dist/index.d.ts +3 -3
- package/dist/index.js +7 -0
- package/dist/indexeddb.js +1 -1
- package/dist/json-schema.js +3 -3
- package/dist/server/fault-to-json-rpc.js +2 -1
- package/dist/server/fleet-event-feed.js +6 -6
- package/dist/server/index.d.ts +11 -0
- package/dist/server/operation-fault.d.ts +2 -0
- package/dist/server/operation-fault.js +2 -1
- package/dist/server/operations/create-schedule.d.ts +2 -0
- package/dist/server/operations/create-schedule.js +10 -6
- package/dist/server/operations/get-catalog-diagnostics.d.ts +28 -0
- package/dist/server/operations/get-catalog-diagnostics.js +10 -2
- package/dist/server/operations/get-task-detail-schema.d.ts +8 -0
- package/dist/server/operations/get-task-detail-schema.js +1 -0
- package/dist/server/operations/get-task-detail.d.ts +8 -0
- package/dist/server/operations/get-task-detail.js +1 -0
- package/dist/server/operations/get-task-diagnostics.d.ts +4 -4
- package/dist/server/operations/preload-workflow-revision.d.ts +28 -0
- package/dist/server/operations/preload-workflow-revision.js +60 -0
- package/dist/server/operations/schedule-faults.d.ts +1 -0
- package/dist/server/operations/schedule-faults.js +32 -4
- package/dist/server/operations/schedule-rest-body.d.ts +1 -0
- package/dist/server/operations/schedule-rest-body.js +2 -1
- package/dist/server/operations/static-registrations.js +6 -0
- package/dist/server/operations/update-schedule.d.ts +2 -0
- package/dist/server/operations/update-schedule.js +4 -2
- package/dist/server/operations/workflow-catalog-operation-helpers.js +27 -0
- package/dist/server/rest-binding.d.ts +11 -0
- package/dist/server/runtime/task-dispatch-envelope.d.ts +41 -0
- package/dist/server/runtime/task-dispatch-envelope.js +54 -0
- package/dist/server/runtime/task-dispatch-revision.d.ts +29 -0
- package/dist/server/runtime/task-dispatch-revision.js +13 -0
- package/dist/server/runtime/task-dispatch.d.ts +3 -1
- package/dist/server/runtime/task-dispatch.js +20 -44
- package/dist/server/runtime/task-ledger-recovery.js +1 -1
- package/dist/server/runtime/task-ledger-runtime.d.ts +14 -2
- package/dist/server/runtime/task-ledger-runtime.js +5 -3
- package/dist/server/runtime/task-polling.js +9 -3
- package/dist/server/runtime/task-reconciliation.js +2 -1
- package/dist/server/runtime/websocket-worker.js +9 -0
- package/dist/server/task-ledger-codec.d.ts +28 -0
- package/dist/server/task-ledger-codec.js +7 -1
- package/dist/server/task-ledger-transition-helpers.js +1 -0
- package/dist/server/task-ledger-types.d.ts +9 -0
- package/dist/server/task-ledger.d.ts +1 -1
- package/dist/server/task-ledger.js +3 -1
- package/dist/server/task-queue-types.d.ts +2 -0
- package/dist/storage/bun-sql.js +135 -100
- package/dist/storage/catalog-keys.d.ts +19 -0
- package/dist/storage/catalog-keys.js +3 -1
- package/dist/storage/compressed-storage.js +1 -1
- package/dist/storage/generation-keys.d.ts +52 -0
- package/dist/storage/generation-keys.js +4 -0
- package/dist/storage/index.d.ts +76 -63
- package/dist/storage/interface.d.ts +24 -230
- package/dist/storage/interface.js +1 -1
- package/dist/storage/key-prefixes.d.ts +1 -1
- package/dist/storage/key-prefixes.js +2 -0
- package/dist/storage/lease-keys.d.ts +48 -0
- package/dist/storage/lease-keys.js +8 -0
- package/dist/storage/lmdb.d.ts +30 -2
- package/dist/storage/lmdb.js +1 -1
- package/dist/storage/memory.js +1 -1
- package/dist/storage/neon.js +3 -3
- package/dist/storage/node-sqlite.js +135 -100
- package/dist/storage/postgres.js +3 -3
- package/dist/storage/resolve.js +1 -1
- package/dist/storage/scoped-storage.js +1 -1
- package/dist/storage/signal-keys.d.ts +23 -0
- package/dist/storage/signal-keys.js +9 -0
- package/dist/storage/testing.js +1 -1
- package/dist/storage/turso.js +2 -2
- package/dist/storage/workflow-lifecycle-keys.d.ts +128 -0
- package/dist/storage/workflow-lifecycle-keys.js +46 -0
- package/dist/storage/workflow-record-keys.d.ts +98 -0
- package/dist/storage/workflow-record-keys.js +47 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/web-extension.js +1 -1
- package/dist/worker/protocol-messages.d.ts +17 -0
- package/dist/worker/protocol-schemas.d.ts +32 -0
- package/dist/worker/protocol-schemas.js +7 -3
- package/dist/worker/protocol-task-result.js +35 -3
- package/dist/worker/protocol.js +1 -1
- package/dist/worker/registry/types.d.ts +2 -0
- package/dist/worker/registry.d.ts +1 -1
- package/dist/worker/registry.js +3 -1
- package/dist/workers/workflow-runner.d.ts +9 -0
- package/dist/workers/workflow-runner.js +9 -2
- package/dist/workers/workflow-worker-entry.js +9 -4
- package/package.json +2 -2
|
@@ -1,211 +1,31 @@
|
|
|
1
1
|
import { KEYS } from "../../../storage/interface.js";
|
|
2
|
-
import { deserializeCheckpoint, serializeCheckpoint } from "../../checkpoint.js";
|
|
3
|
-
import { encode } from "../../codec.js";
|
|
4
|
-
import { Context, setContextWorkflowInterceptor } from "../../context.js";
|
|
5
|
-
import { EventLog } from "../../event-log.js";
|
|
6
|
-
import { WorkflowResumedEvent } from "../../events.js";
|
|
7
|
-
import { buildTimerBatchOperations } from "../../scheduler.js";
|
|
8
|
-
import { createCancelHandlerRegistration, resetCancelHandlers } from "../cancel-handlers.js";
|
|
9
|
-
import { rememberCommittedCheckpointBytes } from "../checkpoint-commit-snapshots.js";
|
|
10
|
-
import { rehydrateChildCancellationHandlers } from "../child-workflow-cancellation.js";
|
|
11
|
-
import { commitFencedEngineWrite } from "../fenced-write.js";
|
|
12
|
-
import { getWorkflowExecutionStartedAt } from "../handles.js";
|
|
13
|
-
import { loadWorkflowState } from "../storage-io.js";
|
|
14
|
-
import { getComposedWorkflowInterceptor } from "../strategy-helpers.js";
|
|
15
2
|
import { decodeWorkflowState } from "../validation.js";
|
|
16
|
-
import {
|
|
17
|
-
import { prepareResumeState } from "./persist.js";
|
|
3
|
+
import { performResumeAfterClaimAcquired } from "./resume-body.js";
|
|
18
4
|
import {
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
} from "./
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
}
|
|
32
|
-
async function runRecoveredWorkflowHookOrFail(internals, state, handle, resolverInfo, onRecoveredWorkflow, callbacks) {
|
|
33
|
-
const info = {
|
|
34
|
-
...resolverInfo,
|
|
35
|
-
handle,
|
|
36
|
-
launchOptions: resolverInfo.launchOptions ?? { id: state.id },
|
|
37
|
-
services: internals.workflowServices.get(state.id)
|
|
38
|
-
};
|
|
39
|
-
try {
|
|
40
|
-
await onRecoveredWorkflow(info);
|
|
41
|
-
return !1;
|
|
42
|
-
} catch (error) {
|
|
43
|
-
const reason = error instanceof Error ? error.message : String(error), hookError = Error(`Recovery hook failed for workflow "${state.id}": ${reason}`);
|
|
44
|
-
try {
|
|
45
|
-
await callbacks.failWorkflowForRecoveryHook(state.id, hookError);
|
|
46
|
-
} catch (commitError) {
|
|
47
|
-
callbacks.handleCleanupError("onRecoveredWorkflow", commitError, state.id);
|
|
48
|
-
}
|
|
49
|
-
return !0;
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
|
-
async function prepareRecoveredWorkflowOrFail(internals, state, callbacks, onRecoveredWorkflow) {
|
|
53
|
-
const resolverInfo = onRecoveredWorkflow === void 0 ? void 0 : await workflowServicesResolverInfoFromState(internals, state), handle = callbacks.getHandle(state.id);
|
|
54
|
-
if (await prepareRecoveredServicesOrFail(internals, state, callbacks, resolverInfo))
|
|
55
|
-
return { handle, shouldStop: !0 };
|
|
56
|
-
if (onRecoveredWorkflow === void 0 || resolverInfo === void 0)
|
|
57
|
-
return { handle, shouldStop: !1 };
|
|
58
|
-
const shouldStop = await runRecoveredWorkflowHookOrFail(internals, state, handle, resolverInfo, onRecoveredWorkflow, callbacks);
|
|
59
|
-
return { handle, shouldStop };
|
|
60
|
-
}
|
|
61
|
-
function assertResumeNotTerminating(internals, workflowId) {
|
|
62
|
-
if (internals.terminalizingWorkflows.has(workflowId))
|
|
63
|
-
throw Error(`Cannot resume workflow "${workflowId}": termination is in progress`);
|
|
64
|
-
}
|
|
65
|
-
function commitSerializedResumeState(internals, args) {
|
|
66
|
-
const {
|
|
67
|
-
workflowId,
|
|
68
|
-
resumeCheckpoint,
|
|
69
|
-
serializedCheckpoint,
|
|
70
|
-
registeredVersionTuple,
|
|
71
|
-
restoredHead,
|
|
72
|
-
callbacks
|
|
73
|
-
} = args;
|
|
74
|
-
internals.checkpoints.set(workflowId, resumeCheckpoint);
|
|
75
|
-
rememberCommittedCheckpointBytes(internals, workflowId, serializedCheckpoint);
|
|
76
|
-
internals.workflowVersionTuples.set(workflowId, registeredVersionTuple);
|
|
77
|
-
internals.eventLogHeads.set(workflowId, restoredHead);
|
|
78
|
-
setWorkflowStartHeaders(internals, workflowId, args.workflowStartHeaders, callbacks);
|
|
79
|
-
internals.parkedInlineWorkflows.delete(workflowId);
|
|
80
|
-
}
|
|
81
|
-
async function relaunchInlineWorkflowAfterResume(internals, latestState, args) {
|
|
82
|
-
const { workflowId, resumeCheckpoint, registration, callbacks } = args, accumulatedResults = new Map(resumeCheckpoint.accumulatedResults), workflowAbort = new AbortController;
|
|
83
|
-
resetCancelHandlers(internals, workflowId);
|
|
84
|
-
await rehydrateChildCancellationHandlers(internals, workflowId, callbacks);
|
|
85
|
-
const context = new Context({
|
|
86
|
-
workflowId,
|
|
87
|
-
...latestState.workflowExecutionToken !== void 0 && {
|
|
88
|
-
workflowExecutionToken: latestState.workflowExecutionToken
|
|
89
|
-
},
|
|
90
|
-
workflowType: latestState.type,
|
|
91
|
-
startedAt: getWorkflowExecutionStartedAt(latestState),
|
|
92
|
-
abortController: workflowAbort,
|
|
93
|
-
getNow: internals.options.getNow,
|
|
94
|
-
resolveWorkflowType: callbacks.resolveWorkflowTypeTarget,
|
|
95
|
-
executionStateOwnerId: latestState.executionStateOwnerId ?? workflowId,
|
|
96
|
-
accumulatedResults,
|
|
97
|
-
locals: resumeCheckpoint.locals,
|
|
98
|
-
searchAttributes: resumeCheckpoint.searchAttributes,
|
|
99
|
-
registerCancelHandler: createCancelHandlerRegistration(internals, workflowId),
|
|
100
|
-
...registration.searchAttributes && {
|
|
101
|
-
searchAttributeSchema: registration.searchAttributes
|
|
102
|
-
},
|
|
103
|
-
sleepReferenceTime: resumeCheckpoint.createdAt,
|
|
104
|
-
...latestState.executionDeadline !== void 0 && {
|
|
105
|
-
deadline: latestState.executionDeadline
|
|
106
|
-
},
|
|
107
|
-
services: internals.workflowServices.get(workflowId),
|
|
108
|
-
...internals.options.onLog != null && { logSink: internals.options.onLog }
|
|
109
|
-
});
|
|
110
|
-
setContextWorkflowInterceptor(context, getComposedWorkflowInterceptor(internals));
|
|
111
|
-
if (internals.options.development)
|
|
112
|
-
context.explain(!0);
|
|
113
|
-
const generator = registration.handler(context, latestState.input), inlineStrategy = internals.inlineStrategy;
|
|
114
|
-
inlineStrategy.adoptWorkflow(workflowId, generator, context, workflowAbort);
|
|
115
|
-
inlineStrategy.continueWorkflow(workflowId, void 0);
|
|
116
|
-
}
|
|
117
|
-
async function relaunchWorkerWorkflowAfterResume(internals, latestState, args) {
|
|
118
|
-
const { workflowId, resumeCheckpoint, workflowStartHeaders, callbacks } = args;
|
|
119
|
-
resetCancelHandlers(internals, workflowId);
|
|
120
|
-
await rehydrateChildCancellationHandlers(internals, workflowId, callbacks);
|
|
121
|
-
const serialized = serializeCheckpoint(resumeCheckpoint);
|
|
122
|
-
internals.strategy.startWorkflow({
|
|
123
|
-
workflowId,
|
|
124
|
-
...latestState.workflowExecutionToken !== void 0 && {
|
|
125
|
-
workflowExecutionToken: latestState.workflowExecutionToken
|
|
126
|
-
},
|
|
127
|
-
workflowType: latestState.type,
|
|
128
|
-
input: latestState.input,
|
|
129
|
-
checkpoint: serialized,
|
|
130
|
-
nestingDepth: internals.workflowNestingDepths.get(workflowId) ?? 0,
|
|
131
|
-
executionStateOwnerId: latestState.executionStateOwnerId ?? workflowId,
|
|
132
|
-
...latestState.executionDeadline !== void 0 && {
|
|
133
|
-
deadline: latestState.executionDeadline
|
|
134
|
-
},
|
|
135
|
-
...workflowStartHeaders !== void 0 && workflowStartHeaders.size > 0 && {
|
|
136
|
-
headers: [...workflowStartHeaders]
|
|
137
|
-
}
|
|
138
|
-
});
|
|
139
|
-
}
|
|
140
|
-
async function performSerializedResume(internals, args) {
|
|
141
|
-
const { workflowId } = args;
|
|
142
|
-
assertResumeNotTerminating(internals, workflowId);
|
|
143
|
-
const latestState = await loadWorkflowState(internals, workflowId);
|
|
144
|
-
assertResumeNotTerminating(internals, workflowId);
|
|
145
|
-
if (!latestState)
|
|
146
|
-
throw Error(`Workflow "${workflowId}" not found in storage`);
|
|
147
|
-
if (latestState.status !== "running" && latestState.status !== "suspended")
|
|
148
|
-
throw Error(`Cannot resume workflow "${workflowId}": status is "${latestState.status}", expected "running" or "suspended"`);
|
|
149
|
-
await reactivateSuspendedWorkflowState(internals, latestState);
|
|
150
|
-
commitSerializedResumeState(internals, args);
|
|
151
|
-
if (internals.inlineStrategy) {
|
|
152
|
-
await relaunchInlineWorkflowAfterResume(internals, latestState, args);
|
|
153
|
-
return;
|
|
154
|
-
}
|
|
155
|
-
await relaunchWorkerWorkflowAfterResume(internals, latestState, args);
|
|
156
|
-
}
|
|
157
|
-
async function reactivateSuspendedWorkflowState(internals, state) {
|
|
158
|
-
if (state.status !== "suspended")
|
|
159
|
-
return;
|
|
160
|
-
const previousState = { ...state };
|
|
161
|
-
state.status = "running";
|
|
162
|
-
state.updatedAt = internals.options.getNow();
|
|
163
|
-
await commitFencedEngineWrite(internals, state.id, [
|
|
164
|
-
{ type: "put", key: KEYS.workflow(state.id), value: encode(state) },
|
|
165
|
-
...buildWorkflowVisibilityIndexTransition(state.id, previousState, state).batchOps,
|
|
166
|
-
...state.executionDeadline !== void 0 ? buildTimerBatchOperations({
|
|
167
|
-
id: `deadline:${state.id}`,
|
|
168
|
-
workflowId: state.id,
|
|
169
|
-
fireAt: state.executionDeadline,
|
|
170
|
-
kind: "execution-deadline"
|
|
171
|
-
}) : []
|
|
172
|
-
], [], () => Error(`Resume of workflow "${state.id}" lost its CAS race.`));
|
|
173
|
-
}
|
|
174
|
-
export async function resumeWorkflowFromStorage(internals, workflowId, dispatchResumedEvent, callbacks, onRecoveredWorkflow) {
|
|
5
|
+
acquireStandaloneClaimBeforeResume,
|
|
6
|
+
releaseFreshlyAcquiredResumeClaim
|
|
7
|
+
} from "./standalone-claim-acquire.js";
|
|
8
|
+
function recordFreshClaimEpoch(acquireResult, tracker) {
|
|
9
|
+
if (acquireResult.freshlyAcquired && tracker !== void 0)
|
|
10
|
+
tracker.epoch = acquireResult.epoch;
|
|
11
|
+
}
|
|
12
|
+
async function releaseFreshClaimOnRejectionUnlessDeferred(internals, workflowId, acquireResult, deferRelease) {
|
|
13
|
+
if (acquireResult.freshlyAcquired && deferRelease !== !0)
|
|
14
|
+
await releaseFreshlyAcquiredResumeClaim(internals, workflowId, acquireResult.epoch);
|
|
15
|
+
}
|
|
16
|
+
export async function resumeWorkflowFromStorage(internals, workflowId, dispatchResumedEvent, callbacks, onRecoveredWorkflow, options) {
|
|
175
17
|
const stateBytes = await internals.storage.get(KEYS.workflow(workflowId));
|
|
176
18
|
if (!stateBytes)
|
|
177
19
|
throw Error(`Workflow "${workflowId}" not found in storage`);
|
|
178
20
|
const state = decodeWorkflowState(stateBytes);
|
|
179
21
|
if (state.status !== "running" && state.status !== "suspended")
|
|
180
22
|
throw Error(`Cannot resume workflow "${workflowId}": status is "${state.status}", expected "running" or "suspended"`);
|
|
181
|
-
await acquireStandaloneClaimBeforeResume(internals, workflowId);
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
throw
|
|
188
|
-
|
|
189
|
-
if (await enforceHistoryPolicyBeforeReplay(internals, workflowId, restoredHead, callbacks))
|
|
190
|
-
return callbacks.getHandle(workflowId);
|
|
191
|
-
const workflowStartHeaders = await loadWorkflowStartHeaders(internals, workflowId, callbacks);
|
|
192
|
-
await loadTerminalCleanupTrackedState(internals, workflowId, callbacks);
|
|
193
|
-
const { handle, shouldStop } = await prepareRecoveredWorkflowOrFail(internals, state, callbacks, onRecoveredWorkflow);
|
|
194
|
-
if (shouldStop)
|
|
195
|
-
return handle;
|
|
196
|
-
await callbacks.runSerializedWorkflowStateWrite(workflowId, () => performSerializedResume(internals, {
|
|
197
|
-
workflowId,
|
|
198
|
-
resumeCheckpoint,
|
|
199
|
-
serializedCheckpoint: preparedResumeState.serializedCheckpoint,
|
|
200
|
-
registeredVersionTuple,
|
|
201
|
-
restoredHead,
|
|
202
|
-
workflowStartHeaders,
|
|
203
|
-
registration,
|
|
204
|
-
callbacks
|
|
205
|
-
}));
|
|
206
|
-
if (dispatchResumedEvent)
|
|
207
|
-
callbacks.dispatchEvent(new WorkflowResumedEvent(workflowId, resumeCheckpoint.step));
|
|
208
|
-
if (internals.inlineStrategy)
|
|
209
|
-
callbacks.swallowPromiseRejection(callbacks.processPendingUpdatesAfterInlineAdvance(workflowId));
|
|
210
|
-
return handle;
|
|
23
|
+
const acquireResult = await acquireStandaloneClaimBeforeResume(internals, workflowId);
|
|
24
|
+
recordFreshClaimEpoch(acquireResult, options?.freshClaimTracker);
|
|
25
|
+
try {
|
|
26
|
+
return await performResumeAfterClaimAcquired(internals, workflowId, state, dispatchResumedEvent, callbacks, onRecoveredWorkflow);
|
|
27
|
+
} catch (error) {
|
|
28
|
+
await releaseFreshClaimOnRejectionUnlessDeferred(internals, workflowId, acquireResult, options?.deferClaimReleaseOnRejection);
|
|
29
|
+
throw error;
|
|
30
|
+
}
|
|
211
31
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { ComposedWorkflowInterceptor } from '../../interceptor.ts';
|
|
2
2
|
import type { WorkflowServicesResolverLaunchOptions, WorkflowServicesResolverScheduleInfo } from '../../types.ts';
|
|
3
|
+
import type { ExecutableRegistration } from '../dynamic-source-execution.ts';
|
|
3
4
|
import type { QueuedInlineWorkflowExecutionStart } from '../engine-internal-types.ts';
|
|
4
5
|
import { type WorkflowHandle } from '../handles.ts';
|
|
5
6
|
import type { EngineInternals } from '../internals.ts';
|
|
@@ -125,6 +126,48 @@ export type LifecycleCallbacks = {
|
|
|
125
126
|
* {@link RecoverAllOptions.versionMismatchPolicy}.
|
|
126
127
|
*/
|
|
127
128
|
failWorkflowForVersionMismatch: (workflowId: string, error: Error) => Promise<void>;
|
|
129
|
+
/**
|
|
130
|
+
* Turn a workflow `type` into an executable registration, awaiting
|
|
131
|
+
* dynamic-source resolution when `type` is not eagerly registered
|
|
132
|
+
* (WFT-15/16). See `dynamic-source-execution.ts`'s own doc for the full
|
|
133
|
+
* contract.
|
|
134
|
+
*
|
|
135
|
+
* `onRevisionChosen`, when given, fires synchronously the moment a lazy
|
|
136
|
+
* `type`'s target revision is picked — BEFORE the (potentially slow)
|
|
137
|
+
* loader is awaited — so a caller like `startWorkflow` can reserve an
|
|
138
|
+
* `inFlightStarts` slot for that exact revision immediately, closing the
|
|
139
|
+
* window where a concurrent `removeWorkflowRevision()` could observe zero
|
|
140
|
+
* references against a revision this call's own source load is about to
|
|
141
|
+
* durably (re)install. Never fires for an eager type (no revision to
|
|
142
|
+
* choose) or when resolution fails before a revision is picked (the
|
|
143
|
+
* ambiguous-revision case).
|
|
144
|
+
*/
|
|
145
|
+
resolveExecutableRegistration: (type: string, onRevisionChosen?: (revision: string) => void) => Promise<ExecutableRegistration>;
|
|
146
|
+
/**
|
|
147
|
+
* Force a recovered workflow to a terminal `failed` state because its
|
|
148
|
+
* dynamic workflow source could not be resolved during the recovery
|
|
149
|
+
* preload barrier — only this run fails; `recoverAll()` continues
|
|
150
|
+
* recovering sibling types.
|
|
151
|
+
*/
|
|
152
|
+
failWorkflowForUnavailableDynamicSource: (workflowId: string, error: Error) => Promise<void>;
|
|
153
|
+
/**
|
|
154
|
+
* Resolve workflow `type` against its EXACT pinned revision (WFT-17's
|
|
155
|
+
* `WorkflowState.revision`, `undefined` for a legacy pre-pinning record)
|
|
156
|
+
* instead of whichever revision the catalog currently considers active.
|
|
157
|
+
* Used by every resume path (`resumeWorkflowFromStorage`) so a pinned run
|
|
158
|
+
* never falls back to the active pointer. See
|
|
159
|
+
* `dynamic-source-execution.ts`'s `resolveExecutableRegistrationForRevision()`
|
|
160
|
+
* for the full classification contract.
|
|
161
|
+
*/
|
|
162
|
+
resolveExecutableRegistrationForRevision: (type: string, revision: string | undefined) => Promise<ExecutableRegistration>;
|
|
163
|
+
/**
|
|
164
|
+
* Force a recovered workflow to a terminal `failed` state because its
|
|
165
|
+
* pinned revision could not be resolved during the recovery preload
|
|
166
|
+
* barrier (WFT-18) — a {@link import('../revision-errors.ts').WorkflowRevisionUnavailableError}.
|
|
167
|
+
* Only this run fails; `recoverAll()` continues recovering sibling
|
|
168
|
+
* `(type, revision)` groups, including other revisions of the same type.
|
|
169
|
+
*/
|
|
170
|
+
failWorkflowForRevisionUnavailable: (workflowId: string, error: Error) => Promise<void>;
|
|
128
171
|
};
|
|
129
172
|
/**
|
|
130
173
|
* Pre-replay history circuit breaker. Called at every restore-from-checkpoint
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
*
|
|
11
11
|
* Deliberately covers `running` and `suspended` the same way. The ADR
|
|
12
12
|
* describes `suspended` as folding `acquire` into the status flip itself
|
|
13
|
-
* (`reactivateSuspendedWorkflowState`'s commit in `resume.ts`), but the
|
|
13
|
+
* (`reactivateSuspendedWorkflowState`'s commit in `resume-body.ts`), but the
|
|
14
14
|
* EARLIER fenced writes above need a held claim regardless of final status,
|
|
15
15
|
* so one early standalone acquire is the pragmatic shape for both — a
|
|
16
16
|
* documented deviation from that row, not an oversight.
|
|
@@ -27,7 +27,85 @@
|
|
|
27
27
|
* re-check (it can win the CAS against a still-valid claim), so only the
|
|
28
28
|
* durable holder is re-read (no write) and compared.
|
|
29
29
|
*
|
|
30
|
+
* **`'holder-absent'` falls through to a fresh `acquire()` instead of hard-
|
|
31
|
+
* failing (WFT-134).** `suspendWorkflow`'s external terminal rotation
|
|
32
|
+
* durably deletes `wf-owner-holder:<id>` as part of its suspend commit, which
|
|
33
|
+
* can outpace this engine's own local cache correction
|
|
34
|
+
* (`WorkflowClaimRegistry.forgetLocalClaim`). A same-engine `resume()`
|
|
35
|
+
* landing in that gap must not be treated as a real conflict — it is
|
|
36
|
+
* exactly the "no other holder to lose to" case a fresh `acquire()` handles
|
|
37
|
+
* safely. `'holder-undecodable'` and `'generation-mismatch'` remain hard
|
|
38
|
+
* failures: both mean a holder record IS present and either corrupt or
|
|
39
|
+
* naming a different generation, a real conflict this function must not
|
|
40
|
+
* paper over.
|
|
41
|
+
*
|
|
30
42
|
* @module core/engine/lifecycle/standalone-claim-acquire
|
|
31
43
|
*/
|
|
32
44
|
import type { EngineInternals } from '../internals.ts';
|
|
33
|
-
|
|
45
|
+
/**
|
|
46
|
+
* Result of {@link acquireStandaloneClaimBeforeResume}. `freshlyAcquired:
|
|
47
|
+
* true` means this call itself performed a fresh `registry.acquire()`
|
|
48
|
+
* (whether because no claim was cached yet, or because the `'holder-absent'`
|
|
49
|
+
* fallback below ran), and carries the exact `epoch` that acquire minted —
|
|
50
|
+
* `freshlyAcquired: false` means no acquisition happened at all (folding
|
|
51
|
+
* disabled, no registry, or the cached claim matched). Callers use this to
|
|
52
|
+
* know whether, and under which exact generation, THEY are now responsible
|
|
53
|
+
* for releasing a claim this call just installed if the resume they were
|
|
54
|
+
* preparing turns out not to be resumable after all — see `resume.ts`'s
|
|
55
|
+
* `resumeWorkflowFromStorage` (WFT-134).
|
|
56
|
+
*/
|
|
57
|
+
export type StandaloneClaimAcquireResult = {
|
|
58
|
+
freshlyAcquired: true;
|
|
59
|
+
epoch: number;
|
|
60
|
+
} | {
|
|
61
|
+
freshlyAcquired: false;
|
|
62
|
+
};
|
|
63
|
+
export declare function acquireStandaloneClaimBeforeResume(internals: EngineInternals, workflowId: string): Promise<StandaloneClaimAcquireResult>;
|
|
64
|
+
/**
|
|
65
|
+
* Best-effort durable release of a claim {@link acquireStandaloneClaimBeforeResume}
|
|
66
|
+
* freshly acquired for THIS resume attempt, called when a later, status-aware
|
|
67
|
+
* check (`performSerializedResume`'s status/generation re-validation in
|
|
68
|
+
* `resume-body.ts`) rejects the resume after all (WFT-134). Without this, a claim
|
|
69
|
+
* acquired for a workflow that turned out to be non-resumable — already
|
|
70
|
+
* terminal, or replaced by an `onTerminalConflict: 'start-new'` run — stays
|
|
71
|
+
* installed: the renewal task keeps refreshing it indefinitely, and a
|
|
72
|
+
* legitimate `start-new` (or the replacement run's own fold-acquire) loses
|
|
73
|
+
* its CAS against a claim nothing is actually driving.
|
|
74
|
+
*
|
|
75
|
+
* `acquiredEpoch` MUST be the exact epoch {@link acquireStandaloneClaimBeforeResume}
|
|
76
|
+
* returned to this same caller — passed straight through to
|
|
77
|
+
* `WorkflowClaimRegistry.release()`'s own `expectedEpoch` guard (WFT-134
|
|
78
|
+
* review round 2). Between this resume's acquire and its rejection, a
|
|
79
|
+
* DIFFERENT `start-new` or resume can legitimately replace the registry's
|
|
80
|
+
* entry for `workflowId`; releasing "whatever is current" instead of this
|
|
81
|
+
* exact generation would drop that replacement's live claim and strand it
|
|
82
|
+
* without local epoch bytes. `release()` no-ops (`'not-held'`) when the
|
|
83
|
+
* registry has moved past `acquiredEpoch`, so this call is always safe to
|
|
84
|
+
* make regardless of what has happened to the entry since.
|
|
85
|
+
*
|
|
86
|
+
* A full durable `release()`, not a local-only `forgetLocalClaim()`: the
|
|
87
|
+
* acquire this undoes durably wrote BOTH `wf-owner-epoch:<id>` and
|
|
88
|
+
* `wf-owner-holder:<id>`, and it is specifically the durable holder record
|
|
89
|
+
* that would otherwise block a legitimate replacement's own acquire — merely
|
|
90
|
+
* clearing this engine's local cache would leave that durable block in place.
|
|
91
|
+
* Swallows a lost-race/thrown release the same way every other best-effort
|
|
92
|
+
* claim release in this codebase does (see `complete.ts`'s
|
|
93
|
+
* `releaseWorkflowClaimAfterTerminalSettlement`): a failed release here just
|
|
94
|
+
* leaves the claim for TTL/grace expiry, never worse than not attempting it.
|
|
95
|
+
* `release()` itself already forgets its LOCAL entry on a thrown durable
|
|
96
|
+
* error (identity/epoch-guarded), so this call needs no matching cleanup of
|
|
97
|
+
* its own to stop the renewal task from renewing a claim nothing drives.
|
|
98
|
+
*/
|
|
99
|
+
export declare function releaseFreshlyAcquiredResumeClaim(internals: EngineInternals, workflowId: string, acquiredEpoch: number): Promise<void>;
|
|
100
|
+
/**
|
|
101
|
+
* Standalone-acquire `workflowId`'s claim (see {@link acquireStandaloneClaimBeforeResume},
|
|
102
|
+
* a no-op once already held) and hydrate its terminal-cleanup tracking from
|
|
103
|
+
* the durable marker — the two preconditions a `'self'`-fenced `failWorkflow()`
|
|
104
|
+
* write needs. Used by `operations-time.ts`'s `startDelayedWorkflow` on every
|
|
105
|
+
* failure exit (dynamic-source resolve, invalid execution-timeout) BEFORE
|
|
106
|
+
* failing: a delayed-start's pending→running write is the only other place
|
|
107
|
+
* this claim would otherwise get acquired (folded atomically, per ADR 0002),
|
|
108
|
+
* so a failure committed ahead of that write needs this standalone path
|
|
109
|
+
* instead, exactly like a recovered `running`/`suspended` workflow does.
|
|
110
|
+
*/
|
|
111
|
+
export declare function ensureDelayedStartClaimAndCleanupBeforeFailure(internals: EngineInternals, workflowId: string): Promise<void>;
|
|
@@ -1,11 +1,12 @@
|
|
|
1
|
+
import { KEYS, storageHas } from "../../../storage/interface.js";
|
|
1
2
|
import { WorkflowClaimUnavailableError } from "../lease-errors.js";
|
|
2
3
|
import { wakeOwnershipCheck } from "../wake-ownership-check.js";
|
|
3
4
|
export async function acquireStandaloneClaimBeforeResume(internals, workflowId) {
|
|
4
5
|
if (internals.options.ownershipMode !== "workflow-lease")
|
|
5
|
-
return;
|
|
6
|
+
return { freshlyAcquired: !1 };
|
|
6
7
|
const registry = internals.workflowClaimRegistry;
|
|
7
8
|
if (registry === null)
|
|
8
|
-
return;
|
|
9
|
+
return { freshlyAcquired: !1 };
|
|
9
10
|
const cachedEpoch = registry.currentEpoch(workflowId);
|
|
10
11
|
if (cachedEpoch !== null) {
|
|
11
12
|
const check = await wakeOwnershipCheck({
|
|
@@ -15,11 +16,26 @@ export async function acquireStandaloneClaimBeforeResume(internals, workflowId)
|
|
|
15
16
|
expectedEngineId: registry.engineId,
|
|
16
17
|
expectedEpoch: cachedEpoch
|
|
17
18
|
});
|
|
18
|
-
if (check.status === "
|
|
19
|
+
if (check.status === "match")
|
|
20
|
+
return { freshlyAcquired: !1 };
|
|
21
|
+
if (check.reason !== "holder-absent")
|
|
19
22
|
throw new WorkflowClaimUnavailableError(workflowId, check.observedEngineId);
|
|
20
|
-
return;
|
|
21
23
|
}
|
|
22
24
|
const result = await registry.acquire(workflowId);
|
|
23
25
|
if (result.status === "lost-race")
|
|
24
26
|
throw new WorkflowClaimUnavailableError(workflowId, result.heldBy);
|
|
27
|
+
return { freshlyAcquired: !0, epoch: result.epoch };
|
|
28
|
+
}
|
|
29
|
+
export async function releaseFreshlyAcquiredResumeClaim(internals, workflowId, acquiredEpoch) {
|
|
30
|
+
const registry = internals.workflowClaimRegistry;
|
|
31
|
+
if (registry === null)
|
|
32
|
+
return;
|
|
33
|
+
try {
|
|
34
|
+
await registry.release(workflowId, acquiredEpoch);
|
|
35
|
+
} catch {}
|
|
36
|
+
}
|
|
37
|
+
export async function ensureDelayedStartClaimAndCleanupBeforeFailure(internals, workflowId) {
|
|
38
|
+
await acquireStandaloneClaimBeforeResume(internals, workflowId);
|
|
39
|
+
if (await storageHas(internals.storage, KEYS.terminalCleanupNeeded(workflowId)))
|
|
40
|
+
internals.workflowsNeedingTerminalCleanup.add(workflowId);
|
|
25
41
|
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Internal sentinel: the idempotent create batch lost its compare-and-swap to a
|
|
3
|
+
* concurrent caller holding the same idempotency key. Never surfaced to users —
|
|
4
|
+
* `start` / `startOrSignal` catch it and resolve to the winning run's handle.
|
|
5
|
+
*
|
|
6
|
+
* Lives in its own module so both `start-commit.ts` and the attribution helper it
|
|
7
|
+
* delegates to (`start-precondition-attribution.ts`) can raise it without an import
|
|
8
|
+
* cycle between them.
|
|
9
|
+
*/
|
|
10
|
+
export declare class StartIdempotencyRaceLostError extends Error {
|
|
11
|
+
constructor();
|
|
12
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type BatchOperation, type ConditionalBatchCondition } from '../../../storage/interface.ts';
|
|
2
2
|
import type { Checkpoint, StartOptions, TimerEntry, WorkflowState } from '../../types.ts';
|
|
3
3
|
import type { EngineInternals } from '../internals.ts';
|
|
4
4
|
import type { WorkflowConcurrencyStartOperations } from '../workflow-concurrency.ts';
|
|
@@ -17,13 +17,19 @@ export type BuildIdempotentStartOperations = (workflowId: string) => {
|
|
|
17
17
|
conditions: ConditionalBatchCondition[];
|
|
18
18
|
};
|
|
19
19
|
/**
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
20
|
+
* The `conditionalBatch` precondition half of {@link buildCatalogEntryStartPrecondition}
|
|
21
|
+
* — a bare `type`/`revision` pair rather than a full `WorkflowState`, so it
|
|
22
|
+
* is also reusable by a checkpoint-backed failed-run retry's reactivation
|
|
23
|
+
* commit (`bulk-operations-retry.ts`, WFT-17/18 Codex review on PR #958),
|
|
24
|
+
* which has no `WorkflowState & { revision: string }` narrowing of its own
|
|
25
|
+
* and — unlike a fresh start — carries no `inFlightStartsByRevision`
|
|
26
|
+
* reservation to close the SAME-process half of this race, so it needs this
|
|
27
|
+
* fence unconditionally rather than only under `ownershipMode !== 'none'`.
|
|
28
|
+
* Exported (rather than kept local to this module) specifically for that
|
|
29
|
+
* reuse; returns the bare {@link ConditionalBatchCondition} so callers never
|
|
30
|
+
* need this module's own non-exported {@link TaggedStartCondition} shape.
|
|
23
31
|
*/
|
|
24
|
-
export declare
|
|
25
|
-
constructor();
|
|
26
|
-
}
|
|
32
|
+
export declare function buildCatalogEntryRevisionCondition(internals: EngineInternals, type: string, revision: string): Promise<ConditionalBatchCondition>;
|
|
27
33
|
/** Everything {@link buildAndCommitStartBatch} needs to assemble the start batch. */
|
|
28
34
|
export type StartBatchContext = {
|
|
29
35
|
internals: EngineInternals;
|
|
@@ -44,6 +50,17 @@ export type StartBatchContext = {
|
|
|
44
50
|
* {@link buildStartBatchOperations}). Undefined for an ordinary start.
|
|
45
51
|
*/
|
|
46
52
|
purgeDeleteOperations: BatchOperation[] | undefined;
|
|
53
|
+
/**
|
|
54
|
+
* Compare-and-swap precondition making the caller-supplied-id duplicate check
|
|
55
|
+
* atomic with this commit (WFT-152). Built by `resolveTerminalConflictForRestart`
|
|
56
|
+
* from the exact bytes its duplicate-id read observed, and carried here as the
|
|
57
|
+
* `duplicateIdCondition` of the `StartDuplicateIdDecision` it returns. Undefined
|
|
58
|
+
* for a generated id, which cannot collide and so keeps the unconditioned hot
|
|
59
|
+
* path.
|
|
60
|
+
*/
|
|
61
|
+
duplicateIdCondition: ConditionalBatchCondition | undefined;
|
|
62
|
+
/** ADDITIONAL WFT-153 precondition on observed `wf-gen:<id>` bytes; see `StartDuplicateIdDecision.duplicateIdGenerationCondition` in `start-terminal-conflict-purge.ts`. Undefined exactly when `duplicateIdCondition` is. */
|
|
63
|
+
duplicateIdGenerationCondition: ConditionalBatchCondition | undefined;
|
|
47
64
|
};
|
|
48
65
|
/**
|
|
49
66
|
* Assemble the start batch — folding in the id-dependent idempotency mapping and
|
|
@@ -1,23 +1,39 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
KEYS,
|
|
3
|
+
requireStorageCapability,
|
|
4
|
+
storageValuesEqual
|
|
5
|
+
} from "../../../storage/interface.js";
|
|
2
6
|
import { AtomicStateConflictError } from "../../atomic-state.js";
|
|
7
|
+
import { WorkflowAlreadyExistsError } from "../errors.js";
|
|
3
8
|
import {
|
|
4
9
|
commitFencedEngineWrite,
|
|
5
10
|
commitFencedEngineWriteAllowingPreconditionFailure
|
|
6
11
|
} from "../fenced-write.js";
|
|
12
|
+
import { WorkflowRevisionUnavailableError } from "../revision-errors.js";
|
|
7
13
|
import {
|
|
8
14
|
commitWithWorkflowClaimFold,
|
|
9
15
|
prepareWorkflowClaimFold,
|
|
10
16
|
throwWorkflowClaimUnavailable
|
|
11
17
|
} from "../workflow-claim-fold.js";
|
|
12
18
|
import { buildStartBatchOperations } from "./start-batch.js";
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
constructor() {
|
|
16
|
-
super("start idempotency compare-and-swap lost to a concurrent caller");
|
|
17
|
-
this.name = "StartIdempotencyRaceLostError";
|
|
18
|
-
}
|
|
19
|
-
}
|
|
19
|
+
import { StartIdempotencyRaceLostError } from "./start-commit-errors.js";
|
|
20
|
+
import { attributeLostStartPreconditionOrRetry } from "./start-precondition-attribution.js";
|
|
20
21
|
const WORKFLOW_CONCURRENCY_ADMISSION_MAX_ATTEMPTS = 5;
|
|
22
|
+
function needsCatalogEntryStartPrecondition(internals, state) {
|
|
23
|
+
return internals.options.ownershipMode !== "none" && state.revision !== void 0;
|
|
24
|
+
}
|
|
25
|
+
async function buildCatalogEntryStartPrecondition(internals, state) {
|
|
26
|
+
return {
|
|
27
|
+
source: "catalog-entry",
|
|
28
|
+
condition: await buildCatalogEntryRevisionCondition(internals, state.type, state.revision)
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
export async function buildCatalogEntryRevisionCondition(internals, type, revision) {
|
|
32
|
+
const key = KEYS.catalogEntry(type, revision), entryBytes = await internals.storage.get(key);
|
|
33
|
+
if (entryBytes === null)
|
|
34
|
+
throw new WorkflowRevisionUnavailableError(type, revision, "not-installed");
|
|
35
|
+
return { key, expectedValue: entryBytes };
|
|
36
|
+
}
|
|
21
37
|
async function persistStartBatch(internals, workflowId, startOperations, conditions, claimFold, isDelayedStart) {
|
|
22
38
|
if (claimFold) {
|
|
23
39
|
const result = await commitWithWorkflowClaimFold(internals, claimFold, startOperations, conditions.map((entry) => entry.condition), "workflow claim acquisition");
|
|
@@ -33,9 +49,19 @@ async function persistStartBatch(internals, workflowId, startOperations, conditi
|
|
|
33
49
|
requireStorageCapability(internals.storage, "conditionalBatch", "start preconditions");
|
|
34
50
|
return await commitFencedEngineWriteAllowingPreconditionFailure(internals, fenceWorkflowId, startOperations, conditions.map((entry) => entry.condition)) ? "committed" : "precondition-lost";
|
|
35
51
|
}
|
|
36
|
-
async function
|
|
52
|
+
async function hasStartConditionConflict(internals, conditions, sources) {
|
|
53
|
+
for (const entry of conditions) {
|
|
54
|
+
if (!sources.includes(entry.source))
|
|
55
|
+
continue;
|
|
56
|
+
const currentValue = await internals.storage.get(entry.condition.key);
|
|
57
|
+
if (!storageValuesEqual(currentValue, entry.condition.expectedValue))
|
|
58
|
+
return !0;
|
|
59
|
+
}
|
|
60
|
+
return !1;
|
|
61
|
+
}
|
|
62
|
+
async function hasCatalogEntryConflict(internals, conditions) {
|
|
37
63
|
for (const entry of conditions) {
|
|
38
|
-
if (entry.source !== "
|
|
64
|
+
if (entry.source !== "catalog-entry")
|
|
39
65
|
continue;
|
|
40
66
|
const currentValue = await internals.storage.get(entry.condition.key);
|
|
41
67
|
if (!storageValuesEqual(currentValue, entry.condition.expectedValue))
|
|
@@ -49,6 +75,14 @@ function tagStartPreconditions(conditions) {
|
|
|
49
75
|
condition
|
|
50
76
|
}));
|
|
51
77
|
}
|
|
78
|
+
function tagDuplicateIdConditions(condition, generationCondition) {
|
|
79
|
+
const tagged = [];
|
|
80
|
+
if (condition !== void 0)
|
|
81
|
+
tagged.push({ source: "duplicate-id", condition });
|
|
82
|
+
if (generationCondition !== void 0)
|
|
83
|
+
tagged.push({ source: "duplicate-id-generation", condition: generationCondition });
|
|
84
|
+
return tagged;
|
|
85
|
+
}
|
|
52
86
|
function tagWorkflowConcurrencyConditions(conditions) {
|
|
53
87
|
return conditions.map((condition) => ({
|
|
54
88
|
source: "workflow-concurrency",
|
|
@@ -66,18 +100,26 @@ export async function buildAndCommitStartBatch(context, buildIdempotentStartOper
|
|
|
66
100
|
for (let attempt = 0;attempt < maxAttempts; attempt += 1) {
|
|
67
101
|
const idempotent = buildIdempotentStartOperations?.(workflowId), workflowConcurrency = await context.buildWorkflowConcurrencyStartOperations?.();
|
|
68
102
|
lastWorkflowConcurrencyStateKey = workflowConcurrency?.stateKey ?? lastWorkflowConcurrencyStateKey;
|
|
69
|
-
const startOperations = buildStartBatchOperations(internals, workflowId, state, checkpoint, registration, options, state.executionDeadline, context.delayedStartTimer, context.persistedWorkflowStartHeaders, mergeAdditionalStartOperations(context.additionalStartOperations, mergeAdditionalStartOperations(idempotent?.operations, workflowConcurrency?.operations)), context.callbacks, context.purgeDeleteOperations), conditions = [
|
|
103
|
+
const catalogEntryPrecondition = needsCatalogEntryStartPrecondition(internals, state) ? await buildCatalogEntryStartPrecondition(internals, state) : void 0, startOperations = buildStartBatchOperations(internals, workflowId, state, checkpoint, registration, options, state.executionDeadline, context.delayedStartTimer, context.persistedWorkflowStartHeaders, mergeAdditionalStartOperations(context.additionalStartOperations, mergeAdditionalStartOperations(idempotent?.operations, workflowConcurrency?.operations)), context.callbacks, context.purgeDeleteOperations), conditions = [
|
|
70
104
|
...tagStartPreconditions(idempotent?.conditions),
|
|
71
|
-
...
|
|
105
|
+
...tagDuplicateIdConditions(context.duplicateIdCondition, context.duplicateIdGenerationCondition),
|
|
106
|
+
...tagWorkflowConcurrencyConditions(workflowConcurrency?.conditions ?? []),
|
|
107
|
+
...catalogEntryPrecondition === void 0 ? [] : [catalogEntryPrecondition]
|
|
72
108
|
], isDelayedStart = context.delayedStartTimer !== void 0, claimFold = isDelayedStart ? void 0 : await prepareWorkflowClaimFold(internals, workflowId), outcome = await persistStartBatch(internals, workflowId, startOperations, conditions, claimFold, isDelayedStart);
|
|
73
109
|
if (outcome === "committed")
|
|
74
110
|
return;
|
|
75
|
-
if (await
|
|
111
|
+
if (outcome !== "claim-lost" && await hasStartConditionConflict(internals, conditions, [
|
|
112
|
+
"duplicate-id",
|
|
113
|
+
"duplicate-id-generation"
|
|
114
|
+
]))
|
|
115
|
+
throw new WorkflowAlreadyExistsError(workflowId);
|
|
116
|
+
if (await hasStartConditionConflict(internals, conditions, ["start-precondition"]))
|
|
76
117
|
throw new StartIdempotencyRaceLostError;
|
|
118
|
+
if (await hasCatalogEntryConflict(internals, conditions))
|
|
119
|
+
throw new WorkflowRevisionUnavailableError(state.type, state.revision, "not-installed");
|
|
77
120
|
if (outcome === "claim-lost")
|
|
78
121
|
return throwWorkflowClaimUnavailable(internals, workflowId);
|
|
79
|
-
|
|
80
|
-
throw new StartIdempotencyRaceLostError;
|
|
122
|
+
await attributeLostStartPreconditionOrRetry(workflowId, context.duplicateIdCondition !== void 0, workflowConcurrency !== void 0, () => hasStartConditionConflict(internals, conditions, ["workflow-concurrency"]));
|
|
81
123
|
}
|
|
82
124
|
throw new AtomicStateConflictError(lastWorkflowConcurrencyStateKey ?? "workflow concurrency admission", WORKFLOW_CONCURRENCY_ADMISSION_MAX_ATTEMPTS);
|
|
83
125
|
}
|
|
@@ -2,8 +2,24 @@ import type { Checkpoint, StartOptions, WorkflowState } from '../../types.ts';
|
|
|
2
2
|
import type { EngineInternals } from '../internals.ts';
|
|
3
3
|
import { type LifecycleCallbacks, type RegistrationEntry } from './shared.ts';
|
|
4
4
|
export declare function runWorkflowStartInterceptor(_internals: EngineInternals, workflowId: string, workflowType: string, input: unknown, parentHeaders: Map<string, string> | undefined, callbacks: LifecycleCallbacks): Map<string, string> | undefined;
|
|
5
|
-
export declare function startWorkflowExecution(internals: EngineInternals, workflowId: string, workflowExecutionToken: string | undefined, workflowType: string,
|
|
6
|
-
|
|
5
|
+
export declare function startWorkflowExecution(internals: EngineInternals, workflowId: string, workflowExecutionToken: string | undefined, workflowType: string,
|
|
6
|
+
/**
|
|
7
|
+
* The starting run's persisted `WorkflowState.revision` (WFT-19/WFT-20):
|
|
8
|
+
* populates the per-instance identity cache readers use for revision-scoped
|
|
9
|
+
* routing, and is echoed to the strategy's `startWorkflow` for worker-protocol
|
|
10
|
+
* revision validation.
|
|
11
|
+
*/
|
|
12
|
+
revision: string | undefined, input: unknown, checkpoint: Checkpoint, nestingDepth: number, executionDeadline: number | undefined, executionStateOwnerId: string, _callbacks?: LifecycleCallbacks): void;
|
|
13
|
+
export declare function beginWorkflowExecution(internals: EngineInternals, workflowId: string, workflowExecutionToken: string | undefined, workflowType: string,
|
|
14
|
+
/**
|
|
15
|
+
* The starting run's persisted `WorkflowState.revision` (WFT-19/WFT-20).
|
|
16
|
+
* Threaded into both the queued-inline path below (which also re-reads
|
|
17
|
+
* `revision` off the RELOADED `WorkflowState` at flush time —
|
|
18
|
+
* `inline-launch-queue.ts` — exactly like it already does for
|
|
19
|
+
* `workflowExecutionToken`) and the non-inline `startWorkflowExecution`
|
|
20
|
+
* branch, which populates the per-instance identity cache with it directly.
|
|
21
|
+
*/
|
|
22
|
+
revision: string | undefined, input: unknown, checkpoint: Checkpoint, executionDeadline: number | undefined, executionStateOwnerId: string, _registration: RegistrationEntry, callbacks: LifecycleCallbacks, onStarted?: () => void): void;
|
|
7
23
|
/**
|
|
8
24
|
* `defer: false` is an inline-only liveness gate: it awaits the moment the
|
|
9
25
|
* generator is first driven. A worker-mode start queues to the Worker transport
|