@lostgradient/weft 0.19.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/cli/generated/operation-catalog.snapshot.json +1334 -0
- package/dist/cli/generated/operation-client.generated.d.ts +9 -2
- package/dist/cli/generated/operation-client.generated.js +2 -0
- package/dist/core/engine/activity-reconciliation.d.ts +2 -2
- package/dist/core/engine/activity-reconciliation.js +12 -8
- package/dist/core/engine/async-activity-completion.js +12 -4
- package/dist/core/engine/async-activity-records.d.ts +11 -1
- package/dist/core/engine/async-activity-records.js +4 -3
- package/dist/core/engine/attributes-tags.d.ts +11 -1
- package/dist/core/engine/attributes-tags.js +4 -3
- package/dist/core/engine/bulk-operations-purge.js +5 -2
- package/dist/core/engine/bulk-operations.js +17 -5
- package/dist/core/engine/callback-creators-bundles.js +1 -1
- package/dist/core/engine/callback-creators-core.js +4 -2
- package/dist/core/engine/checkpoint-io.js +1 -1
- package/dist/core/engine/child-workflow.js +2 -1
- package/dist/core/engine/completed-review-storage.js +1 -1
- package/dist/core/engine/condition-waiters.d.ts +45 -3
- package/dist/core/engine/condition-waiters.js +12 -3
- package/dist/core/engine/disposal.js +6 -0
- package/dist/core/engine/engine-internal-types.d.ts +12 -4
- package/dist/core/engine/fenced-write.d.ts +46 -15
- package/dist/core/engine/fenced-write.js +39 -6
- package/dist/core/engine/finalizer-state.d.ts +3 -1
- package/dist/core/engine/handle-result.d.ts +49 -1
- package/dist/core/engine/handle-result.js +95 -20
- package/dist/core/engine/index.d.ts +13 -5
- package/dist/core/engine/index.js +155 -5
- package/dist/core/engine/inline-launch-queue.js +9 -0
- package/dist/core/engine/internals.d.ts +73 -0
- package/dist/core/engine/invoke-update-handler.d.ts +19 -0
- package/dist/core/engine/invoke-update-handler.js +7 -0
- package/dist/core/engine/lease-deposition.d.ts +132 -0
- package/dist/core/engine/lease-deposition.js +31 -0
- package/dist/core/engine/lease-errors.d.ts +80 -9
- package/dist/core/engine/lease-errors.js +28 -2
- package/dist/core/engine/lifecycle/resume.js +3 -1
- package/dist/core/engine/lifecycle/standalone-claim-acquire.d.ts +33 -0
- package/dist/core/engine/lifecycle/standalone-claim-acquire.js +25 -0
- package/dist/core/engine/lifecycle/start-commit.js +20 -6
- package/dist/core/engine/lifecycle/transition.d.ts +15 -1
- package/dist/core/engine/lifecycle/transition.js +6 -3
- package/dist/core/engine/operations-activity.js +1 -1
- package/dist/core/engine/operations-time.js +21 -7
- package/dist/core/engine/owner-side-signal-poll.d.ts +132 -0
- package/dist/core/engine/owner-side-signal-poll.js +22 -0
- package/dist/core/engine/owner-side-update-poll.d.ts +116 -0
- package/dist/core/engine/owner-side-update-poll.js +22 -0
- package/dist/core/engine/ownership-bootstrap.d.ts +280 -0
- package/dist/core/engine/ownership-bootstrap.js +89 -0
- package/dist/core/engine/ownership-mode-marker.d.ts +82 -0
- package/dist/core/engine/ownership-mode-marker.js +52 -0
- package/dist/core/engine/ownership-options.d.ts +22 -6
- package/dist/core/engine/ownership-options.js +42 -12
- package/dist/core/engine/pending-updates.js +35 -24
- package/dist/core/engine/queries.d.ts +70 -0
- package/dist/core/engine/queries.js +58 -4
- package/dist/core/engine/schedules.js +1 -1
- package/dist/core/engine/signals.d.ts +11 -1
- package/dist/core/engine/signals.js +11 -7
- package/dist/core/engine/sleep-timer-acknowledgements.d.ts +68 -0
- package/dist/core/engine/sleep-timer-acknowledgements.js +19 -0
- package/dist/core/engine/storage-io.d.ts +58 -10
- package/dist/core/engine/storage-io.js +27 -9
- package/dist/core/engine/termination/cleanup.d.ts +5 -4
- package/dist/core/engine/termination/complete.js +23 -4
- package/dist/core/engine/termination/finalizer-claim.js +4 -4
- package/dist/core/engine/termination/state-commit-callbacks.d.ts +15 -0
- package/dist/core/engine/termination/state-commit-callbacks.js +0 -0
- package/dist/core/engine/termination/suspend.js +1 -1
- package/dist/core/engine/update-validation.d.ts +28 -0
- package/dist/core/engine/update-validation.js +37 -0
- package/dist/core/engine/updates.d.ts +17 -16
- package/dist/core/engine/updates.js +24 -47
- package/dist/core/engine/wake-ownership-check.d.ts +86 -0
- package/dist/core/engine/wake-ownership-check.js +19 -0
- package/dist/core/engine/wake-ownership-guard.d.ts +46 -0
- package/dist/core/engine/wake-ownership-guard.js +23 -0
- package/dist/core/engine/workflow-claim-codec.d.ts +64 -0
- package/dist/core/engine/workflow-claim-codec.js +44 -0
- package/dist/core/engine/workflow-claim-cooldown.d.ts +60 -0
- package/dist/core/engine/workflow-claim-cooldown.js +17 -0
- package/dist/core/engine/workflow-claim-fold.d.ts +82 -0
- package/dist/core/engine/workflow-claim-fold.js +53 -0
- package/dist/core/engine/workflow-claim-metrics.d.ts +131 -0
- package/dist/core/engine/workflow-claim-metrics.js +62 -0
- package/dist/core/engine/workflow-claim-reclaim-scan.d.ts +96 -0
- package/dist/core/engine/workflow-claim-reclaim-scan.js +66 -0
- package/dist/core/engine/workflow-claim-reclaim-target.d.ts +107 -0
- package/dist/core/engine/workflow-claim-reclaim-target.js +155 -0
- package/dist/core/engine/workflow-claim-registry.d.ts +227 -0
- package/dist/core/engine/workflow-claim-registry.js +198 -0
- package/dist/core/engine/workflow-claim-renewal-interval.d.ts +50 -0
- package/dist/core/engine/workflow-claim-renewal-interval.js +132 -0
- package/dist/core/engine/workflow-claim-renewal-subpasses.d.ts +265 -0
- package/dist/core/engine/workflow-claim-renewal-subpasses.js +104 -0
- package/dist/core/engine/workflow-claim-renewal-task.d.ts +159 -0
- package/dist/core/engine/workflow-claim-renewal-task.js +65 -0
- package/dist/core/engine/workflow-claim-transitions.d.ts +186 -0
- package/dist/core/engine/workflow-claim-transitions.js +120 -0
- package/dist/core/inline-execution-strategy.d.ts +20 -0
- package/dist/core/inline-execution-strategy.js +15 -4
- package/dist/core/scheduler/scheduler-class.js +3 -3
- package/dist/core/scheduler/timer-sources.d.ts +13 -0
- package/dist/core/scheduler/timer-sources.js +1 -1
- package/dist/core/types/options.d.ts +17 -1
- package/dist/core/weft-error.d.ts +1 -1
- package/dist/core/weft-error.js +2 -1
- package/dist/http.js +2 -2
- package/dist/index.d.ts +2 -1
- package/dist/index.js +8 -0
- package/dist/indexeddb.js +1 -1
- package/dist/json-schema.js +3 -3
- package/dist/observability/index.js +2 -2
- package/dist/server/operations/get-task-detail-schema.d.ts +341 -0
- package/dist/server/operations/get-task-detail-schema.js +118 -0
- package/dist/server/operations/get-task-detail.d.ts +346 -0
- package/dist/server/operations/get-task-detail.js +215 -0
- package/dist/server/operations/static-registrations.js +3 -0
- package/dist/storage/auto.js +1 -1
- package/dist/storage/bun-sql.js +113 -0
- package/dist/storage/compressed-storage.js +1 -1
- package/dist/storage/index.d.ts +38 -35
- package/dist/storage/interface.d.ts +87 -125
- package/dist/storage/interface.js +1 -1
- package/dist/storage/key-encoding.d.ts +51 -0
- package/dist/storage/key-encoding.js +13 -0
- package/dist/storage/key-prefixes.d.ts +1 -1
- package/dist/storage/key-prefixes.js +3 -0
- package/dist/storage/lmdb.js +1 -1
- package/dist/storage/memory.js +1 -1
- package/dist/storage/neon.js +2 -2
- package/dist/storage/node-sqlite.js +113 -0
- package/dist/storage/ownership-keys.d.ts +55 -0
- package/dist/storage/ownership-keys.js +6 -0
- package/dist/storage/postgres.js +2 -2
- package/dist/storage/resolve.js +1 -1
- package/dist/storage/scoped-storage.js +1 -1
- package/dist/storage/testing.js +1 -1
- package/dist/storage/turso.js +2 -2
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/web-extension.js +1 -1
- package/dist/worker/protocol.js +1 -1
- package/package.json +1 -1
|
@@ -4,21 +4,37 @@
|
|
|
4
4
|
* file under the `max-lines` ceiling; mirrors the second-instance resolver's
|
|
5
5
|
* "only validate when the feature is enabled" discipline.
|
|
6
6
|
*
|
|
7
|
+
* `ownership` is a three-member discriminated union — `'none'`, `'lease'`
|
|
8
|
+
* (a single store-wide lease), and `'workflow-lease'` (per-workflow fencing,
|
|
9
|
+
* ADR 0002 at
|
|
10
|
+
* `documentation/contributing/architecture-decisions/0002-multiengine-per-workflow-ownership.md`).
|
|
11
|
+
* This module resolves and validates the option surface only: it resolves the
|
|
12
|
+
* `workflowClaimTtl`/`workflowClaimRenewInterval` tuning fields and rejects
|
|
13
|
+
* incoherent configuration. The fencing behavior itself lives in the claim
|
|
14
|
+
* registry, the ownership bootstrap, and the fenced-write path.
|
|
15
|
+
*
|
|
7
16
|
* @module core/engine/ownership-options
|
|
8
17
|
*/
|
|
9
18
|
import type { EngineConstructorOptions, ResolvedOptions } from './engine-internal-types.ts';
|
|
19
|
+
import { WORKFLOW_CLAIM_TTL_SAFETY_MULTIPLIER } from './workflow-claim-transitions.ts';
|
|
10
20
|
/** Default lease time-to-live for `ownership: 'lease'`. */
|
|
11
21
|
export declare const DEFAULT_LEASE_TTL_MS = 30000;
|
|
12
22
|
/** Default lease renewal interval for `ownership: 'lease'`. */
|
|
13
23
|
export declare const DEFAULT_LEASE_RENEW_INTERVAL_MS = 5000;
|
|
14
24
|
/** Default boot-time lease acquisition wait window for `ownership: 'lease'`. */
|
|
15
25
|
export declare const DEFAULT_LEASE_WAIT_TIMEOUT_MS = 60000;
|
|
26
|
+
/** Default per-workflow claim time-to-live for `ownership: 'workflow-lease'`. */
|
|
27
|
+
export declare const DEFAULT_WORKFLOW_CLAIM_TTL_MS = 30000;
|
|
28
|
+
/** Default per-workflow claim renewal interval for `ownership: 'workflow-lease'`. */
|
|
29
|
+
export declare const DEFAULT_WORKFLOW_CLAIM_RENEW_INTERVAL_MS = 5000;
|
|
30
|
+
export { WORKFLOW_CLAIM_TTL_SAFETY_MULTIPLIER };
|
|
16
31
|
export declare function resolveBackgroundTaskMode(options: EngineConstructorOptions | undefined): ResolvedOptions['backgroundTaskMode'];
|
|
17
32
|
/**
|
|
18
|
-
* Resolve the ownership posture and
|
|
19
|
-
* fields. Defaults to `'none'`. The lease
|
|
20
|
-
*
|
|
21
|
-
* able to throw on an invalid value) when
|
|
22
|
-
* an invalid duration must not make an off-by-
|
|
33
|
+
* Resolve the ownership posture and its per-mode tuning into their
|
|
34
|
+
* `ResolvedOptions` fields. Defaults to `'none'`. The lease and workflow-claim
|
|
35
|
+
* durations are documented as "ignored" outside their own mode, so they are
|
|
36
|
+
* only normalized (and thus only able to throw on an invalid value) when that
|
|
37
|
+
* mode is actually selected — an invalid duration must not make an off-by-
|
|
38
|
+
* default config fatal at boot.
|
|
23
39
|
*/
|
|
24
|
-
export declare function resolveOwnershipFields(options: EngineConstructorOptions | undefined): Pick<ResolvedOptions, 'ownershipMode' | 'leaseTtlMs' | 'leaseRenewIntervalMs' | 'leaseWaitTimeoutMs'>;
|
|
40
|
+
export declare function resolveOwnershipFields(options: EngineConstructorOptions | undefined): Pick<ResolvedOptions, 'ownershipMode' | 'leaseTtlMs' | 'leaseRenewIntervalMs' | 'leaseWaitTimeoutMs' | 'workflowClaimTtlMs' | 'workflowClaimRenewIntervalMs'>;
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import { normalizeRetentionDuration } from "./validation.js";
|
|
2
|
-
|
|
2
|
+
import { WORKFLOW_CLAIM_TTL_SAFETY_MULTIPLIER } from "./workflow-claim-transitions.js";
|
|
3
|
+
export const DEFAULT_LEASE_TTL_MS = 30000, DEFAULT_LEASE_RENEW_INTERVAL_MS = 5000, DEFAULT_LEASE_WAIT_TIMEOUT_MS = 60000, DEFAULT_WORKFLOW_CLAIM_TTL_MS = 30000, DEFAULT_WORKFLOW_CLAIM_RENEW_INTERVAL_MS = 5000;
|
|
4
|
+
|
|
5
|
+
export { WORKFLOW_CLAIM_TTL_SAFETY_MULTIPLIER };
|
|
3
6
|
export function resolveBackgroundTaskMode(options) {
|
|
4
7
|
const mode = options?.backgroundTasks ?? "automatic";
|
|
5
8
|
if (mode !== "automatic" && mode !== "manual")
|
|
@@ -15,26 +18,53 @@ function assertManualBackgroundTaskCompatibility(options) {
|
|
|
15
18
|
throw Error('ownership cannot be "lease" when backgroundTasks is "manual"');
|
|
16
19
|
}
|
|
17
20
|
export function resolveOwnershipFields(options) {
|
|
18
|
-
const ownershipMode = assertKnownOwnershipMode(options?.ownership ?? "none")
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
21
|
+
const ownershipMode = assertKnownOwnershipMode(options?.ownership ?? "none"), leaseDefaults = {
|
|
22
|
+
leaseTtlMs: DEFAULT_LEASE_TTL_MS,
|
|
23
|
+
leaseRenewIntervalMs: DEFAULT_LEASE_RENEW_INTERVAL_MS,
|
|
24
|
+
leaseWaitTimeoutMs: DEFAULT_LEASE_WAIT_TIMEOUT_MS
|
|
25
|
+
}, workflowClaimDefaults = {
|
|
26
|
+
workflowClaimTtlMs: DEFAULT_WORKFLOW_CLAIM_TTL_MS,
|
|
27
|
+
workflowClaimRenewIntervalMs: DEFAULT_WORKFLOW_CLAIM_RENEW_INTERVAL_MS
|
|
28
|
+
};
|
|
29
|
+
if (ownershipMode === "lease")
|
|
30
|
+
return { ownershipMode, ...resolveLeaseTiming(options), ...workflowClaimDefaults };
|
|
31
|
+
if (ownershipMode === "workflow-lease") {
|
|
32
|
+
assertWorkflowLeaseCompatibleWithSecondInstanceDetection(options);
|
|
33
|
+
return { ownershipMode, ...leaseDefaults, ...resolveWorkflowClaimTiming(options) };
|
|
34
|
+
}
|
|
35
|
+
return { ownershipMode, ...leaseDefaults, ...workflowClaimDefaults };
|
|
36
|
+
}
|
|
37
|
+
function resolveLeaseTiming(options) {
|
|
26
38
|
const leaseTtlMs = normalizeRetentionDuration(options?.leaseTtl, "options.leaseTtl") ?? DEFAULT_LEASE_TTL_MS, leaseRenewIntervalMs = normalizeRetentionDuration(options?.leaseRenewInterval, "options.leaseRenewInterval") ?? DEFAULT_LEASE_RENEW_INTERVAL_MS, leaseWaitTimeoutMs = normalizeRetentionDuration(options?.leaseWaitTimeout, "options.leaseWaitTimeout") ?? DEFAULT_LEASE_WAIT_TIMEOUT_MS;
|
|
27
39
|
assertLeaseTimingCoherent(leaseTtlMs, leaseRenewIntervalMs, leaseWaitTimeoutMs);
|
|
28
|
-
return {
|
|
40
|
+
return { leaseTtlMs, leaseRenewIntervalMs, leaseWaitTimeoutMs };
|
|
41
|
+
}
|
|
42
|
+
function resolveWorkflowClaimTiming(options) {
|
|
43
|
+
const workflowClaimTtlMs = normalizeRetentionDuration(options?.workflowClaimTtl, "options.workflowClaimTtl") ?? DEFAULT_WORKFLOW_CLAIM_TTL_MS, workflowClaimRenewIntervalMs = normalizeRetentionDuration(options?.workflowClaimRenewInterval, "options.workflowClaimRenewInterval") ?? DEFAULT_WORKFLOW_CLAIM_RENEW_INTERVAL_MS;
|
|
44
|
+
assertWorkflowClaimTimingCoherent(workflowClaimTtlMs, workflowClaimRenewIntervalMs);
|
|
45
|
+
return { workflowClaimTtlMs, workflowClaimRenewIntervalMs };
|
|
29
46
|
}
|
|
30
47
|
function assertKnownOwnershipMode(mode) {
|
|
31
|
-
if (mode !== "none" && mode !== "lease")
|
|
32
|
-
throw Error(`Unknown ownership posture "${
|
|
48
|
+
if (mode !== "none" && mode !== "lease" && mode !== "workflow-lease")
|
|
49
|
+
throw Error(`Unknown ownership posture "${mode}". Expected 'none', 'lease', or 'workflow-lease'.`);
|
|
33
50
|
return mode;
|
|
34
51
|
}
|
|
52
|
+
function assertWorkflowLeaseCompatibleWithSecondInstanceDetection(options) {
|
|
53
|
+
if (options?.detectSecondInstance === !0)
|
|
54
|
+
throw Error("ownership: 'workflow-lease' cannot be combined with detectSecondInstance: true " + "\u2014 second-instance detection assumes a single global owner.");
|
|
55
|
+
}
|
|
35
56
|
function assertLeaseTimingCoherent(leaseTtlMs, leaseRenewIntervalMs, leaseWaitTimeoutMs) {
|
|
36
57
|
if (leaseTtlMs <= 0 || leaseRenewIntervalMs <= 0 || leaseWaitTimeoutMs <= 0)
|
|
37
58
|
throw Error(`ownership: 'lease' requires positive leaseTtl, leaseRenewInterval, and leaseWaitTimeout (got leaseTtl=${leaseTtlMs}ms, leaseRenewInterval=${leaseRenewIntervalMs}ms, leaseWaitTimeout=${leaseWaitTimeoutMs}ms).`);
|
|
38
59
|
if (leaseRenewIntervalMs >= leaseTtlMs)
|
|
39
60
|
throw Error(`ownership: 'lease' requires leaseRenewInterval (${leaseRenewIntervalMs}ms) to be strictly less than leaseTtl (${leaseTtlMs}ms), so a renewal fires before the lease can lapse.`);
|
|
40
61
|
}
|
|
62
|
+
function assertWorkflowClaimTimingCoherent(workflowClaimTtlMs, workflowClaimRenewIntervalMs) {
|
|
63
|
+
if (workflowClaimTtlMs <= 0 || workflowClaimRenewIntervalMs <= 0)
|
|
64
|
+
throw Error(`ownership: 'workflow-lease' requires positive workflowClaimTtl and workflowClaimRenewInterval (got workflowClaimTtl=${workflowClaimTtlMs}ms, workflowClaimRenewInterval=${workflowClaimRenewIntervalMs}ms).`);
|
|
65
|
+
if (workflowClaimRenewIntervalMs >= workflowClaimTtlMs)
|
|
66
|
+
throw Error(`ownership: 'workflow-lease' requires workflowClaimRenewInterval (${workflowClaimRenewIntervalMs}ms) to be strictly less than workflowClaimTtl (${workflowClaimTtlMs}ms), so a renewal fires before the claim can lapse.`);
|
|
67
|
+
const requiredTtlMs = WORKFLOW_CLAIM_TTL_SAFETY_MULTIPLIER * workflowClaimRenewIntervalMs;
|
|
68
|
+
if (workflowClaimTtlMs < requiredTtlMs)
|
|
69
|
+
throw Error(`ownership: 'workflow-lease' requires workflowClaimTtl (${workflowClaimTtlMs}ms) to be at least WORKFLOW_CLAIM_TTL_SAFETY_MULTIPLIER (${WORKFLOW_CLAIM_TTL_SAFETY_MULTIPLIER}) times workflowClaimRenewInterval (${workflowClaimRenewIntervalMs}ms), i.e. at least ${requiredTtlMs}ms, so a stalled renewal has margin before the claim can be taken over.`);
|
|
70
|
+
}
|
|
@@ -2,10 +2,9 @@ import { UpdateCompletedEvent, UpdateReceivedEvent } from "../events.js";
|
|
|
2
2
|
import { UpdateValidationError } from "../updates.js";
|
|
3
3
|
import { notifyConditionWaiters } from "./condition-waiters.js";
|
|
4
4
|
import { commitFencedEngineWrite } from "./fenced-write.js";
|
|
5
|
-
import {
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
} from "./updates.js";
|
|
5
|
+
import { invokeUpdateHandler as invokeUpdateHandlerFromInternals } from "./invoke-update-handler.js";
|
|
6
|
+
import { isLiveContextStale } from "./queries.js";
|
|
7
|
+
import { extractStandardSchemaIssues } from "./update-validation.js";
|
|
9
8
|
async function waitForNextMacrotask() {
|
|
10
9
|
await new Promise((resolve) => {
|
|
11
10
|
setTimeout(resolve, 0);
|
|
@@ -63,32 +62,44 @@ async function runPendingUpdateValidator(validator, updateName, payload) {
|
|
|
63
62
|
return new UpdateValidationError(updateName, issues).message;
|
|
64
63
|
return null;
|
|
65
64
|
}
|
|
65
|
+
async function resolveLiveHandlerContext(internals, workflowId) {
|
|
66
|
+
const stale = isLiveContextStale(internals, workflowId);
|
|
67
|
+
if (stale !== !1 && await stale)
|
|
68
|
+
return;
|
|
69
|
+
return internals.inlineStrategy?.getContext(workflowId);
|
|
70
|
+
}
|
|
71
|
+
async function processOnePendingUpdate(internals, workflowId, update, handlers, context, callbacks) {
|
|
72
|
+
const handler = handlers.get(update.name);
|
|
73
|
+
if (!handler)
|
|
74
|
+
return !1;
|
|
75
|
+
if (!claimPendingUpdateForDelivery(internals, workflowId, update.updateId))
|
|
76
|
+
return !1;
|
|
77
|
+
const validatorRejectionError = await runValidatorIfPresent(context, update);
|
|
78
|
+
if (validatorRejectionError !== null) {
|
|
79
|
+
await rejectPendingUpdate(internals, workflowId, update, validatorRejectionError, callbacks);
|
|
80
|
+
return !1;
|
|
81
|
+
}
|
|
82
|
+
await deliverPendingUpdate(internals, workflowId, update, handler, callbacks);
|
|
83
|
+
return !0;
|
|
84
|
+
}
|
|
66
85
|
export async function processPendingUpdatesForHandlers(internals, workflowId, callbacks) {
|
|
67
|
-
const
|
|
86
|
+
const initialContext = internals.inlineStrategy?.getContext(workflowId);
|
|
87
|
+
if (!initialContext || initialContext.updateHandlers.size === 0)
|
|
88
|
+
return;
|
|
89
|
+
const pendingUpdates = await internals.updateCoordinator.getPendingUpdates(workflowId);
|
|
90
|
+
if (pendingUpdates.length === 0)
|
|
91
|
+
return;
|
|
92
|
+
const context = await resolveLiveHandlerContext(internals, workflowId);
|
|
68
93
|
if (!context)
|
|
69
94
|
return;
|
|
70
95
|
const handlers = context.updateHandlers;
|
|
71
96
|
if (handlers.size === 0)
|
|
72
97
|
return;
|
|
73
|
-
const pendingUpdates = await internals.updateCoordinator.getPendingUpdates(workflowId);
|
|
74
|
-
if (pendingUpdates.length === 0)
|
|
75
|
-
return;
|
|
76
98
|
let handlerRan = !1;
|
|
77
99
|
try {
|
|
78
|
-
for (const update of pendingUpdates)
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
continue;
|
|
82
|
-
if (!claimPendingUpdateForDelivery(internals, workflowId, update.updateId))
|
|
83
|
-
continue;
|
|
84
|
-
const validatorRejectionError = await runValidatorIfPresent(context, update);
|
|
85
|
-
if (validatorRejectionError !== null) {
|
|
86
|
-
await rejectPendingUpdate(internals, workflowId, update, validatorRejectionError, callbacks);
|
|
87
|
-
continue;
|
|
88
|
-
}
|
|
89
|
-
handlerRan = !0;
|
|
90
|
-
await deliverPendingUpdate(internals, workflowId, update, handler, callbacks);
|
|
91
|
-
}
|
|
100
|
+
for (const update of pendingUpdates)
|
|
101
|
+
if (await processOnePendingUpdate(internals, workflowId, update, handlers, context, callbacks))
|
|
102
|
+
handlerRan = !0;
|
|
92
103
|
} finally {
|
|
93
104
|
if (handlerRan)
|
|
94
105
|
notifyConditionWaiters(internals, workflowId);
|
|
@@ -113,7 +124,7 @@ async function runValidatorIfPresent(context, update) {
|
|
|
113
124
|
}
|
|
114
125
|
async function rejectPendingUpdate(internals, workflowId, update, errorMessage, callbacks) {
|
|
115
126
|
const responseOperations = internals.updateCoordinator.buildResponseOperations(update.updateId, workflowId, void 0, errorMessage, update.idempotencyKey);
|
|
116
|
-
await commitFencedEngineWrite(internals, responseOperations, [], () => Error(`Update rejection for workflow "${workflowId}" lost its CAS race.`));
|
|
127
|
+
await commitFencedEngineWrite(internals, workflowId, responseOperations, [], () => Error(`Update rejection for workflow "${workflowId}" lost its CAS race.`));
|
|
117
128
|
callbacks.broadcast({ type: "update:completed", workflowId, updateId: update.updateId });
|
|
118
129
|
}
|
|
119
130
|
async function deliverPendingUpdate(internals, workflowId, update, handler, callbacks) {
|
|
@@ -125,7 +136,7 @@ async function deliverPendingUpdate(internals, workflowId, update, handler, call
|
|
|
125
136
|
error = handlerError instanceof Error ? handlerError.message : String(handlerError);
|
|
126
137
|
}
|
|
127
138
|
const responseOperations = internals.updateCoordinator.buildResponseOperations(update.updateId, workflowId, result, error, update.idempotencyKey);
|
|
128
|
-
await commitFencedEngineWrite(internals, responseOperations, [], () => Error(`Update response for workflow "${workflowId}" lost its CAS race.`));
|
|
139
|
+
await commitFencedEngineWrite(internals, workflowId, responseOperations, [], () => Error(`Update response for workflow "${workflowId}" lost its CAS race.`));
|
|
129
140
|
callbacks.dispatchEvent(new UpdateCompletedEvent(update.updateId, workflowId, update.name, result, error));
|
|
130
141
|
callbacks.broadcast({ type: "update:completed", workflowId, updateId: update.updateId });
|
|
131
142
|
}
|
|
@@ -1,3 +1,73 @@
|
|
|
1
1
|
import type { EngineInternals } from './internals.ts';
|
|
2
|
+
/**
|
|
3
|
+
* Thrown by `query()` when this engine has no live local execution context for
|
|
4
|
+
* `workflowId` AND a durable `ownership: 'workflow-lease'` claim shows another
|
|
5
|
+
* engine currently holds it — distinguishing "not owned here, retry against
|
|
6
|
+
* the owner" from a query handler that legitimately resolved to `undefined`.
|
|
7
|
+
* A workflow that is simply terminal, purged, or never existed still resolves
|
|
8
|
+
* to plain `undefined`, unchanged — this error fires only when a live holder
|
|
9
|
+
* record names a DIFFERENT engine.
|
|
10
|
+
*
|
|
11
|
+
* Not a public `WeftError`: ADR 0002 § Open questions leaves the cross-engine
|
|
12
|
+
* query/update ROUTING mechanism open (forward to the true owner vs. surface
|
|
13
|
+
* a retry-elsewhere error to the caller); this is the smaller, non-speculative
|
|
14
|
+
* fix for the silent-`undefined` bug alone, reusing the same "untyped `Error`"
|
|
15
|
+
* shape Gate 1 (`requireStorageCapability`) already ships for a similarly
|
|
16
|
+
* not-yet-typed diagnostic rather than adding a new public error code and its
|
|
17
|
+
* full registration surface for a routing question ADR 0002 has not settled.
|
|
18
|
+
* `query()` has no durable fallback path the way `update()`'s coordinated
|
|
19
|
+
* path does (see `updates.ts`), so surfacing an error here — rather than a
|
|
20
|
+
* value indistinguishable from a real `undefined` result — is the only
|
|
21
|
+
* correct option.
|
|
22
|
+
*/
|
|
23
|
+
export declare class WorkflowNotLocallyOwnedError extends Error {
|
|
24
|
+
readonly workflowId: string;
|
|
25
|
+
constructor(workflowId: string);
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Cheap, read-only check for whether `workflowId` is durably claimed by a
|
|
29
|
+
* DIFFERENT engine under `ownership: 'workflow-lease'` — a
|
|
30
|
+
* `wakeOwnershipCheck`-style storage read (one `storage.get`, no
|
|
31
|
+
* `conditionalBatch`), not the actual safety mechanism. Always `false`
|
|
32
|
+
* (inert) under `ownership: 'none'`/`'lease'`, where no claim registry is
|
|
33
|
+
* installed.
|
|
34
|
+
*
|
|
35
|
+
* Always re-reads the durable holder rather than trusting this engine's own
|
|
36
|
+
* cached `registry.currentEpoch(workflowId)`: after a takeover elsewhere,
|
|
37
|
+
* this engine's local claim entry stays populated (and its cached epoch
|
|
38
|
+
* stale) until its next renewal CAS independently detects the loss — a
|
|
39
|
+
* `registry.currentEpoch(workflowId) !== null` shortcut would let a stale
|
|
40
|
+
* engine keep answering `false` (and so keep serving queries/updates from a
|
|
41
|
+
* deposed context) for up to a full renewal interval. Deciding by the
|
|
42
|
+
* durable holder's `engineId` alone — not epoch — is sufficient here: this
|
|
43
|
+
* engine reacquiring the same workflow at a new epoch after a release still
|
|
44
|
+
* means the holder names THIS engine, correctly answering `false`.
|
|
45
|
+
*/
|
|
46
|
+
export declare function isWorkflowClaimedByAnotherEngine(internals: EngineInternals, workflowId: string): Promise<boolean>;
|
|
47
|
+
/**
|
|
48
|
+
* `true` when `workflowId`'s persisted `WorkflowState` is missing (never
|
|
49
|
+
* existed, or already purged) or terminal. Used to ignore a durably stale
|
|
50
|
+
* `wf-owner-holder:<id>` record: a normal owner-side complete/fail commit is
|
|
51
|
+
* fenced but does not delete the holder record (only an explicit `release`
|
|
52
|
+
* — graceful shutdown or the reclaim scan's redrive-detects-terminal path —
|
|
53
|
+
* or a `takeover`/rotation does), so a contextless query for an already-
|
|
54
|
+
* terminal workflow can otherwise see a holder record naming a DIFFERENT
|
|
55
|
+
* engine and incorrectly throw `WorkflowNotLocallyOwnedError` for a workflow
|
|
56
|
+
* every engine agrees has already finished. Documented behavior is that
|
|
57
|
+
* terminal, purged, or unknown workflows keep resolving to plain
|
|
58
|
+
* `undefined` — this check restores that for the `workflow-lease` claim
|
|
59
|
+
* path specifically.
|
|
60
|
+
*/
|
|
61
|
+
export declare function isWorkflowLocallyTerminalOrMissing(internals: EngineInternals, workflowId: string): Promise<boolean>;
|
|
62
|
+
/**
|
|
63
|
+
* `internals.workflowClaimRegistry !== null && (await isWorkflowClaimedByAnotherEngine(...))`,
|
|
64
|
+
* factored out so every "revalidate before serving a possibly-stale live
|
|
65
|
+
* Context" call site — `query()` below and `tryInlineUpdateHandler` in
|
|
66
|
+
* `updates.ts` — shares the exact same registry-gated short-circuit. The
|
|
67
|
+
* registry-null check is a plain property read, not an unconditional await,
|
|
68
|
+
* so `ownership: 'none'`/`'lease'` stay exactly as synchronous as they are
|
|
69
|
+
* today; only `'workflow-lease'` pays for the cheap durable read.
|
|
70
|
+
*/
|
|
71
|
+
export declare function isLiveContextStale(internals: EngineInternals, workflowId: string): boolean | Promise<boolean>;
|
|
2
72
|
/** Resolve a workflow query from built-in progress state or exposed inline accessors. */
|
|
3
73
|
export declare function query(internals: EngineInternals, workflowId: string, name: string, input?: unknown): Promise<unknown>;
|
|
@@ -1,16 +1,70 @@
|
|
|
1
|
+
import { KEYS } from "../../storage/interface.js";
|
|
2
|
+
import { decodeWorkflowState, isTerminalWorkflowStatus } from "./validation.js";
|
|
3
|
+
import { decodeWorkflowClaimHolder } from "./workflow-claim-codec.js";
|
|
4
|
+
|
|
5
|
+
export class WorkflowNotLocallyOwnedError extends Error {
|
|
6
|
+
workflowId;
|
|
7
|
+
constructor(workflowId) {
|
|
8
|
+
super(`Workflow "${workflowId}" has no live query context on this engine, and a durable workflow-lease claim shows another engine currently owns it. Retry the query against the engine that currently holds the claim.`);
|
|
9
|
+
this.name = "WorkflowNotLocallyOwnedError";
|
|
10
|
+
this.workflowId = workflowId;
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
export async function isWorkflowClaimedByAnotherEngine(internals, workflowId) {
|
|
14
|
+
const registry = internals.workflowClaimRegistry;
|
|
15
|
+
if (registry === null)
|
|
16
|
+
return !1;
|
|
17
|
+
const raw = await internals.storage.get(KEYS.workflowOwnerHolder(workflowId));
|
|
18
|
+
if (raw === null)
|
|
19
|
+
return !1;
|
|
20
|
+
const holder = decodeWorkflowClaimHolder(raw);
|
|
21
|
+
if (holder === null)
|
|
22
|
+
return !1;
|
|
23
|
+
return holder.engineId !== registry.engineId;
|
|
24
|
+
}
|
|
25
|
+
export async function isWorkflowLocallyTerminalOrMissing(internals, workflowId) {
|
|
26
|
+
const stateBytes = await internals.storage.get(KEYS.workflow(workflowId));
|
|
27
|
+
if (stateBytes === null)
|
|
28
|
+
return !0;
|
|
29
|
+
const state = decodeWorkflowState(stateBytes);
|
|
30
|
+
return isTerminalWorkflowStatus(state.status);
|
|
31
|
+
}
|
|
32
|
+
export function isLiveContextStale(internals, workflowId) {
|
|
33
|
+
if (internals.workflowClaimRegistry === null)
|
|
34
|
+
return !1;
|
|
35
|
+
return isWorkflowClaimedByAnotherEngine(internals, workflowId);
|
|
36
|
+
}
|
|
37
|
+
function resolveDispatchContext(inlineStrategy, workflowId) {
|
|
38
|
+
return inlineStrategy.getContext(workflowId) ?? inlineStrategy.getParkedContext(workflowId);
|
|
39
|
+
}
|
|
40
|
+
async function resolveContextlessQueryResult(internals, workflowId) {
|
|
41
|
+
if (await isWorkflowClaimedByAnotherEngine(internals, workflowId) && !await isWorkflowLocallyTerminalOrMissing(internals, workflowId))
|
|
42
|
+
throw new WorkflowNotLocallyOwnedError(workflowId);
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
1
45
|
export async function query(internals, workflowId, name, input) {
|
|
2
46
|
if (name === "activityProgress")
|
|
3
47
|
return internals.heartbeatDetails.get(workflowId);
|
|
4
48
|
const inlineStrategy = internals.inlineStrategy;
|
|
5
49
|
if (!inlineStrategy)
|
|
6
50
|
throw Error("Workflow queries are not supported when using the worker execution strategy.");
|
|
7
|
-
const context = inlineStrategy
|
|
51
|
+
const context = resolveDispatchContext(inlineStrategy, workflowId);
|
|
8
52
|
if (!context)
|
|
9
|
-
return;
|
|
10
|
-
const
|
|
53
|
+
return resolveContextlessQueryResult(internals, workflowId);
|
|
54
|
+
const staleCheck = isLiveContextStale(internals, workflowId);
|
|
55
|
+
let liveContext = context;
|
|
56
|
+
if (staleCheck !== !1) {
|
|
57
|
+
if (await staleCheck)
|
|
58
|
+
throw new WorkflowNotLocallyOwnedError(workflowId);
|
|
59
|
+
const refreshedContext = resolveDispatchContext(inlineStrategy, workflowId);
|
|
60
|
+
if (!refreshedContext)
|
|
61
|
+
return;
|
|
62
|
+
liveContext = refreshedContext;
|
|
63
|
+
}
|
|
64
|
+
const queryHandler = liveContext.queryHandlers.get(name);
|
|
11
65
|
if (queryHandler)
|
|
12
66
|
return queryHandler(input);
|
|
13
|
-
const accessor =
|
|
67
|
+
const accessor = liveContext.exposedAccessors.get(name);
|
|
14
68
|
if (!accessor)
|
|
15
69
|
return;
|
|
16
70
|
return accessor();
|
|
@@ -238,5 +238,5 @@ export async function handleScheduledWorkflowTerminal(internals, workflowId, cal
|
|
|
238
238
|
});
|
|
239
239
|
}
|
|
240
240
|
async function deleteTransientScheduleRunMetadata(internals, workflowId) {
|
|
241
|
-
await commitFencedEngineWrite(internals, [{ type: "delete", key: KEYS.scheduleRun(workflowId) }], [], () => Error(`Schedule-run cleanup for workflow "${workflowId}" lost its precondition.`));
|
|
241
|
+
await commitFencedEngineWrite(internals, workflowId, [{ type: "delete", key: KEYS.scheduleRun(workflowId) }], [], () => Error(`Schedule-run cleanup for workflow "${workflowId}" lost its precondition.`));
|
|
242
242
|
}
|
|
@@ -30,7 +30,17 @@ export type ConsumedSignalResult = {
|
|
|
30
30
|
payload: unknown;
|
|
31
31
|
};
|
|
32
32
|
export declare function signal(internals: EngineInternals, workflowId: string, name: string, payload: unknown, callbacks: SignalCallbacks, options?: SignalDeliveryOptions): Promise<void>;
|
|
33
|
-
|
|
33
|
+
/**
|
|
34
|
+
* Remove the signal waiter registered under `waiterKey`, optionally only when
|
|
35
|
+
* it is still `expectedResolve`.
|
|
36
|
+
*
|
|
37
|
+
* Returns whether THIS call removed that exact waiter. Callers that go on to
|
|
38
|
+
* invoke the resolver must gate on the return value: a waiter captured before
|
|
39
|
+
* an await can be replaced by replay or a fresh park while that await is in
|
|
40
|
+
* flight, and invoking the captured resolver anyway advances a superseded
|
|
41
|
+
* generator even though the replacement was correctly left in place.
|
|
42
|
+
*/
|
|
43
|
+
export declare function releaseSignalWaiter(internals: EngineInternals, workflowId: string, waiterKey: string, expectedResolve?: () => void): boolean;
|
|
34
44
|
export declare function bufferSignalPayloads(internals: EngineInternals, workflowId: string, deliveries: BufferedSignalDelivery[], callbacks: SignalCallbacks, defaultOptions?: SignalDeliveryOptions): Promise<void>;
|
|
35
45
|
/**
|
|
36
46
|
* Build the durable operations for a single keyed signal so it can be folded
|
|
@@ -11,6 +11,7 @@ import { validateSignalId } from "../signal-id.js";
|
|
|
11
11
|
import { commitAnonymousSignalOperations } from "./anonymous-signal-sequence.js";
|
|
12
12
|
import { stageAtomicWorkflowCommitSideEffects } from "./checkpoint-side-effects.js";
|
|
13
13
|
import { isTerminalWorkflowStatus } from "./validation.js";
|
|
14
|
+
import { confirmWakeOwnership } from "./wake-ownership-guard.js";
|
|
14
15
|
const EMPTY_STORAGE_VALUE = new Uint8Array(0), SIGNAL_ACCEPTED_RESPONSE = { ok: !0 }, SIGNAL_KEY_COMPONENT_COUNT = 5;
|
|
15
16
|
export async function signal(internals, workflowId, name, payload, callbacks, options = {}) {
|
|
16
17
|
const deliverSignal = async (targetWorkflowId, signalName, signalPayload) => {
|
|
@@ -44,11 +45,12 @@ export async function signal(internals, workflowId, name, payload, callbacks, op
|
|
|
44
45
|
export function releaseSignalWaiter(internals, workflowId, waiterKey, expectedResolve) {
|
|
45
46
|
const currentWaiter = internals.signalWaiters.get(waiterKey);
|
|
46
47
|
if (!currentWaiter)
|
|
47
|
-
return;
|
|
48
|
+
return !1;
|
|
48
49
|
if (expectedResolve && currentWaiter !== expectedResolve)
|
|
49
|
-
return;
|
|
50
|
+
return !1;
|
|
50
51
|
internals.signalWaiters.delete(waiterKey);
|
|
51
52
|
untrackWaiterKey(internals.signalWaitersByWorkflow, workflowId, waiterKey);
|
|
53
|
+
return !0;
|
|
52
54
|
}
|
|
53
55
|
export async function bufferSignalPayloads(internals, workflowId, deliveries, callbacks, defaultOptions = {}) {
|
|
54
56
|
if (deliveries.length === 0)
|
|
@@ -69,11 +71,11 @@ export async function bufferSignalPayloads(internals, workflowId, deliveries, ca
|
|
|
69
71
|
if (!await storageConditionalBatch(internals.storage, [{ key: acceptedResponseKey, expectedValue: null }], [...operations, { type: "put", key: acceptedResponseKey, value: acceptedResponse }]))
|
|
70
72
|
return;
|
|
71
73
|
markTerminalCleanupTracked(internals, workflowId);
|
|
72
|
-
deliverBufferedSignals(internals, workflowId, deliveries, callbacks);
|
|
74
|
+
await deliverBufferedSignals(internals, workflowId, deliveries, callbacks);
|
|
73
75
|
return;
|
|
74
76
|
}
|
|
75
77
|
await commitAnonymousSignalOperations(internals, workflowId, deliveries, (operations) => appendTerminalCleanupOperation(internals, workflowId, operations), () => markTerminalCleanupTracked(internals, workflowId));
|
|
76
|
-
deliverBufferedSignals(internals, workflowId, deliveries, callbacks);
|
|
78
|
+
await deliverBufferedSignals(internals, workflowId, deliveries, callbacks);
|
|
77
79
|
}
|
|
78
80
|
function createExplicitSignalOperations(internals, workflowId, deliveries, signalId) {
|
|
79
81
|
return deliveries.map(({ signalName, payload }) => ({
|
|
@@ -109,7 +111,7 @@ function appendTerminalCleanupOperation(internals, workflowId, operations) {
|
|
|
109
111
|
function markTerminalCleanupTracked(internals, workflowId) {
|
|
110
112
|
internals.workflowsNeedingTerminalCleanup.add(workflowId);
|
|
111
113
|
}
|
|
112
|
-
function deliverBufferedSignals(internals, workflowId, deliveries, callbacks) {
|
|
114
|
+
async function deliverBufferedSignals(internals, workflowId, deliveries, callbacks) {
|
|
113
115
|
let shouldResumeParkedWorkflow = !1;
|
|
114
116
|
for (const { signalName, payload, options } of deliveries) {
|
|
115
117
|
if (options?.emitPublicEvent !== !1) {
|
|
@@ -118,8 +120,10 @@ function deliverBufferedSignals(internals, workflowId, deliveries, callbacks) {
|
|
|
118
120
|
}
|
|
119
121
|
const waiterKey = `${workflowId}:${signalName}`, waiter = internals.signalWaiters.get(waiterKey);
|
|
120
122
|
if (waiter) {
|
|
121
|
-
|
|
122
|
-
|
|
123
|
+
if (internals.workflowClaimRegistry !== null && await confirmWakeOwnership(internals, workflowId, "signal") === "discard")
|
|
124
|
+
continue;
|
|
125
|
+
if (releaseSignalWaiter(internals, workflowId, waiterKey, waiter))
|
|
126
|
+
waiter();
|
|
123
127
|
continue;
|
|
124
128
|
}
|
|
125
129
|
if (internals.parkedInlineWorkflows.has(workflowId))
|
|
@@ -5,7 +5,75 @@ export type SleepTimerAcknowledgement = {
|
|
|
5
5
|
cancel: () => void;
|
|
6
6
|
promise: Promise<void>;
|
|
7
7
|
};
|
|
8
|
+
/**
|
|
9
|
+
* Durable sleep timers fire globally — under `ownership: 'workflow-lease'`
|
|
10
|
+
* every engine sharing the store observes the same expired timer, not only
|
|
11
|
+
* the workflow's owner (see ADR 0002's entry-point classification: the
|
|
12
|
+
* scheduler dispatch shell is claim-acquiring only for claim-ACQUIRING
|
|
13
|
+
* branches; this one is claim-REQUIRING and checks for itself). Running
|
|
14
|
+
* `confirmWakeOwnership` FIRST — before `shouldIgnoreUnclaimedSleepTimer`
|
|
15
|
+
* even loads workflow state — matters: on a non-owning engine that state
|
|
16
|
+
* legitimately reads `'running'` (the true owner is actively driving it),
|
|
17
|
+
* which would otherwise hit `shouldIgnoreUnclaimedSleepTimer`'s "fired
|
|
18
|
+
* before ready" throw meant for a same-engine registration race, not a
|
|
19
|
+
* cross-engine ownership miss.
|
|
20
|
+
*
|
|
21
|
+
* A discard is NOT a silent no-op: `handleTimerFired`'s caller — the
|
|
22
|
+
* `Scheduler` — treats a callback that returns without throwing as
|
|
23
|
+
* "processed" and durably deletes the fired timer key
|
|
24
|
+
* (`commitTimerCleanup`, engine-scoped and unfenced on any single workflow's
|
|
25
|
+
* claim, since one tick's cleanup batch can span fired timers from many
|
|
26
|
+
* workflows — see `src/core/engine/index.ts`'s `Scheduler` wiring). If a
|
|
27
|
+
* discarding non-owner let that deletion proceed, it would delete the true
|
|
28
|
+
* owner's only durable record of this fire before the owner ever observes
|
|
29
|
+
* it — see `retainDiscardedDurableTimer` below.
|
|
30
|
+
*/
|
|
8
31
|
export declare function handleSleepTimerWithAcknowledgement(internals: EngineInternals, entry: TimerEntry, loadWorkflowState: (workflowId: string) => Promise<WorkflowState | null>): Promise<void>;
|
|
32
|
+
/** What to do with a durable timer key once a claim-requiring wake has discarded it. */
|
|
33
|
+
export type DiscardedTimerDisposition = 'retain' | 'collect';
|
|
34
|
+
/**
|
|
35
|
+
* Decide what a discarded claim-requiring timer fire (ADR 0002's `sleep` and
|
|
36
|
+
* `wait-condition` wake kinds — see `operations-time.ts`'s
|
|
37
|
+
* `resolveConditionTimer` for the second caller) should do with its durable
|
|
38
|
+
* timer key. Shared by both wake kinds because they share the exact same
|
|
39
|
+
* hazard: `commitTimerCleanup` batches deletes across many workflows in one
|
|
40
|
+
* engine-scoped, unfenced write, so a non-owner that discards must not let
|
|
41
|
+
* the Scheduler treat the fire as "processed" while some other engine still
|
|
42
|
+
* needs the same durable record to perform the real wake.
|
|
43
|
+
*
|
|
44
|
+
* "Not locally owned" is not by itself proof the timer is stale, though —
|
|
45
|
+
* `resolveSleepTimer`'s own comment documents that a durable sleep timer
|
|
46
|
+
* OUTLIVES terminal cleanup (cleanup only drops the in-memory resolver, not
|
|
47
|
+
* the durable key). Blindly retaining on every discard would turn an
|
|
48
|
+
* orphaned post-terminal timer into an immortal one: every engine sharing
|
|
49
|
+
* the store would rediscover it, discard it, and retain it again on every
|
|
50
|
+
* Scheduler tick forever. So this reads the workflow's CURRENT persisted
|
|
51
|
+
* status to disambiguate:
|
|
52
|
+
* - `null`, or a terminal status ({@link isTerminalWorkflowStatus}):
|
|
53
|
+
* `'collect'` — no engine holds or will ever again acquire a claim for
|
|
54
|
+
* this workflow, so nothing will ever consume this fire; let the
|
|
55
|
+
* Scheduler's normal cleanup remove the orphaned key, matching
|
|
56
|
+
* pre-ADR-0002 behavior for an unclaimed timer.
|
|
57
|
+
* - `'suspended'`: `'collect'` — `engine.suspend()`'s durable re-arm
|
|
58
|
+
* establishes its own fresh timer on resume; the pre-suspend fire being
|
|
59
|
+
* discarded here is not the one that wakes the resumed run.
|
|
60
|
+
* - any other status (`'running'`, `'pending'`): `'retain'` — some engine
|
|
61
|
+
* still holds, or will still acquire, a live claim for this workflow;
|
|
62
|
+
* leave the durable key so that engine's own copy of this same fire
|
|
63
|
+
* performs the real wake and deletes it for real, bounded by that
|
|
64
|
+
* engine's own next Scheduler poll.
|
|
65
|
+
*/
|
|
66
|
+
export declare function resolveDiscardedTimerDisposition(workflowId: string, loadWorkflowState: (workflowId: string) => Promise<WorkflowState | null>): Promise<DiscardedTimerDisposition>;
|
|
67
|
+
/**
|
|
68
|
+
* Apply {@link resolveDiscardedTimerDisposition} to a discarded claim-requiring
|
|
69
|
+
* timer fire: throws when the durable key must be retained, so the
|
|
70
|
+
* `Scheduler`'s `#processSelectedTimer` catch block treats this fire as
|
|
71
|
+
* `'retry'` — leaving the timer key in storage instead of collecting it —
|
|
72
|
+
* and resolves normally when the workflow is gone, terminal, or suspended,
|
|
73
|
+
* so a genuinely orphaned timer is still collected exactly as it was before
|
|
74
|
+
* this ownership check existed.
|
|
75
|
+
*/
|
|
76
|
+
export declare function retainDiscardedDurableTimer(timerId: string, workflowId: string, loadWorkflowState: (workflowId: string) => Promise<WorkflowState | null>): Promise<void>;
|
|
9
77
|
export declare function createSleepTimerAcknowledgement(internals: EngineInternals, workflowId: string, operationId: string, fireAt: number): SleepTimerAcknowledgement;
|
|
10
78
|
export declare function recordDurableInlineOperation(internals: EngineInternals, workflowId: string, operation: ContextOperationRequest): void;
|
|
11
79
|
export declare function settleSleepTimerAcknowledgements(internals: EngineInternals, workflowId: string, disposition: 'suspended' | 'terminal'): void;
|
|
@@ -1,4 +1,10 @@
|
|
|
1
|
+
import { isTerminalWorkflowStatus } from "./validation.js";
|
|
2
|
+
import { confirmWakeOwnership } from "./wake-ownership-guard.js";
|
|
1
3
|
export async function handleSleepTimerWithAcknowledgement(internals, entry, loadWorkflowState) {
|
|
4
|
+
if (await confirmWakeOwnership(internals, entry.workflowId, "sleep") === "discard") {
|
|
5
|
+
await retainDiscardedDurableTimer(entry.id, entry.workflowId, loadWorkflowState);
|
|
6
|
+
return;
|
|
7
|
+
}
|
|
2
8
|
const operationId = entry.id.replace("sleep:", "");
|
|
3
9
|
if (await shouldIgnoreUnclaimedSleepTimer(internals, entry, operationId, loadWorkflowState))
|
|
4
10
|
return;
|
|
@@ -9,6 +15,19 @@ export async function handleSleepTimerWithAcknowledgement(internals, entry, load
|
|
|
9
15
|
}
|
|
10
16
|
await acknowledgement?.promise;
|
|
11
17
|
}
|
|
18
|
+
export async function resolveDiscardedTimerDisposition(workflowId, loadWorkflowState) {
|
|
19
|
+
const state = await loadWorkflowState(workflowId);
|
|
20
|
+
if (state === null)
|
|
21
|
+
return "collect";
|
|
22
|
+
if (state.status === "suspended" || isTerminalWorkflowStatus(state.status))
|
|
23
|
+
return "collect";
|
|
24
|
+
return "retain";
|
|
25
|
+
}
|
|
26
|
+
export async function retainDiscardedDurableTimer(timerId, workflowId, loadWorkflowState) {
|
|
27
|
+
if (await resolveDiscardedTimerDisposition(workflowId, loadWorkflowState) !== "retain")
|
|
28
|
+
return;
|
|
29
|
+
throw Error(`Durable timer "${timerId}" for workflow "${workflowId}" was discarded by a non-owning engine under ownership: 'workflow-lease'; retaining it in storage for the true owner instead of letting the scheduler delete it.`);
|
|
30
|
+
}
|
|
12
31
|
async function shouldIgnoreUnclaimedSleepTimer(internals, entry, operationId, loadWorkflowState) {
|
|
13
32
|
const resolverKey = `${entry.workflowId}:${operationId}`;
|
|
14
33
|
if (internals.sleepResolvers.has(resolverKey) || internals.inlineStrategy === null)
|