@lostgradient/weft 0.24.1 → 0.24.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/core/engine/bulk-operations-purge.d.ts +1 -1
- package/dist/core/engine/bulk-operations-purge.js +3 -2
- package/dist/core/engine/bulk-operations-retry.d.ts +16 -1
- package/dist/core/engine/bulk-operations-retry.js +19 -6
- package/dist/core/engine/bulk-operations.d.ts +3 -2
- package/dist/core/engine/bulk-operations.js +2 -2
- package/dist/core/engine/callback-creators-bundles.js +1 -1
- package/dist/core/engine/callback-creators-schedule.js +2 -2
- package/dist/core/engine/child-workflow.d.ts +13 -1
- package/dist/core/engine/child-workflow.js +22 -4
- package/dist/core/engine/generation-codec.d.ts +22 -0
- package/dist/core/engine/generation-codec.js +2 -0
- package/dist/core/engine/index.js +1 -1
- package/dist/core/engine/lifecycle/recovery-isolation.d.ts +20 -0
- package/dist/core/engine/lifecycle/recovery-isolation.js +26 -0
- package/dist/core/engine/lifecycle/resume-body.d.ts +44 -0
- package/dist/core/engine/lifecycle/resume-body.js +211 -0
- package/dist/core/engine/lifecycle/resume.d.ts +46 -3
- package/dist/core/engine/lifecycle/resume.js +21 -210
- package/dist/core/engine/lifecycle/standalone-claim-acquire.d.ts +68 -2
- package/dist/core/engine/lifecycle/standalone-claim-acquire.js +14 -4
- package/dist/core/engine/lifecycle/start-commit.d.ts +2 -0
- package/dist/core/engine/lifecycle/start-commit.js +16 -8
- package/dist/core/engine/lifecycle/start-or-signal-resolution.d.ts +8 -2
- package/dist/core/engine/lifecycle/start-or-signal-resolution.js +11 -3
- package/dist/core/engine/lifecycle/start-precondition-attribution.d.ts +10 -4
- package/dist/core/engine/lifecycle/start-schedule-timing.d.ts +11 -0
- package/dist/core/engine/lifecycle/start-schedule-timing.js +21 -0
- package/dist/core/engine/lifecycle/start-terminal-conflict-purge.d.ts +90 -11
- package/dist/core/engine/lifecycle/start-terminal-conflict-purge.js +24 -5
- package/dist/core/engine/lifecycle/start.d.ts +40 -3
- package/dist/core/engine/lifecycle/start.js +19 -26
- package/dist/core/engine/lifecycle/transition.d.ts +9 -0
- package/dist/core/engine/lifecycle/transition.js +20 -25
- package/dist/core/engine/lifecycle.d.ts +2 -1
- package/dist/core/engine/lifecycle.js +2 -1
- package/dist/core/engine/schedule-overlap.js +2 -1
- package/dist/core/engine/schedule-run.d.ts +10 -0
- package/dist/core/engine/schedule-run.js +1 -1
- package/dist/core/engine/schedules.d.ts +11 -2
- package/dist/core/engine/termination/complete.js +1 -1
- package/dist/core/engine/termination/suspend.js +3 -0
- package/dist/core/engine/validation/schedule.d.ts +25 -1
- package/dist/core/engine/validation/schedule.js +4 -10
- package/dist/core/engine/validation.js +10 -17
- package/dist/core/engine/workflow-claim-reclaim-target.js +6 -6
- package/dist/core/engine/workflow-claim-registry.d.ts +79 -75
- package/dist/core/engine/workflow-claim-registry.js +15 -2
- package/dist/core/engine/workflow-generation-fence.d.ts +74 -0
- package/dist/core/engine/workflow-generation-fence.js +19 -0
- package/dist/core/start-workflow-validation.d.ts +15 -0
- package/dist/core/start-workflow-validation.js +11 -1
- package/dist/core/workflow-identifiers.d.ts +37 -0
- package/dist/core/workflow-identifiers.js +19 -1
- package/dist/http.js +2 -2
- package/dist/indexeddb.js +1 -1
- package/dist/server/fleet-event-feed.js +6 -6
- package/dist/server/rest-binding.d.ts +11 -0
- package/dist/server/runtime/task-dispatch-envelope.d.ts +32 -5
- package/dist/server/runtime/task-dispatch-envelope.js +23 -1
- package/dist/server/runtime/task-dispatch.d.ts +3 -1
- package/dist/server/runtime/task-dispatch.js +9 -8
- package/dist/server/task-ledger-codec.d.ts +19 -0
- package/dist/server/task-ledger-codec.js +3 -0
- package/dist/server/task-ledger.d.ts +1 -1
- package/dist/server/task-ledger.js +1 -0
- package/dist/storage/bun-sql.js +132 -99
- package/dist/storage/compressed-storage.js +1 -1
- package/dist/storage/generation-keys.d.ts +52 -0
- package/dist/storage/generation-keys.js +4 -0
- package/dist/storage/index.d.ts +76 -65
- package/dist/storage/interface.d.ts +24 -232
- package/dist/storage/interface.js +1 -1
- package/dist/storage/key-prefixes.d.ts +1 -1
- package/dist/storage/key-prefixes.js +1 -0
- package/dist/storage/lease-keys.d.ts +48 -0
- package/dist/storage/lease-keys.js +8 -0
- package/dist/storage/lmdb.js +1 -1
- package/dist/storage/memory.js +1 -1
- package/dist/storage/neon.js +3 -3
- package/dist/storage/node-sqlite.js +132 -99
- package/dist/storage/postgres.js +3 -3
- package/dist/storage/resolve.js +1 -1
- package/dist/storage/scoped-storage.js +1 -1
- package/dist/storage/signal-keys.d.ts +23 -0
- package/dist/storage/signal-keys.js +9 -0
- package/dist/storage/testing.js +1 -1
- package/dist/storage/turso.js +2 -2
- package/dist/storage/workflow-lifecycle-keys.d.ts +128 -0
- package/dist/storage/workflow-lifecycle-keys.js +46 -0
- package/dist/storage/workflow-record-keys.d.ts +98 -0
- package/dist/storage/workflow-record-keys.js +47 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/web-extension.js +1 -1
- package/package.json +1 -1
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { decode } from "../codec.js";
|
|
2
2
|
import { isRecord } from "../debug-output.js";
|
|
3
3
|
import { normalizeFailureCategory } from "../failure-categories.js";
|
|
4
|
-
import {
|
|
4
|
+
import { parseStartWorkflowDuration } from "../start-workflow-validation.js";
|
|
5
5
|
import { DEFAULT_WORKFLOW_VERSION } from "../versioning.js";
|
|
6
|
+
import { isDecodableWorkflowId } from "../workflow-identifiers.js";
|
|
6
7
|
import { isWorkflowTagArray } from "../workflow-tags.js";
|
|
7
8
|
import { sanitizeDecodedRevision } from "./decode-revision.js";
|
|
8
9
|
import {
|
|
@@ -133,25 +134,18 @@ export function decodeWorkflowState(bytes) {
|
|
|
133
134
|
else
|
|
134
135
|
state.failureCategory = normalizedFailureCategory;
|
|
135
136
|
}
|
|
136
|
-
if (state.executionStateOwnerId !== void 0)
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
console.warn(`[weft] Decoded workflow state for "${state.id}" has an invalid executionStateOwnerId field; falling back to the workflow id as the execution owner. This usually indicates corruption or tampering of the storage record.`);
|
|
141
|
-
delete state.executionStateOwnerId;
|
|
142
|
-
}
|
|
137
|
+
if (state.executionStateOwnerId !== void 0 && !isDecodableWorkflowId(state.executionStateOwnerId)) {
|
|
138
|
+
console.warn(`[weft] Decoded workflow state for "${state.id}" has an invalid executionStateOwnerId field; falling back to the workflow id as the execution owner. This usually indicates corruption or tampering of the storage record.`);
|
|
139
|
+
delete state.executionStateOwnerId;
|
|
140
|
+
}
|
|
143
141
|
sanitizeDecodedParentLineage(state);
|
|
144
142
|
sanitizeDecodedRestartLineage(state);
|
|
145
143
|
sanitizeDecodedRevision(state);
|
|
146
144
|
return decodedRecord === void 0 ? state : stripUnknownWorkflowStateFields(state, decodedRecord);
|
|
147
145
|
}
|
|
148
146
|
function sanitizeDecodedParentLineage(state) {
|
|
149
|
-
if (state.parentWorkflowId !== void 0)
|
|
150
|
-
|
|
151
|
-
coerceStartWorkflowId(state.parentWorkflowId, "parentWorkflowId");
|
|
152
|
-
} catch {
|
|
153
|
-
delete state.parentWorkflowId;
|
|
154
|
-
}
|
|
147
|
+
if (state.parentWorkflowId !== void 0 && !isDecodableWorkflowId(state.parentWorkflowId))
|
|
148
|
+
delete state.parentWorkflowId;
|
|
155
149
|
if (typeof state.parentWorkflowExecutionToken !== "string" || state.parentWorkflowExecutionToken.length === 0 || state.parentWorkflowId === void 0)
|
|
156
150
|
delete state.parentWorkflowExecutionToken;
|
|
157
151
|
}
|
|
@@ -161,9 +155,8 @@ function sanitizeDecodedRestartLineage(state) {
|
|
|
161
155
|
delete state.restartedFrom;
|
|
162
156
|
return;
|
|
163
157
|
}
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
} catch {
|
|
158
|
+
const restartWorkflowId = restartedFrom.workflowId;
|
|
159
|
+
if (typeof restartWorkflowId !== "string" || !isDecodableWorkflowId(restartWorkflowId)) {
|
|
167
160
|
delete state.restartedFrom;
|
|
168
161
|
return;
|
|
169
162
|
}
|
|
@@ -53,14 +53,14 @@ export function createWorkflowClaimReclaimTarget(registry, storage, metrics, onR
|
|
|
53
53
|
throw error;
|
|
54
54
|
}
|
|
55
55
|
}
|
|
56
|
-
async function releaseClaimAcquiredWhileDisposing(workflowId) {
|
|
57
|
-
await registry.release(workflowId);
|
|
56
|
+
async function releaseClaimAcquiredWhileDisposing(workflowId, acquiredEpoch) {
|
|
57
|
+
await registry.release(workflowId, acquiredEpoch);
|
|
58
58
|
}
|
|
59
59
|
async function confirmStillRunningOrReleaseFreshClaim(workflowId, acquiredEpoch) {
|
|
60
60
|
if (await isWorkflowStillRunningAndEligible(storage, workflowId, isTypeRegistered))
|
|
61
61
|
return !0;
|
|
62
62
|
if (registry.currentEpoch(workflowId) === acquiredEpoch)
|
|
63
|
-
await registry.release(workflowId);
|
|
63
|
+
await registry.release(workflowId, acquiredEpoch);
|
|
64
64
|
return !1;
|
|
65
65
|
}
|
|
66
66
|
async function redriveAlreadyHeldClaim(workflowId, expectedEpoch) {
|
|
@@ -69,7 +69,7 @@ export function createWorkflowClaimReclaimTarget(registry, storage, metrics, onR
|
|
|
69
69
|
if (!await isWorkflowStillRunning(storage, workflowId)) {
|
|
70
70
|
pendingRedriveWorkflowIds.delete(workflowId);
|
|
71
71
|
if (registry.currentEpoch(workflowId) === expectedEpoch)
|
|
72
|
-
await registry.release(workflowId);
|
|
72
|
+
await registry.release(workflowId, expectedEpoch);
|
|
73
73
|
return { status: "not-eligible" };
|
|
74
74
|
}
|
|
75
75
|
if (disposing)
|
|
@@ -90,7 +90,7 @@ export function createWorkflowClaimReclaimTarget(registry, storage, metrics, onR
|
|
|
90
90
|
}
|
|
91
91
|
metrics.recordClaimAttempt("acquired");
|
|
92
92
|
if (disposing) {
|
|
93
|
-
await releaseClaimAcquiredWhileDisposing(workflowId);
|
|
93
|
+
await releaseClaimAcquiredWhileDisposing(workflowId, acquireResult.epoch);
|
|
94
94
|
return { status: "not-eligible" };
|
|
95
95
|
}
|
|
96
96
|
if (!await confirmStillRunningOrReleaseFreshClaim(workflowId, acquireResult.epoch))
|
|
@@ -100,7 +100,7 @@ export function createWorkflowClaimReclaimTarget(registry, storage, metrics, onR
|
|
|
100
100
|
async function handleTakeoverAcquired(workflowId, acquiredEpoch) {
|
|
101
101
|
metrics.recordClaimAttempt("takeover");
|
|
102
102
|
if (disposing) {
|
|
103
|
-
await releaseClaimAcquiredWhileDisposing(workflowId);
|
|
103
|
+
await releaseClaimAcquiredWhileDisposing(workflowId, acquiredEpoch);
|
|
104
104
|
return { status: "not-eligible" };
|
|
105
105
|
}
|
|
106
106
|
if (!await confirmStillRunningOrReleaseFreshClaim(workflowId, acquiredEpoch))
|
|
@@ -8,39 +8,43 @@
|
|
|
8
8
|
* fragments; this module is the thin, stateful layer around them that reads
|
|
9
9
|
* storage, executes `storageConditionalBatch`, and tracks — for every claim
|
|
10
10
|
* THIS engine currently holds — the exact epoch and holder bytes it last
|
|
11
|
-
* wrote
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
* "never round-trip encode(decode(raw))" discipline.
|
|
11
|
+
* wrote, extracted from the fragment they were just written by rather than
|
|
12
|
+
* re-encoded, mirroring `lease-manager.ts`'s "never round-trip
|
|
13
|
+
* encode(decode(raw))" discipline.
|
|
15
14
|
*
|
|
16
15
|
* **Scope.** The unit itself: acquire, renew, release, takeover, and
|
|
17
16
|
* release-all — plus, additively, {@link WorkflowClaimRegistry.prepareAcquireFragment}
|
|
18
17
|
* and {@link WorkflowClaimRegistry.recordFoldedAcquire}, the two-step seam a
|
|
19
|
-
* caller uses to fold `acquire` into ITS OWN atomic enabling write
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
* decision to the caller — `takeover` also gates on a per-workflow-id
|
|
30
|
-
* anti-thrash cooldown ({@link WorkflowClaimTakeoverCooldown}). Still out of
|
|
31
|
-
* scope: `weft_workflow_claim_*` metrics, `wakeOwnershipCheck`, and external
|
|
32
|
-
* terminal-transition rotation (any engine may commit those unconditioned).
|
|
18
|
+
* caller uses to fold `acquire` into ITS OWN atomic enabling write instead of
|
|
19
|
+
* committing the fragment through this registry's own `acquire()`. Driving
|
|
20
|
+
* `renew`/the reclaim scan, and turning a lost `acquire`/`takeover` into
|
|
21
|
+
* `WorkflowClaimUnavailableError` for explicit single-workflow callers, stay
|
|
22
|
+
* each call site's responsibility — this registry never throws that error,
|
|
23
|
+
* and per the ADR background scanning never does either, so every method
|
|
24
|
+
* returns a discriminated result instead. `takeover` also gates on a
|
|
25
|
+
* per-workflow-id anti-thrash cooldown ({@link WorkflowClaimTakeoverCooldown}).
|
|
26
|
+
* Still out of scope: `weft_workflow_claim_*` metrics, `wakeOwnershipCheck`,
|
|
27
|
+
* and external terminal-transition rotation.
|
|
33
28
|
*
|
|
34
29
|
* **Renewal-vs-release serialization.** A per-workflow in-flight-renewal
|
|
35
30
|
* promise (mirroring `lease-manager.ts`'s single `inFlightRenewal`) lets
|
|
36
31
|
* `release()` await a renewal already in progress before reading the cached
|
|
37
|
-
* bytes it conditions on; otherwise both race the same holder bytes
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
32
|
+
* bytes it conditions on; otherwise both race the same holder bytes. A
|
|
33
|
+
* `releasing` set stops a NEW renewal from starting once release has begun,
|
|
34
|
+
* so a `renew()` arriving mid-release fails fast as `'not-held'`. A renewal
|
|
35
|
+
* that THROWS (transient, not a CAS-false result) is not a lost claim: it
|
|
36
|
+
* propagates leaving cached bytes untouched, and `release()`'s wait swallows
|
|
37
|
+
* it — a storage hiccup must not fail a terminal or shutdown release.
|
|
38
|
+
*
|
|
39
|
+
* **Epoch-guarded release (WFT-134).** `release(workflowId, expectedEpoch?)`
|
|
40
|
+
* takes an OPTIONAL epoch a caller itself acquired; without it, `release`
|
|
41
|
+
* drops whatever entry is CURRENTLY tracked — correct only when nothing else
|
|
42
|
+
* could have replaced the claim meanwhile. A caller undoing ITS OWN
|
|
43
|
+
* acquisition must pass that epoch: a `recordFoldedAcquire` replacement can
|
|
44
|
+
* land even during this call's own in-flight-renewal await, and
|
|
45
|
+
* `expectedEpoch` is re-checked against the freshly-read entry AFTER that
|
|
46
|
+
* await, so a stale pre-await snapshot never authorizes releasing a newer,
|
|
47
|
+
* live claim.
|
|
44
48
|
*
|
|
45
49
|
* @module core/engine/workflow-claim-registry
|
|
46
50
|
*/
|
|
@@ -56,10 +60,7 @@ export type WorkflowClaimRegistryOptions = {
|
|
|
56
60
|
getNow: () => number;
|
|
57
61
|
/** `workflowClaimTtl` (ms), resolved by `ownership-options.ts`. */
|
|
58
62
|
claimTtlMs: number;
|
|
59
|
-
/**
|
|
60
|
-
* `workflowClaimRenewInterval` (ms), resolved by `ownership-options.ts` —
|
|
61
|
-
* feeds `isWorkflowClaimExpired`'s grace term.
|
|
62
|
-
*/
|
|
63
|
+
/** `workflowClaimRenewInterval` (ms) — feeds `isWorkflowClaimExpired`'s grace term. */
|
|
63
64
|
claimRenewIntervalMs: number;
|
|
64
65
|
/** Operator-warning seam; defaults to `process.emitWarning` via {@link emitWorkflowClaimLostWarning}. */
|
|
65
66
|
warn?: EmitWorkflowLeaseWarning;
|
|
@@ -98,8 +99,8 @@ export type WorkflowClaimReleaseResult = {
|
|
|
98
99
|
};
|
|
99
100
|
/**
|
|
100
101
|
* A prepared, not-yet-committed `acquire` — the output of
|
|
101
|
-
* {@link WorkflowClaimRegistry.prepareAcquireFragment},
|
|
102
|
-
*
|
|
102
|
+
* {@link WorkflowClaimRegistry.prepareAcquireFragment}, merged into a
|
|
103
|
+
* caller's own atomic write and handed to
|
|
103
104
|
* {@link WorkflowClaimRegistry.recordFoldedAcquire} once that write commits.
|
|
104
105
|
*/
|
|
105
106
|
export type WorkflowClaimAcquirePreparation = {
|
|
@@ -130,8 +131,8 @@ export type WorkflowClaimTakeoverResult = {
|
|
|
130
131
|
};
|
|
131
132
|
/**
|
|
132
133
|
* Owns this engine's per-workflow ownership claims: reads storage, executes
|
|
133
|
-
* the pure transition fragments, and tracks the exact bytes
|
|
134
|
-
* every claim
|
|
134
|
+
* the pure transition fragments, and tracks the exact bytes last written for
|
|
135
|
+
* every held claim. See the module doc for scope.
|
|
135
136
|
*/
|
|
136
137
|
export declare class WorkflowClaimRegistry {
|
|
137
138
|
#private;
|
|
@@ -142,25 +143,20 @@ export declare class WorkflowClaimRegistry {
|
|
|
142
143
|
get engineId(): string;
|
|
143
144
|
/**
|
|
144
145
|
* Every workflow id this engine currently tracks a live claim for — active
|
|
145
|
-
* or parked. A defensive-copy snapshot
|
|
146
|
-
*
|
|
147
|
-
*
|
|
148
|
-
* internal map while iterating a snapshot they already took.
|
|
146
|
+
* or parked. A defensive-copy snapshot: callers (the claim-renewal task, an
|
|
147
|
+
* active-claims metrics gauge) must not observe mutations to this
|
|
148
|
+
* registry's internal map while iterating a snapshot they already took.
|
|
149
149
|
*/
|
|
150
150
|
listHeldWorkflowIds(): readonly string[];
|
|
151
151
|
/**
|
|
152
152
|
* Defensive copy of the epoch bytes this engine last wrote for
|
|
153
|
-
* `workflowId`, for fencing durable writes — `null` if untracked.
|
|
154
|
-
* a caller mutating the returned buffer cannot corrupt this registry's
|
|
155
|
-
* cached fencing token.
|
|
153
|
+
* `workflowId`, for fencing durable writes — `null` if untracked.
|
|
156
154
|
*/
|
|
157
155
|
currentEpochBytes(workflowId: string): Uint8Array | null;
|
|
158
156
|
/**
|
|
159
|
-
* `acquire`: always reads both keys fresh
|
|
160
|
-
*
|
|
161
|
-
*
|
|
162
|
-
* non-null; otherwise (a competitor raced in between the read and the
|
|
163
|
-
* write) re-reads to report the true current holder.
|
|
157
|
+
* `acquire`: always reads both keys fresh, then builds and attempts the
|
|
158
|
+
* fragment from what it read. On a lost CAS, resolves `heldBy` from the
|
|
159
|
+
* already-read holder bytes when non-null; otherwise re-reads.
|
|
164
160
|
*/
|
|
165
161
|
acquire(workflowId: string): Promise<WorkflowClaimAcquireResult>;
|
|
166
162
|
/**
|
|
@@ -168,60 +164,68 @@ export declare class WorkflowClaimRegistry {
|
|
|
168
164
|
* `acquire` fragment WITHOUT committing it or updating this registry's
|
|
169
165
|
* tracking — for a caller that folds the fragment into ITS OWN atomic
|
|
170
166
|
* enabling write instead of letting {@link acquire} commit it alone. The
|
|
171
|
-
* caller merges
|
|
172
|
-
*
|
|
173
|
-
*
|
|
174
|
-
*
|
|
175
|
-
* re-reads fresh bytes, so a stale epoch from an earlier attempt never
|
|
176
|
-
* dooms a later one.
|
|
167
|
+
* caller merges the fragment into its own operation list, commits ONE
|
|
168
|
+
* atomic `storageConditionalBatch`, and — ONLY on success — calls
|
|
169
|
+
* {@link recordFoldedAcquire} with this SAME preparation. Safe to call
|
|
170
|
+
* again on every retry: it always re-reads fresh bytes.
|
|
177
171
|
*/
|
|
178
172
|
prepareAcquireFragment(workflowId: string): Promise<WorkflowClaimAcquirePreparation>;
|
|
179
173
|
/**
|
|
180
174
|
* Install the tracking entry for a claim acquired via a FOLDED enabling
|
|
181
175
|
* write (see {@link prepareAcquireFragment}) — call ONLY after the
|
|
182
|
-
* caller's own atomic commit
|
|
183
|
-
*
|
|
184
|
-
*
|
|
185
|
-
* encode(decode(raw))" discipline every other grant path in this class
|
|
186
|
-
* uses.
|
|
176
|
+
* caller's own atomic commit of `preparation.fragment` has succeeded.
|
|
177
|
+
* Extracts the exact written bytes, the same "never round-trip
|
|
178
|
+
* encode(decode(raw))" discipline every other grant path here uses.
|
|
187
179
|
*/
|
|
188
180
|
recordFoldedAcquire(workflowId: string, preparation: WorkflowClaimAcquirePreparation): void;
|
|
189
181
|
/**
|
|
190
182
|
* `renew`: conditions on the exact holder bytes this engine last wrote.
|
|
191
183
|
* Concurrent calls for the same id share the one in-flight promise. A
|
|
192
184
|
* CAS-false result marks the claim lost locally and emits
|
|
193
|
-
* `WeftWorkflowClaimLostWarning
|
|
194
|
-
* touches any other tracked claim.
|
|
185
|
+
* `WeftWorkflowClaimLostWarning`.
|
|
195
186
|
*/
|
|
196
187
|
renew(workflowId: string): Promise<WorkflowClaimRenewResult>;
|
|
197
188
|
/**
|
|
198
189
|
* `release`: stops new renewals for `workflowId` and awaits any renewal
|
|
199
|
-
* already in flight (
|
|
200
|
-
*
|
|
201
|
-
*
|
|
202
|
-
*
|
|
203
|
-
*
|
|
204
|
-
*
|
|
205
|
-
*
|
|
190
|
+
* already in flight (best-effort — swallows a thrown rejection) before
|
|
191
|
+
* building the expected bytes, so the two never race the same holder
|
|
192
|
+
* bytes. Deletes only the holder key, never the epoch key, so a
|
|
193
|
+
* successor's next `acquire` reads the true prior epoch.
|
|
194
|
+
*
|
|
195
|
+
* `expectedEpoch`, when given, makes this a no-op (`'not-held'`) against
|
|
196
|
+
* any entry but the exact generation the caller means to undo — see the
|
|
197
|
+
* module doc's "Epoch-guarded release". A thrown storage error during the
|
|
198
|
+
* CAS still forgets the LOCAL entry (identity-guarded like the
|
|
199
|
+
* success/failure paths below) before rethrowing, so a caller giving up
|
|
200
|
+
* after a failed release does not leave the renewal task renewing forever
|
|
201
|
+
* (WFT-134).
|
|
206
202
|
*/
|
|
207
|
-
release(workflowId: string): Promise<WorkflowClaimReleaseResult>;
|
|
203
|
+
release(workflowId: string, expectedEpoch?: number): Promise<WorkflowClaimReleaseResult>;
|
|
208
204
|
/**
|
|
209
205
|
* `takeover`: reads the holder and epoch keys fresh, and only attempts the
|
|
210
206
|
* CAS once the holder is not live — either its grace-adjusted `expiresAt`
|
|
211
207
|
* has passed ({@link isWorkflowClaimExpired}), or the holder bytes are
|
|
212
208
|
* foreign/undecodable garbage no valid engine could have written (mirrors
|
|
213
|
-
* `lease-manager.ts`'s "garbage is not a live owner" treatment
|
|
214
|
-
*
|
|
215
|
-
*
|
|
216
|
-
*
|
|
217
|
-
* treated defensively as nothing safe to fence a takeover against.
|
|
209
|
+
* `lease-manager.ts`'s "garbage is not a live owner" treatment). A holder
|
|
210
|
+
* with no epoch key violates the write invariant — the two are always
|
|
211
|
+
* written together, and the epoch key is never deleted — so it is treated
|
|
212
|
+
* defensively as nothing safe to fence a takeover against.
|
|
218
213
|
*/
|
|
219
214
|
takeover(workflowId: string): Promise<WorkflowClaimTakeoverResult>;
|
|
220
215
|
/**
|
|
221
|
-
*
|
|
222
|
-
*
|
|
223
|
-
*
|
|
224
|
-
*
|
|
216
|
+
* Forget this engine's LOCAL tracking entry for `workflowId` — no durable
|
|
217
|
+
* write, just `this.#claims.delete`. Caller: `suspendWorkflow` (WFT-134),
|
|
218
|
+
* whose commit already durably deletes `wf-owner-holder:<id>`, leaving only
|
|
219
|
+
* this stale cache entry — else {@link currentEpoch} stays non-null for a
|
|
220
|
+
* gone holder, routing a same-engine `resume()` onto the stale-cache fast
|
|
221
|
+
* path in `acquireStandaloneClaimBeforeResume`. No-op on an untracked id;
|
|
222
|
+
* check {@link currentEpoch} first to avoid clobbering a newer generation.
|
|
223
|
+
*/
|
|
224
|
+
forgetLocalClaim(workflowId: string): void;
|
|
225
|
+
/**
|
|
226
|
+
* Best-effort release of every claim this engine tracks, for graceful
|
|
227
|
+
* shutdown. A failed release (thrown or lost-race) is swallowed per
|
|
228
|
+
* workflow — the reclaim scan later collects any stranded claim.
|
|
225
229
|
*/
|
|
226
230
|
releaseAll(): Promise<void>;
|
|
227
231
|
}
|
|
@@ -125,7 +125,7 @@ export class WorkflowClaimRegistry {
|
|
|
125
125
|
this.#claims.set(workflowId, { ...entry, holderBytes });
|
|
126
126
|
return { status: "renewed", workflowId };
|
|
127
127
|
}
|
|
128
|
-
async release(workflowId) {
|
|
128
|
+
async release(workflowId, expectedEpoch) {
|
|
129
129
|
this.#releasing.add(workflowId);
|
|
130
130
|
try {
|
|
131
131
|
const inFlight = this.#inFlightRenewals.get(workflowId);
|
|
@@ -136,11 +136,21 @@ export class WorkflowClaimRegistry {
|
|
|
136
136
|
const entry = this.#claims.get(workflowId);
|
|
137
137
|
if (entry === void 0)
|
|
138
138
|
return { status: "not-held", workflowId };
|
|
139
|
+
if (expectedEpoch !== void 0 && entry.epoch !== expectedEpoch)
|
|
140
|
+
return { status: "not-held", workflowId };
|
|
139
141
|
const fragment = buildWorkflowClaimReleaseTransition({
|
|
140
142
|
workflowId,
|
|
141
143
|
currentEpochBytes: entry.epochBytes,
|
|
142
144
|
currentHolderBytes: entry.holderBytes
|
|
143
|
-
})
|
|
145
|
+
});
|
|
146
|
+
let committed;
|
|
147
|
+
try {
|
|
148
|
+
committed = await storageConditionalBatch(this.#claimStorage, fragment.conditions, fragment.operations);
|
|
149
|
+
} catch (error) {
|
|
150
|
+
if (this.#claims.get(workflowId) === entry)
|
|
151
|
+
this.#claims.delete(workflowId);
|
|
152
|
+
throw error;
|
|
153
|
+
}
|
|
144
154
|
if (this.#claims.get(workflowId) === entry)
|
|
145
155
|
this.#claims.delete(workflowId);
|
|
146
156
|
return { status: committed ? "released" : "lost-race", workflowId };
|
|
@@ -187,6 +197,9 @@ export class WorkflowClaimRegistry {
|
|
|
187
197
|
this.#takeoverCooldown.clear(workflowId);
|
|
188
198
|
return { status: "acquired", workflowId, epoch };
|
|
189
199
|
}
|
|
200
|
+
forgetLocalClaim(workflowId) {
|
|
201
|
+
this.#claims.delete(workflowId);
|
|
202
|
+
}
|
|
190
203
|
async releaseAll() {
|
|
191
204
|
const workflowIds = [...this.#claims.keys()];
|
|
192
205
|
await Promise.all(workflowIds.map(async (workflowId) => {
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bump helpers for the durable per-workflow-id generation counter
|
|
3
|
+
* (`wf-gen:<id>`, WFT-153). See `storage/generation-keys.ts` for the
|
|
4
|
+
* keyspace's full rationale.
|
|
5
|
+
*
|
|
6
|
+
* Mirrors `workflow-claim-transitions.ts`'s epoch-rotation fragment shape
|
|
7
|
+
* (`nextEpochFromObservedBytes` / `buildWorkflowClaimExternalTerminalRotationTransition`)
|
|
8
|
+
* deliberately: both are "mint `(observed ?? 0) + 1`, never a literal"
|
|
9
|
+
* permanently-retained counters bumped in the same atomic batch as the
|
|
10
|
+
* transition that motivates the bump.
|
|
11
|
+
*
|
|
12
|
+
* @module core/engine/workflow-generation-fence
|
|
13
|
+
*/
|
|
14
|
+
import { type BatchOperation } from '../../storage/interface.ts';
|
|
15
|
+
import type { EngineInternals } from './internals.ts';
|
|
16
|
+
import type { WorkflowClaimTransitionFragment } from './workflow-claim-transitions.ts';
|
|
17
|
+
/** Mint the next generation from bytes just read: `(decode(bytes) ?? 0) + 1`, never a literal. */
|
|
18
|
+
export declare function nextGenerationFromObservedBytes(observedGenerationBytes: Uint8Array | null): number;
|
|
19
|
+
/**
|
|
20
|
+
* Build just the `wf-gen:<id>` bump PUT operation from an ALREADY-OBSERVED
|
|
21
|
+
* value — no storage read of its own. Used by the `onTerminalConflict:
|
|
22
|
+
* 'start-new'` restart path (`start-terminal-conflict-purge.ts`), which folds
|
|
23
|
+
* this operation into its create batch and relies on that batch's own outer
|
|
24
|
+
* `duplicateIdGenerationCondition` — built from the SAME observed bytes this
|
|
25
|
+
* function bumps from — as the CAS fence, rather than a second, independent
|
|
26
|
+
* condition here. Reusing one observed value for both the fence and the bump
|
|
27
|
+
* amount is what makes the restart's own CAS trivially self-consistent: it
|
|
28
|
+
* can never fence itself out on its own legitimate restart, because the
|
|
29
|
+
* value it bumps from is exactly the value its own precondition checks.
|
|
30
|
+
*/
|
|
31
|
+
export declare function buildWorkflowGenerationBumpOperation(workflowId: string, observedGenerationBytes: Uint8Array | null): BatchOperation;
|
|
32
|
+
/**
|
|
33
|
+
* Read the current `wf-gen:<id>` value, build its bump PUT operation and (CAS
|
|
34
|
+
* condition permitting) fold BOTH into `base` — the caller's own fragment, in
|
|
35
|
+
* `purgeWorkflow` the `wf-owner-epoch` rotation fragment
|
|
36
|
+
* `buildExternalTerminalRotationFragment` already built — for a STANDALONE
|
|
37
|
+
* purge/retention commit that, unlike the `'start-new'` restart path above,
|
|
38
|
+
* has no pre-existing duplicate-id-style condition of its own to piggyback
|
|
39
|
+
* on. Folding happens HERE, not at the call site, so `purgeWorkflow` stays a
|
|
40
|
+
* flat read-then-commit without its own merge step.
|
|
41
|
+
*
|
|
42
|
+
* RESOLVED (chatgpt-codex-connector review, WFT-153): this bump PUT was
|
|
43
|
+
* previously unconditioned, on the theory that a lost update between two
|
|
44
|
+
* concurrent purges of the same id was harmless — either purge's bump moves
|
|
45
|
+
* the value away from whatever an earlier duplicate-id read observed. That
|
|
46
|
+
* theory misses a THIRD purge landing in between: purge A reads generation
|
|
47
|
+
* N and (slowly) prepares a bump to N+1; purge B — a later purge of the
|
|
48
|
+
* SAME id, after the id was reused and purged again — reads the CURRENT
|
|
49
|
+
* value N+1 and commits a bump to N+2; if A's stale N+1 write then commits
|
|
50
|
+
* UNCONDITIONED, it overwrites B's N+2 with A's own N+1, rolling the
|
|
51
|
+
* "monotonic" counter backward. A cross-engine start that captured the
|
|
52
|
+
* intermediate absent/N+1 pair during the window before B's purge could then
|
|
53
|
+
* pass its `duplicateIdGenerationCondition` re-check after the rollback, even
|
|
54
|
+
* though a run genuinely executed and was purged in between — reopening the
|
|
55
|
+
* exact ABA this key exists to close.
|
|
56
|
+
*
|
|
57
|
+
* The condition folded in now closes that: `expectedValue` is the SAME
|
|
58
|
+
* `observedGenerationBytes` the bump amount is minted from, so a lost race
|
|
59
|
+
* (another purge already changed `wf-gen:<id>` since this read) fails the
|
|
60
|
+
* CAS instead of overwriting a newer generation with a stale one. Gated on
|
|
61
|
+
* `internals.storage.capabilities().conditionalBatch` (the same
|
|
62
|
+
* capability-conditioned pattern `buildWorkflowStateCommit` uses in
|
|
63
|
+
* `storage-io.ts`) rather than required unconditionally: a backend that
|
|
64
|
+
* honestly reports no `conditionalBatch` support keeps the pre-existing
|
|
65
|
+
* unconditioned bump (a residual, capability-limited ABA window, not a new
|
|
66
|
+
* regression) instead of newly requiring a capability purge never required
|
|
67
|
+
* before this fix — purge must keep working, degraded, on such backends.
|
|
68
|
+
*
|
|
69
|
+
* Runs under EVERY ownership mode — unlike the `wf-owner-epoch` rotation
|
|
70
|
+
* fragment, which is a no-op outside `ownership: 'workflow-lease'`, this
|
|
71
|
+
* always reads and bumps: the ABA hole it closes exists under `'none'` and
|
|
72
|
+
* `'lease'` too.
|
|
73
|
+
*/
|
|
74
|
+
export declare function foldWorkflowGenerationBumpForPurge(internals: EngineInternals, workflowId: string, base: WorkflowClaimTransitionFragment): Promise<WorkflowClaimTransitionFragment>;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { KEYS } from "../../storage/interface.js";
|
|
2
|
+
import { decodeGeneration, encodeGeneration } from "./generation-codec.js";
|
|
3
|
+
export function nextGenerationFromObservedBytes(observedGenerationBytes) {
|
|
4
|
+
return ((observedGenerationBytes === null ? null : decodeGeneration(observedGenerationBytes)) ?? 0) + 1;
|
|
5
|
+
}
|
|
6
|
+
export function buildWorkflowGenerationBumpOperation(workflowId, observedGenerationBytes) {
|
|
7
|
+
return {
|
|
8
|
+
type: "put",
|
|
9
|
+
key: KEYS.workflowGeneration(workflowId),
|
|
10
|
+
value: encodeGeneration(nextGenerationFromObservedBytes(observedGenerationBytes))
|
|
11
|
+
};
|
|
12
|
+
}
|
|
13
|
+
export async function foldWorkflowGenerationBumpForPurge(internals, workflowId, base) {
|
|
14
|
+
const key = KEYS.workflowGeneration(workflowId), observedGenerationBytes = await internals.storage.get(key), operation = buildWorkflowGenerationBumpOperation(workflowId, observedGenerationBytes), condition = internals.storage.capabilities().conditionalBatch ? { key, expectedValue: observedGenerationBytes } : void 0;
|
|
15
|
+
return {
|
|
16
|
+
operations: [...base.operations, operation],
|
|
17
|
+
conditions: base.conditions.concat(condition ?? [])
|
|
18
|
+
};
|
|
19
|
+
}
|
|
@@ -14,6 +14,21 @@ export declare class StartWorkflowValidationError extends WeftError<'StartWorkfl
|
|
|
14
14
|
}
|
|
15
15
|
export declare const assertExclusiveStartWorkflowOptions: (startAt: unknown, startAfter: unknown) => void;
|
|
16
16
|
export declare const coerceStartWorkflowId: (value: unknown, fieldName: string) => string;
|
|
17
|
+
/**
|
|
18
|
+
* Coerce a caller-supplied `options.id` for an internal REPLAY of a start
|
|
19
|
+
* that was already accepted once before (WFT-95). Deliberately uses the
|
|
20
|
+
* decode-compatible {@link assertDecodableWorkflowId}, not the strict
|
|
21
|
+
* `.`/`..`-rejecting {@link assertValidWorkflowId} that
|
|
22
|
+
* {@link coerceStartWorkflowId} enforces: this path exists only for the three
|
|
23
|
+
* internal callers that replay an id which was already durably admitted
|
|
24
|
+
* before strict admission existed (a drained schedule queued-run, a bulk
|
|
25
|
+
* failed-workflow retry rebuilding from persisted input, or a child-workflow
|
|
26
|
+
* crash-reattach) — see `startWorkflow`'s `skipAdmissionIdCheck` parameter.
|
|
27
|
+
* It must never be reachable from a public start surface (REST, JSON-RPC,
|
|
28
|
+
* `engine.start`, `ctx.startChild`), because that would let a genuinely
|
|
29
|
+
* fresh caller admit `.`/`..` again.
|
|
30
|
+
*/
|
|
31
|
+
export declare const coerceReplayWorkflowId: (value: unknown, fieldName: string) => string;
|
|
17
32
|
/**
|
|
18
33
|
* Coerce a transport-supplied idempotency key to a non-empty string. The key is
|
|
19
34
|
* a caller-chosen dedup token (it becomes part of a `start-idem:` storage key),
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { parseDuration } from "./scheduler.js";
|
|
2
2
|
import { WeftError } from "./weft-error.js";
|
|
3
|
-
import { assertValidWorkflowId } from "./workflow-identifiers.js";
|
|
3
|
+
import { assertDecodableWorkflowId, assertValidWorkflowId } from "./workflow-identifiers.js";
|
|
4
4
|
export const MAX_WORKFLOW_TAGS = 32, MAX_WORKFLOW_TAG_BYTES = 128, MAX_IDEMPOTENCY_KEY_BYTES = 117;
|
|
5
5
|
const textEncoder = new TextEncoder, EXCLUSIVE_START_WORKFLOW_OPTIONS_ERROR = "Provide only one of startAt or startAfter";
|
|
6
6
|
|
|
@@ -22,6 +22,16 @@ export const assertExclusiveStartWorkflowOptions = (startAt, startAfter) => {
|
|
|
22
22
|
const message = error instanceof Error ? error.message : String(error);
|
|
23
23
|
throw new StartWorkflowValidationError(message);
|
|
24
24
|
}
|
|
25
|
+
}, coerceReplayWorkflowId = (value, fieldName) => {
|
|
26
|
+
if (typeof value !== "string")
|
|
27
|
+
throw new StartWorkflowValidationError(`${fieldName} must be a string`);
|
|
28
|
+
try {
|
|
29
|
+
assertDecodableWorkflowId(value, fieldName);
|
|
30
|
+
return value;
|
|
31
|
+
} catch (error) {
|
|
32
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
33
|
+
throw new StartWorkflowValidationError(message);
|
|
34
|
+
}
|
|
25
35
|
}, coerceStartWorkflowIdempotencyKey = (value, fieldName) => {
|
|
26
36
|
if (typeof value !== "string")
|
|
27
37
|
throw new StartWorkflowValidationError(`${fieldName} must be a string`);
|
|
@@ -1 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Assert every workflow-id constraint that predates WFT-95: a string,
|
|
3
|
+
* non-empty, at most {@link MAX_WORKFLOW_ID_LENGTH} characters, and free of
|
|
4
|
+
* control characters. Deliberately does NOT reject the exact strings `.` or
|
|
5
|
+
* `..` — those were valid workflow ids before WFT-95 and may already be
|
|
6
|
+
* durably persisted (a schedule id, a persisted `currentWorkflowId`, a
|
|
7
|
+
* queued run's `workflowId`, schedule-run metadata, an `executionStateOwnerId`
|
|
8
|
+
* or `parentWorkflowId`/`restartedFrom.workflowId` on a decoded
|
|
9
|
+
* `WorkflowState`). Decode and schedule-control (lookup, pause, resume,
|
|
10
|
+
* cancel, update) paths must keep accepting them so an upgrade doesn't
|
|
11
|
+
* strand pre-existing data or make a pre-existing schedule/workflow
|
|
12
|
+
* unmanageable; only fresh admission ({@link assertValidWorkflowId}) adds
|
|
13
|
+
* the `.`/`..` rejection.
|
|
14
|
+
*
|
|
15
|
+
* Takes `unknown`, not `string` (WFT-95 review): every caller passes an
|
|
16
|
+
* already-decoded field whose static `WorkflowState`/`ScheduleState` type
|
|
17
|
+
* says `string` but whose runtime shape is untrusted — the storage record
|
|
18
|
+
* could be corrupted. Without an explicit `typeof` guard here, a decoded
|
|
19
|
+
* array of strings would pass (`.length`, iteration, and
|
|
20
|
+
* `containsControlCharacter()`'s per-element `codePointAt()` all succeed on
|
|
21
|
+
* an array too), silently accepting a malformed field instead of dropping
|
|
22
|
+
* it — the same guard `coerceStartWorkflowId()` performed before this
|
|
23
|
+
* predicate existed.
|
|
24
|
+
*/
|
|
25
|
+
export declare function assertDecodableWorkflowId(id: unknown, fieldName?: string): asserts id is string;
|
|
26
|
+
/** Whether `id` satisfies {@link assertDecodableWorkflowId}. */
|
|
27
|
+
export declare function isDecodableWorkflowId(id: unknown): boolean;
|
|
28
|
+
/**
|
|
29
|
+
* Whether `id` is exactly `.` or `..` — the two literals
|
|
30
|
+
* {@link assertValidWorkflowId} rejects at fresh admission (WFT-95). Exported
|
|
31
|
+
* so callers that need to recognize "this id is the one strict admission
|
|
32
|
+
* would reject" without re-running the full assertion (for example, to
|
|
33
|
+
* decide whether a caller-facing id MIGHT be a legacy pre-WFT-95 record
|
|
34
|
+
* worth checking storage for) share one definition instead of re-deriving
|
|
35
|
+
* the literal comparison.
|
|
36
|
+
*/
|
|
37
|
+
export declare function isReservedWorkflowIdLiteral(id: string): boolean;
|
|
1
38
|
export declare function assertValidWorkflowId(id: string, fieldName?: string): void;
|
|
@@ -9,7 +9,9 @@ function containsControlCharacter(value) {
|
|
|
9
9
|
}
|
|
10
10
|
return !1;
|
|
11
11
|
}
|
|
12
|
-
export function
|
|
12
|
+
export function assertDecodableWorkflowId(id, fieldName = "options.id") {
|
|
13
|
+
if (typeof id !== "string")
|
|
14
|
+
throw Error(`${fieldName} must be a string`);
|
|
13
15
|
if (id.length === 0)
|
|
14
16
|
throw Error(`${fieldName} must not be an empty string`);
|
|
15
17
|
if (id.length > MAX_WORKFLOW_ID_LENGTH)
|
|
@@ -17,3 +19,19 @@ export function assertValidWorkflowId(id, fieldName = "options.id") {
|
|
|
17
19
|
if (containsControlCharacter(id))
|
|
18
20
|
throw Error(`${fieldName} must not contain control characters`);
|
|
19
21
|
}
|
|
22
|
+
export function isDecodableWorkflowId(id) {
|
|
23
|
+
try {
|
|
24
|
+
assertDecodableWorkflowId(id);
|
|
25
|
+
return !0;
|
|
26
|
+
} catch {
|
|
27
|
+
return !1;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
export function isReservedWorkflowIdLiteral(id) {
|
|
31
|
+
return id === "." || id === "..";
|
|
32
|
+
}
|
|
33
|
+
export function assertValidWorkflowId(id, fieldName = "options.id") {
|
|
34
|
+
if (isReservedWorkflowIdLiteral(id))
|
|
35
|
+
throw Error(`${fieldName} must not be "." or ".."`);
|
|
36
|
+
assertDecodableWorkflowId(id, fieldName);
|
|
37
|
+
}
|