@lostgradient/weft 0.20.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/cli/generated/operation-catalog.snapshot.json +207 -80
- package/dist/cli/generated/operation-client.generated.d.ts +120 -43
- package/dist/client/http-client-storage.js +1 -1
- package/dist/core/engine/activity-reconciliation.d.ts +2 -2
- package/dist/core/engine/activity-reconciliation.js +12 -8
- package/dist/core/engine/async-activity-completion.js +12 -4
- package/dist/core/engine/async-activity-records.d.ts +11 -1
- package/dist/core/engine/async-activity-records.js +4 -3
- package/dist/core/engine/attributes-tags.d.ts +11 -1
- package/dist/core/engine/attributes-tags.js +4 -3
- package/dist/core/engine/bulk-operations-purge.js +5 -2
- package/dist/core/engine/bulk-operations.js +17 -5
- package/dist/core/engine/callback-creators-bundles.js +1 -1
- package/dist/core/engine/callback-creators-core.js +4 -2
- package/dist/core/engine/checkpoint-io.js +1 -1
- package/dist/core/engine/child-workflow.js +2 -1
- package/dist/core/engine/completed-review-storage.js +1 -1
- package/dist/core/engine/condition-waiters.d.ts +45 -3
- package/dist/core/engine/condition-waiters.js +12 -3
- package/dist/core/engine/disposal.js +6 -0
- package/dist/core/engine/engine-internal-types.d.ts +12 -4
- package/dist/core/engine/fenced-write.d.ts +46 -15
- package/dist/core/engine/fenced-write.js +39 -6
- package/dist/core/engine/finalizer-state.d.ts +3 -1
- package/dist/core/engine/handle-result.d.ts +49 -1
- package/dist/core/engine/handle-result.js +95 -20
- package/dist/core/engine/index.d.ts +13 -5
- package/dist/core/engine/index.js +155 -5
- package/dist/core/engine/inline-launch-queue.js +9 -0
- package/dist/core/engine/internals.d.ts +73 -0
- package/dist/core/engine/invoke-update-handler.d.ts +19 -0
- package/dist/core/engine/invoke-update-handler.js +7 -0
- package/dist/core/engine/lease-deposition.d.ts +132 -0
- package/dist/core/engine/lease-deposition.js +31 -0
- package/dist/core/engine/lease-errors.d.ts +80 -9
- package/dist/core/engine/lease-errors.js +28 -2
- package/dist/core/engine/lifecycle/resume.js +3 -1
- package/dist/core/engine/lifecycle/standalone-claim-acquire.d.ts +33 -0
- package/dist/core/engine/lifecycle/standalone-claim-acquire.js +25 -0
- package/dist/core/engine/lifecycle/start-commit.js +20 -6
- package/dist/core/engine/lifecycle/transition.d.ts +15 -1
- package/dist/core/engine/lifecycle/transition.js +6 -3
- package/dist/core/engine/operations-activity.js +1 -1
- package/dist/core/engine/operations-time.js +21 -7
- package/dist/core/engine/owner-side-signal-poll.d.ts +132 -0
- package/dist/core/engine/owner-side-signal-poll.js +22 -0
- package/dist/core/engine/owner-side-update-poll.d.ts +116 -0
- package/dist/core/engine/owner-side-update-poll.js +22 -0
- package/dist/core/engine/ownership-bootstrap.d.ts +280 -0
- package/dist/core/engine/ownership-bootstrap.js +89 -0
- package/dist/core/engine/ownership-mode-marker.d.ts +82 -0
- package/dist/core/engine/ownership-mode-marker.js +52 -0
- package/dist/core/engine/ownership-options.d.ts +22 -6
- package/dist/core/engine/ownership-options.js +42 -12
- package/dist/core/engine/pending-updates.js +35 -24
- package/dist/core/engine/queries.d.ts +70 -0
- package/dist/core/engine/queries.js +58 -4
- package/dist/core/engine/schedules.js +1 -1
- package/dist/core/engine/signals.d.ts +11 -1
- package/dist/core/engine/signals.js +11 -7
- package/dist/core/engine/sleep-timer-acknowledgements.d.ts +68 -0
- package/dist/core/engine/sleep-timer-acknowledgements.js +19 -0
- package/dist/core/engine/storage-io.d.ts +58 -10
- package/dist/core/engine/storage-io.js +27 -9
- package/dist/core/engine/termination/cleanup.d.ts +5 -4
- package/dist/core/engine/termination/complete.js +23 -4
- package/dist/core/engine/termination/finalizer-claim.js +4 -4
- package/dist/core/engine/termination/state-commit-callbacks.d.ts +15 -0
- package/dist/core/engine/termination/state-commit-callbacks.js +0 -0
- package/dist/core/engine/termination/suspend.js +1 -1
- package/dist/core/engine/update-validation.d.ts +28 -0
- package/dist/core/engine/update-validation.js +37 -0
- package/dist/core/engine/updates.d.ts +17 -16
- package/dist/core/engine/updates.js +24 -47
- package/dist/core/engine/wake-ownership-check.d.ts +86 -0
- package/dist/core/engine/wake-ownership-check.js +19 -0
- package/dist/core/engine/wake-ownership-guard.d.ts +46 -0
- package/dist/core/engine/wake-ownership-guard.js +23 -0
- package/dist/core/engine/workflow-claim-codec.d.ts +64 -0
- package/dist/core/engine/workflow-claim-codec.js +44 -0
- package/dist/core/engine/workflow-claim-cooldown.d.ts +60 -0
- package/dist/core/engine/workflow-claim-cooldown.js +17 -0
- package/dist/core/engine/workflow-claim-fold.d.ts +82 -0
- package/dist/core/engine/workflow-claim-fold.js +53 -0
- package/dist/core/engine/workflow-claim-metrics.d.ts +131 -0
- package/dist/core/engine/workflow-claim-metrics.js +62 -0
- package/dist/core/engine/workflow-claim-reclaim-scan.d.ts +96 -0
- package/dist/core/engine/workflow-claim-reclaim-scan.js +66 -0
- package/dist/core/engine/workflow-claim-reclaim-target.d.ts +107 -0
- package/dist/core/engine/workflow-claim-reclaim-target.js +155 -0
- package/dist/core/engine/workflow-claim-registry.d.ts +227 -0
- package/dist/core/engine/workflow-claim-registry.js +198 -0
- package/dist/core/engine/workflow-claim-renewal-interval.d.ts +50 -0
- package/dist/core/engine/workflow-claim-renewal-interval.js +132 -0
- package/dist/core/engine/workflow-claim-renewal-subpasses.d.ts +265 -0
- package/dist/core/engine/workflow-claim-renewal-subpasses.js +104 -0
- package/dist/core/engine/workflow-claim-renewal-task.d.ts +159 -0
- package/dist/core/engine/workflow-claim-renewal-task.js +65 -0
- package/dist/core/engine/workflow-claim-transitions.d.ts +186 -0
- package/dist/core/engine/workflow-claim-transitions.js +120 -0
- package/dist/core/inline-execution-strategy.d.ts +20 -0
- package/dist/core/inline-execution-strategy.js +15 -4
- package/dist/core/scheduler/scheduler-class.js +3 -3
- package/dist/core/scheduler/timer-sources.d.ts +13 -0
- package/dist/core/scheduler/timer-sources.js +1 -1
- package/dist/core/types/options.d.ts +17 -1
- package/dist/core/weft-error.d.ts +1 -1
- package/dist/core/weft-error.js +2 -1
- package/dist/http.js +2 -2
- package/dist/index.d.ts +2 -1
- package/dist/index.js +8 -0
- package/dist/indexeddb.js +1 -1
- package/dist/json-schema.js +3 -3
- package/dist/observability/index.js +2 -2
- package/dist/server/operations/get-task-diagnostics.d.ts +115 -25
- package/dist/server/operations/get-task-diagnostics.js +93 -7
- package/dist/storage/auto.js +1 -1
- package/dist/storage/bun-sql.js +115 -2
- package/dist/storage/compressed-storage.js +1 -1
- package/dist/storage/http.js +1 -1
- package/dist/storage/index.d.ts +38 -35
- package/dist/storage/interface.d.ts +87 -125
- package/dist/storage/interface.js +1 -1
- package/dist/storage/key-encoding.d.ts +51 -0
- package/dist/storage/key-encoding.js +13 -0
- package/dist/storage/key-prefixes.d.ts +1 -1
- package/dist/storage/key-prefixes.js +3 -0
- package/dist/storage/lmdb.js +1 -1
- package/dist/storage/memory.js +1 -1
- package/dist/storage/neon.js +4 -4
- package/dist/storage/node-sqlite.js +115 -2
- package/dist/storage/ownership-keys.d.ts +55 -0
- package/dist/storage/ownership-keys.js +6 -0
- package/dist/storage/postgres.js +4 -4
- package/dist/storage/resolve.js +1 -1
- package/dist/storage/scoped-storage.js +1 -1
- package/dist/storage/testing.js +1 -1
- package/dist/storage/turso.js +2 -2
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/web-extension.js +1 -1
- package/dist/worker/protocol.js +1 -1
- package/package.json +3 -2
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The stateful per-workflow claim registry — the object that actually
|
|
3
|
+
* performs durable IO for the `wf-owner-epoch:<id>` / `wf-owner-holder:<id>`
|
|
4
|
+
* claim transitions specified in
|
|
5
|
+
* [ADR 0002 § Ownership transitions](../../../documentation/contributing/architecture-decisions/0002-multiengine-per-workflow-ownership.md#ownership-transitions).
|
|
6
|
+
*
|
|
7
|
+
* `workflow-claim-transitions.ts` supplies pure `{ conditions, operations }`
|
|
8
|
+
* fragments; this module is the thin, stateful layer around them that reads
|
|
9
|
+
* storage, executes `storageConditionalBatch`, and tracks — for every claim
|
|
10
|
+
* THIS engine currently holds — the exact epoch and holder bytes it last
|
|
11
|
+
* wrote. Renewal and release condition on those exact bytes, extracted from
|
|
12
|
+
* the fragment they were just written by rather than re-encoded from the
|
|
13
|
+
* fields the registry happens to know, mirroring `lease-manager.ts`'s
|
|
14
|
+
* "never round-trip encode(decode(raw))" discipline.
|
|
15
|
+
*
|
|
16
|
+
* **Scope.** The unit itself: acquire, renew, release, takeover, and
|
|
17
|
+
* release-all — plus, additively, {@link WorkflowClaimRegistry.prepareAcquireFragment}
|
|
18
|
+
* and {@link WorkflowClaimRegistry.recordFoldedAcquire}, the two-step seam a
|
|
19
|
+
* caller uses to fold `acquire` into ITS OWN atomic enabling write (a create
|
|
20
|
+
* batch, a delayed-start pending→running transition, a failed-workflow
|
|
21
|
+
* reactivation) instead of committing the fragment through this registry's
|
|
22
|
+
* own `acquire()`. Driving `renew` and the reclaim scan from a lifecycle
|
|
23
|
+
* task, and turning a lost `acquire`/`takeover` into
|
|
24
|
+
* `WorkflowClaimUnavailableError` for explicit single-workflow callers, are
|
|
25
|
+
* still each call site's own responsibility — this registry never throws
|
|
26
|
+
* that error itself. Per the ADR, background scanning never throws it either
|
|
27
|
+
* — it skips the workflow and continues — so every method here returns a
|
|
28
|
+
* discriminated result instead of throwing on a lost CAS, leaving that
|
|
29
|
+
* decision to the caller — `takeover` also gates on a per-workflow-id
|
|
30
|
+
* anti-thrash cooldown ({@link WorkflowClaimTakeoverCooldown}). Still out of
|
|
31
|
+
* scope: `weft_workflow_claim_*` metrics, `wakeOwnershipCheck`, and external
|
|
32
|
+
* terminal-transition rotation (any engine may commit those unconditioned).
|
|
33
|
+
*
|
|
34
|
+
* **Renewal-vs-release serialization.** A per-workflow in-flight-renewal
|
|
35
|
+
* promise (mirroring `lease-manager.ts`'s single `inFlightRenewal`) lets
|
|
36
|
+
* `release()` await a renewal already in progress before reading the cached
|
|
37
|
+
* bytes it conditions on; otherwise both race the same holder bytes and
|
|
38
|
+
* whichever commits second loses its CAS. A `releasing` set stops a NEW
|
|
39
|
+
* renewal from starting once release has begun, so a `renew()` arriving
|
|
40
|
+
* mid-release fails fast as `'not-held'`. A renewal that THROWS (a transient
|
|
41
|
+
* storage error, not a CAS-false result) is not a lost claim: it propagates
|
|
42
|
+
* to its caller leaving cached bytes untouched, and `release()`'s wait
|
|
43
|
+
* swallows it — a storage hiccup must not fail a terminal or shutdown release.
|
|
44
|
+
*
|
|
45
|
+
* @module core/engine/workflow-claim-registry
|
|
46
|
+
*/
|
|
47
|
+
import { type Storage } from '../../storage/interface.ts';
|
|
48
|
+
import { type EmitWorkflowLeaseWarning } from './lease-deposition.ts';
|
|
49
|
+
import { type WorkflowClaimTransitionFragment } from './workflow-claim-transitions.ts';
|
|
50
|
+
/** Options for {@link WorkflowClaimRegistry}. */
|
|
51
|
+
export type WorkflowClaimRegistryOptions = {
|
|
52
|
+
storage: Storage;
|
|
53
|
+
/** This engine's identity, minted once per process — the holder record's `engineId`. */
|
|
54
|
+
engineId: string;
|
|
55
|
+
/** Wall-clock source (ms), injected so tests can drive time deterministically. */
|
|
56
|
+
getNow: () => number;
|
|
57
|
+
/** `workflowClaimTtl` (ms), resolved by `ownership-options.ts`. */
|
|
58
|
+
claimTtlMs: number;
|
|
59
|
+
/**
|
|
60
|
+
* `workflowClaimRenewInterval` (ms), resolved by `ownership-options.ts` —
|
|
61
|
+
* feeds `isWorkflowClaimExpired`'s grace term.
|
|
62
|
+
*/
|
|
63
|
+
claimRenewIntervalMs: number;
|
|
64
|
+
/** Operator-warning seam; defaults to `process.emitWarning` via {@link emitWorkflowClaimLostWarning}. */
|
|
65
|
+
warn?: EmitWorkflowLeaseWarning;
|
|
66
|
+
};
|
|
67
|
+
/** Result of {@link WorkflowClaimRegistry.acquire}. */
|
|
68
|
+
export type WorkflowClaimAcquireResult = {
|
|
69
|
+
status: 'acquired';
|
|
70
|
+
workflowId: string;
|
|
71
|
+
epoch: number;
|
|
72
|
+
} | {
|
|
73
|
+
status: 'lost-race';
|
|
74
|
+
workflowId: string;
|
|
75
|
+
heldBy: string | null;
|
|
76
|
+
};
|
|
77
|
+
/** Result of {@link WorkflowClaimRegistry.renew}. */
|
|
78
|
+
export type WorkflowClaimRenewResult = {
|
|
79
|
+
status: 'renewed';
|
|
80
|
+
workflowId: string;
|
|
81
|
+
} | {
|
|
82
|
+
status: 'lost';
|
|
83
|
+
workflowId: string;
|
|
84
|
+
} | {
|
|
85
|
+
status: 'not-held';
|
|
86
|
+
workflowId: string;
|
|
87
|
+
};
|
|
88
|
+
/** Result of {@link WorkflowClaimRegistry.release}. */
|
|
89
|
+
export type WorkflowClaimReleaseResult = {
|
|
90
|
+
status: 'released';
|
|
91
|
+
workflowId: string;
|
|
92
|
+
} | {
|
|
93
|
+
status: 'lost-race';
|
|
94
|
+
workflowId: string;
|
|
95
|
+
} | {
|
|
96
|
+
status: 'not-held';
|
|
97
|
+
workflowId: string;
|
|
98
|
+
};
|
|
99
|
+
/**
|
|
100
|
+
* A prepared, not-yet-committed `acquire` — the output of
|
|
101
|
+
* {@link WorkflowClaimRegistry.prepareAcquireFragment}, meant to be merged
|
|
102
|
+
* into a caller's own atomic enabling write and then handed back to
|
|
103
|
+
* {@link WorkflowClaimRegistry.recordFoldedAcquire} once that write commits.
|
|
104
|
+
*/
|
|
105
|
+
export type WorkflowClaimAcquirePreparation = {
|
|
106
|
+
fragment: WorkflowClaimTransitionFragment;
|
|
107
|
+
epoch: number;
|
|
108
|
+
claimedAt: number;
|
|
109
|
+
};
|
|
110
|
+
/** Result of {@link WorkflowClaimRegistry.takeover}. */
|
|
111
|
+
export type WorkflowClaimTakeoverResult = {
|
|
112
|
+
status: 'acquired';
|
|
113
|
+
workflowId: string;
|
|
114
|
+
epoch: number;
|
|
115
|
+
} | {
|
|
116
|
+
status: 'lost-race';
|
|
117
|
+
workflowId: string;
|
|
118
|
+
heldBy: string | null;
|
|
119
|
+
} | {
|
|
120
|
+
status: 'not-expired';
|
|
121
|
+
workflowId: string;
|
|
122
|
+
heldBy: string | null;
|
|
123
|
+
expiresAt: number;
|
|
124
|
+
} | {
|
|
125
|
+
status: 'no-claim';
|
|
126
|
+
workflowId: string;
|
|
127
|
+
} | {
|
|
128
|
+
status: 'backoff-skipped';
|
|
129
|
+
workflowId: string;
|
|
130
|
+
};
|
|
131
|
+
/**
|
|
132
|
+
* Owns this engine's per-workflow ownership claims: reads storage, executes
|
|
133
|
+
* the pure transition fragments, and tracks the exact bytes it last wrote for
|
|
134
|
+
* every claim it currently holds. See the module doc for scope.
|
|
135
|
+
*/
|
|
136
|
+
export declare class WorkflowClaimRegistry {
|
|
137
|
+
#private;
|
|
138
|
+
constructor(options: WorkflowClaimRegistryOptions);
|
|
139
|
+
/** The epoch this engine currently believes it holds for `workflowId`, or `null` if untracked. */
|
|
140
|
+
currentEpoch(workflowId: string): number | null;
|
|
141
|
+
/** This engine's identity; with {@link currentEpoch} it forms the generation a wake check compares. */
|
|
142
|
+
get engineId(): string;
|
|
143
|
+
/**
|
|
144
|
+
* Every workflow id this engine currently tracks a live claim for — active
|
|
145
|
+
* or parked. A defensive-copy snapshot, mirroring `releaseAll`'s own
|
|
146
|
+
* `[...this.#claims.keys()]` read: callers (the claim-renewal task, an
|
|
147
|
+
* active-claims metrics gauge) must not observe mutations to this registry's
|
|
148
|
+
* internal map while iterating a snapshot they already took.
|
|
149
|
+
*/
|
|
150
|
+
listHeldWorkflowIds(): readonly string[];
|
|
151
|
+
/**
|
|
152
|
+
* Defensive copy of the epoch bytes this engine last wrote for
|
|
153
|
+
* `workflowId`, for fencing durable writes — `null` if untracked. A copy so
|
|
154
|
+
* a caller mutating the returned buffer cannot corrupt this registry's
|
|
155
|
+
* cached fencing token.
|
|
156
|
+
*/
|
|
157
|
+
currentEpochBytes(workflowId: string): Uint8Array | null;
|
|
158
|
+
/**
|
|
159
|
+
* `acquire`: always reads both keys fresh — never assumes absence — then
|
|
160
|
+
* builds and attempts the fragment from what it read. On a lost CAS,
|
|
161
|
+
* resolves `heldBy` from the holder bytes already read when they were
|
|
162
|
+
* non-null; otherwise (a competitor raced in between the read and the
|
|
163
|
+
* write) re-reads to report the true current holder.
|
|
164
|
+
*/
|
|
165
|
+
acquire(workflowId: string): Promise<WorkflowClaimAcquireResult>;
|
|
166
|
+
/**
|
|
167
|
+
* Read fresh `wf-owner-epoch:<workflowId>` bytes and build the pure
|
|
168
|
+
* `acquire` fragment WITHOUT committing it or updating this registry's
|
|
169
|
+
* tracking — for a caller that folds the fragment into ITS OWN atomic
|
|
170
|
+
* enabling write instead of letting {@link acquire} commit it alone. The
|
|
171
|
+
* caller merges `fragment.conditions`/`fragment.operations` into its own
|
|
172
|
+
* operation list, commits ONE atomic `storageConditionalBatch`, and —
|
|
173
|
+
* ONLY on success — calls {@link recordFoldedAcquire} with this SAME
|
|
174
|
+
* preparation. Safe to call again on every retry attempt: this always
|
|
175
|
+
* re-reads fresh bytes, so a stale epoch from an earlier attempt never
|
|
176
|
+
* dooms a later one.
|
|
177
|
+
*/
|
|
178
|
+
prepareAcquireFragment(workflowId: string): Promise<WorkflowClaimAcquirePreparation>;
|
|
179
|
+
/**
|
|
180
|
+
* Install the tracking entry for a claim acquired via a FOLDED enabling
|
|
181
|
+
* write (see {@link prepareAcquireFragment}) — call ONLY after the
|
|
182
|
+
* caller's own atomic commit that included `preparation.fragment`'s
|
|
183
|
+
* conditions and operations has actually succeeded. Extracts the exact
|
|
184
|
+
* bytes the fragment wrote using the same "never round-trip
|
|
185
|
+
* encode(decode(raw))" discipline every other grant path in this class
|
|
186
|
+
* uses.
|
|
187
|
+
*/
|
|
188
|
+
recordFoldedAcquire(workflowId: string, preparation: WorkflowClaimAcquirePreparation): void;
|
|
189
|
+
/**
|
|
190
|
+
* `renew`: conditions on the exact holder bytes this engine last wrote.
|
|
191
|
+
* Concurrent calls for the same id share the one in-flight promise. A
|
|
192
|
+
* CAS-false result marks the claim lost locally and emits
|
|
193
|
+
* `WeftWorkflowClaimLostWarning` — losing one workflow's claim never
|
|
194
|
+
* touches any other tracked claim.
|
|
195
|
+
*/
|
|
196
|
+
renew(workflowId: string): Promise<WorkflowClaimRenewResult>;
|
|
197
|
+
/**
|
|
198
|
+
* `release`: stops new renewals for `workflowId` and awaits any renewal
|
|
199
|
+
* already in flight (swallowing a thrown rejection — best-effort, never
|
|
200
|
+
* reject on a renewal's storage error) before building the expected bytes,
|
|
201
|
+
* so the two can never race the same holder bytes. Deletes only the holder
|
|
202
|
+
* key — the epoch key is never touched, so a successor's next `acquire`
|
|
203
|
+
* reads the true prior epoch. A lost CAS means this engine was already
|
|
204
|
+
* fenced out; the local entry is dropped either way, since there is
|
|
205
|
+
* nothing left to protect.
|
|
206
|
+
*/
|
|
207
|
+
release(workflowId: string): Promise<WorkflowClaimReleaseResult>;
|
|
208
|
+
/**
|
|
209
|
+
* `takeover`: reads the holder and epoch keys fresh, and only attempts the
|
|
210
|
+
* CAS once the holder is not live — either its grace-adjusted `expiresAt`
|
|
211
|
+
* has passed ({@link isWorkflowClaimExpired}), or the holder bytes are
|
|
212
|
+
* foreign/undecodable garbage no valid engine could have written (mirrors
|
|
213
|
+
* `lease-manager.ts`'s "garbage is not a live owner" treatment: not live,
|
|
214
|
+
* so it can be stolen via CAS on its exact observed bytes). A holder
|
|
215
|
+
* present with no epoch key violates the write invariant — the two are
|
|
216
|
+
* always written together, and the epoch key is never deleted — so it is
|
|
217
|
+
* treated defensively as nothing safe to fence a takeover against.
|
|
218
|
+
*/
|
|
219
|
+
takeover(workflowId: string): Promise<WorkflowClaimTakeoverResult>;
|
|
220
|
+
/**
|
|
221
|
+
* Best-effort release of every claim this engine currently tracks, for
|
|
222
|
+
* graceful shutdown. A failed release (thrown or lost-race) is swallowed
|
|
223
|
+
* per workflow so shutdown proceeds — the reclaim scan (a later stage)
|
|
224
|
+
* collects any stranded claim once its grace-adjusted expiry passes.
|
|
225
|
+
*/
|
|
226
|
+
releaseAll(): Promise<void>;
|
|
227
|
+
}
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
import { KEYS, storageConditionalBatch } from "../../storage/interface.js";
|
|
2
|
+
import { emitWorkflowClaimLostWarning } from "./lease-deposition.js";
|
|
3
|
+
import { decodeEpoch, decodeWorkflowClaimHolder } from "./workflow-claim-codec.js";
|
|
4
|
+
import { WorkflowClaimTakeoverCooldown } from "./workflow-claim-cooldown.js";
|
|
5
|
+
import {
|
|
6
|
+
buildWorkflowClaimAcquireTransition,
|
|
7
|
+
buildWorkflowClaimReleaseTransition,
|
|
8
|
+
buildWorkflowClaimRenewTransition,
|
|
9
|
+
buildWorkflowClaimTakeoverTransition,
|
|
10
|
+
extractPutOperationValue,
|
|
11
|
+
isWorkflowClaimExpired
|
|
12
|
+
} from "./workflow-claim-transitions.js";
|
|
13
|
+
function mintNextEpoch(observedEpochBytes) {
|
|
14
|
+
return ((observedEpochBytes === null ? null : decodeEpoch(observedEpochBytes)) ?? 0) + 1;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export class WorkflowClaimRegistry {
|
|
18
|
+
#claimStorage;
|
|
19
|
+
#engineId;
|
|
20
|
+
#getNow;
|
|
21
|
+
#claimTtlMs;
|
|
22
|
+
#claimRenewIntervalMs;
|
|
23
|
+
#warn;
|
|
24
|
+
#claims = new Map;
|
|
25
|
+
#inFlightRenewals = new Map;
|
|
26
|
+
#releasing = new Set;
|
|
27
|
+
#takeoverCooldown;
|
|
28
|
+
constructor(options) {
|
|
29
|
+
this.#claimStorage = options.storage;
|
|
30
|
+
this.#engineId = options.engineId;
|
|
31
|
+
this.#getNow = options.getNow;
|
|
32
|
+
this.#claimTtlMs = options.claimTtlMs;
|
|
33
|
+
this.#claimRenewIntervalMs = options.claimRenewIntervalMs;
|
|
34
|
+
this.#warn = options.warn;
|
|
35
|
+
this.#takeoverCooldown = new WorkflowClaimTakeoverCooldown(options);
|
|
36
|
+
}
|
|
37
|
+
currentEpoch(workflowId) {
|
|
38
|
+
return this.#claims.get(workflowId)?.epoch ?? null;
|
|
39
|
+
}
|
|
40
|
+
get engineId() {
|
|
41
|
+
return this.#engineId;
|
|
42
|
+
}
|
|
43
|
+
listHeldWorkflowIds() {
|
|
44
|
+
return [...this.#claims.keys()];
|
|
45
|
+
}
|
|
46
|
+
currentEpochBytes(workflowId) {
|
|
47
|
+
const entry = this.#claims.get(workflowId);
|
|
48
|
+
return entry === void 0 ? null : entry.epochBytes.slice();
|
|
49
|
+
}
|
|
50
|
+
async#resolveHeldBy(workflowId) {
|
|
51
|
+
const raw = await this.#claimStorage.get(KEYS.workflowOwnerHolder(workflowId));
|
|
52
|
+
if (raw === null)
|
|
53
|
+
return null;
|
|
54
|
+
return decodeWorkflowClaimHolder(raw)?.engineId ?? null;
|
|
55
|
+
}
|
|
56
|
+
async acquire(workflowId) {
|
|
57
|
+
const observedHolderBytes = await this.#claimStorage.get(KEYS.workflowOwnerHolder(workflowId)), observedEpochBytes = await this.#claimStorage.get(KEYS.workflowOwnerEpoch(workflowId)), now = this.#getNow(), fragment = buildWorkflowClaimAcquireTransition({
|
|
58
|
+
workflowId,
|
|
59
|
+
engineId: this.#engineId,
|
|
60
|
+
now,
|
|
61
|
+
claimTtlMs: this.#claimTtlMs,
|
|
62
|
+
observedEpochBytes
|
|
63
|
+
});
|
|
64
|
+
if (!await storageConditionalBatch(this.#claimStorage, fragment.conditions, fragment.operations)) {
|
|
65
|
+
const heldBy = observedHolderBytes !== null ? decodeWorkflowClaimHolder(observedHolderBytes)?.engineId ?? null : await this.#resolveHeldBy(workflowId);
|
|
66
|
+
return { status: "lost-race", workflowId, heldBy };
|
|
67
|
+
}
|
|
68
|
+
const epoch = mintNextEpoch(observedEpochBytes), epochBytes = extractPutOperationValue(fragment.operations, KEYS.workflowOwnerEpoch(workflowId)), holderBytes = extractPutOperationValue(fragment.operations, KEYS.workflowOwnerHolder(workflowId));
|
|
69
|
+
this.#claims.set(workflowId, { epoch, claimedAt: now, epochBytes, holderBytes });
|
|
70
|
+
this.#takeoverCooldown.clear(workflowId);
|
|
71
|
+
return { status: "acquired", workflowId, epoch };
|
|
72
|
+
}
|
|
73
|
+
async prepareAcquireFragment(workflowId) {
|
|
74
|
+
const observedEpochBytes = await this.#claimStorage.get(KEYS.workflowOwnerEpoch(workflowId)), now = this.#getNow();
|
|
75
|
+
return { fragment: buildWorkflowClaimAcquireTransition({
|
|
76
|
+
workflowId,
|
|
77
|
+
engineId: this.#engineId,
|
|
78
|
+
now,
|
|
79
|
+
claimTtlMs: this.#claimTtlMs,
|
|
80
|
+
observedEpochBytes
|
|
81
|
+
}), epoch: mintNextEpoch(observedEpochBytes), claimedAt: now };
|
|
82
|
+
}
|
|
83
|
+
recordFoldedAcquire(workflowId, preparation) {
|
|
84
|
+
const epochBytes = extractPutOperationValue(preparation.fragment.operations, KEYS.workflowOwnerEpoch(workflowId)), holderBytes = extractPutOperationValue(preparation.fragment.operations, KEYS.workflowOwnerHolder(workflowId));
|
|
85
|
+
this.#claims.set(workflowId, {
|
|
86
|
+
epoch: preparation.epoch,
|
|
87
|
+
claimedAt: preparation.claimedAt,
|
|
88
|
+
epochBytes,
|
|
89
|
+
holderBytes
|
|
90
|
+
});
|
|
91
|
+
this.#takeoverCooldown.clear(workflowId);
|
|
92
|
+
}
|
|
93
|
+
async renew(workflowId) {
|
|
94
|
+
const existing = this.#inFlightRenewals.get(workflowId);
|
|
95
|
+
if (existing !== void 0)
|
|
96
|
+
return existing;
|
|
97
|
+
const promise = this.#performRenew(workflowId).finally(() => {
|
|
98
|
+
this.#inFlightRenewals.delete(workflowId);
|
|
99
|
+
});
|
|
100
|
+
this.#inFlightRenewals.set(workflowId, promise);
|
|
101
|
+
return promise;
|
|
102
|
+
}
|
|
103
|
+
async#performRenew(workflowId) {
|
|
104
|
+
if (this.#releasing.has(workflowId))
|
|
105
|
+
return { status: "not-held", workflowId };
|
|
106
|
+
const entry = this.#claims.get(workflowId);
|
|
107
|
+
if (entry === void 0)
|
|
108
|
+
return { status: "not-held", workflowId };
|
|
109
|
+
const fragment = buildWorkflowClaimRenewTransition({
|
|
110
|
+
workflowId,
|
|
111
|
+
now: this.#getNow(),
|
|
112
|
+
claimTtlMs: this.#claimTtlMs,
|
|
113
|
+
currentHolderBytes: entry.holderBytes
|
|
114
|
+
});
|
|
115
|
+
if (!await storageConditionalBatch(this.#claimStorage, fragment.conditions, fragment.operations)) {
|
|
116
|
+
if (this.#claims.get(workflowId) === entry) {
|
|
117
|
+
this.#claims.delete(workflowId);
|
|
118
|
+
this.#takeoverCooldown.recordDeposition(workflowId, this.#getNow());
|
|
119
|
+
}
|
|
120
|
+
emitWorkflowClaimLostWarning(workflowId, this.#warn);
|
|
121
|
+
return { status: "lost", workflowId };
|
|
122
|
+
}
|
|
123
|
+
const holderBytes = extractPutOperationValue(fragment.operations, KEYS.workflowOwnerHolder(workflowId));
|
|
124
|
+
if (this.#claims.get(workflowId) === entry)
|
|
125
|
+
this.#claims.set(workflowId, { ...entry, holderBytes });
|
|
126
|
+
return { status: "renewed", workflowId };
|
|
127
|
+
}
|
|
128
|
+
async release(workflowId) {
|
|
129
|
+
this.#releasing.add(workflowId);
|
|
130
|
+
try {
|
|
131
|
+
const inFlight = this.#inFlightRenewals.get(workflowId);
|
|
132
|
+
if (inFlight !== void 0)
|
|
133
|
+
await inFlight.catch(() => {
|
|
134
|
+
return;
|
|
135
|
+
});
|
|
136
|
+
const entry = this.#claims.get(workflowId);
|
|
137
|
+
if (entry === void 0)
|
|
138
|
+
return { status: "not-held", workflowId };
|
|
139
|
+
const fragment = buildWorkflowClaimReleaseTransition({
|
|
140
|
+
workflowId,
|
|
141
|
+
currentEpochBytes: entry.epochBytes,
|
|
142
|
+
currentHolderBytes: entry.holderBytes
|
|
143
|
+
}), committed = await storageConditionalBatch(this.#claimStorage, fragment.conditions, fragment.operations);
|
|
144
|
+
if (this.#claims.get(workflowId) === entry)
|
|
145
|
+
this.#claims.delete(workflowId);
|
|
146
|
+
return { status: committed ? "released" : "lost-race", workflowId };
|
|
147
|
+
} finally {
|
|
148
|
+
this.#releasing.delete(workflowId);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
async takeover(workflowId) {
|
|
152
|
+
const now = this.#getNow();
|
|
153
|
+
if (this.#takeoverCooldown.isActive(workflowId, now))
|
|
154
|
+
return { status: "backoff-skipped", workflowId };
|
|
155
|
+
const observedHolderBytes = await this.#claimStorage.get(KEYS.workflowOwnerHolder(workflowId));
|
|
156
|
+
if (observedHolderBytes === null)
|
|
157
|
+
return { status: "no-claim", workflowId };
|
|
158
|
+
const observedEpochBytes = await this.#claimStorage.get(KEYS.workflowOwnerEpoch(workflowId));
|
|
159
|
+
if (observedEpochBytes === null)
|
|
160
|
+
return { status: "no-claim", workflowId };
|
|
161
|
+
const holder = decodeWorkflowClaimHolder(observedHolderBytes);
|
|
162
|
+
if (holder !== null && !isWorkflowClaimExpired({
|
|
163
|
+
expiresAt: holder.expiresAt,
|
|
164
|
+
now,
|
|
165
|
+
renewIntervalMs: this.#claimRenewIntervalMs
|
|
166
|
+
}))
|
|
167
|
+
return {
|
|
168
|
+
status: "not-expired",
|
|
169
|
+
workflowId,
|
|
170
|
+
heldBy: holder.engineId,
|
|
171
|
+
expiresAt: holder.expiresAt
|
|
172
|
+
};
|
|
173
|
+
const fragment = buildWorkflowClaimTakeoverTransition({
|
|
174
|
+
workflowId,
|
|
175
|
+
engineId: this.#engineId,
|
|
176
|
+
now,
|
|
177
|
+
claimTtlMs: this.#claimTtlMs,
|
|
178
|
+
observedHolderBytes,
|
|
179
|
+
observedEpochBytes
|
|
180
|
+
});
|
|
181
|
+
if (!await storageConditionalBatch(this.#claimStorage, fragment.conditions, fragment.operations)) {
|
|
182
|
+
const heldBy = await this.#resolveHeldBy(workflowId);
|
|
183
|
+
return { status: "lost-race", workflowId, heldBy };
|
|
184
|
+
}
|
|
185
|
+
const epoch = mintNextEpoch(observedEpochBytes), epochBytes = extractPutOperationValue(fragment.operations, KEYS.workflowOwnerEpoch(workflowId)), holderBytes = extractPutOperationValue(fragment.operations, KEYS.workflowOwnerHolder(workflowId));
|
|
186
|
+
this.#claims.set(workflowId, { epoch, claimedAt: now, epochBytes, holderBytes });
|
|
187
|
+
this.#takeoverCooldown.clear(workflowId);
|
|
188
|
+
return { status: "acquired", workflowId, epoch };
|
|
189
|
+
}
|
|
190
|
+
async releaseAll() {
|
|
191
|
+
const workflowIds = [...this.#claims.keys()];
|
|
192
|
+
await Promise.all(workflowIds.map(async (workflowId) => {
|
|
193
|
+
try {
|
|
194
|
+
await this.release(workflowId);
|
|
195
|
+
} catch {}
|
|
196
|
+
}));
|
|
197
|
+
}
|
|
198
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Interval-mode driving for `workflow-claim-renewal-task.ts`'s claim-renewal
|
|
3
|
+
* task. Split out purely to keep that module under the repository's
|
|
4
|
+
* implementation-file-size ceiling as the number of independent interval-mode
|
|
5
|
+
* sub-passes has grown (renewal, reclaim, signal poll, update poll) — see
|
|
6
|
+
* that module's own doc for the "each sub-pass gets its own single-flight
|
|
7
|
+
* slot" design this file implements. `workflow-claim-renewal-task.ts` is the
|
|
8
|
+
* only expected caller; `runOnce()`/the manual combined pass stays there,
|
|
9
|
+
* since it shares none of this file's interval-scheduling state.
|
|
10
|
+
*
|
|
11
|
+
* @module core/engine/workflow-claim-renewal-interval
|
|
12
|
+
*/
|
|
13
|
+
import type { OwnerSideSignalPollTarget } from './owner-side-signal-poll.ts';
|
|
14
|
+
import type { OwnerSideUpdatePollTarget } from './owner-side-update-poll.ts';
|
|
15
|
+
import { type WorkflowClaimReclaimTarget, type WorkflowClaimRenewalTarget } from './workflow-claim-renewal-subpasses.ts';
|
|
16
|
+
import type { WorkflowClaimRenewalIntervalScheduler, WorkflowClaimRenewalPassResult } from './workflow-claim-renewal-task.ts';
|
|
17
|
+
/** Options for {@link createWorkflowClaimRenewalIntervalDriver}. Mirrors the relevant subset of `WorkflowClaimRenewalTaskOptions`. */
|
|
18
|
+
export type WorkflowClaimRenewalIntervalDriverOptions = {
|
|
19
|
+
target: WorkflowClaimRenewalTarget;
|
|
20
|
+
getNow: () => number;
|
|
21
|
+
intervalMs: number;
|
|
22
|
+
scheduler: WorkflowClaimRenewalIntervalScheduler;
|
|
23
|
+
reclaimTarget?: WorkflowClaimReclaimTarget;
|
|
24
|
+
signalPollTarget?: OwnerSideSignalPollTarget;
|
|
25
|
+
updatePollTarget?: OwnerSideUpdatePollTarget;
|
|
26
|
+
onPassComplete?: (result: WorkflowClaimRenewalPassResult) => void;
|
|
27
|
+
};
|
|
28
|
+
/** A running interval-mode driver. */
|
|
29
|
+
export type WorkflowClaimRenewalIntervalDriver = {
|
|
30
|
+
/** Start interval-driven passes. Idempotent — a second call while already started does not create a second interval. */
|
|
31
|
+
start(): void;
|
|
32
|
+
/** Stop interval-driven passes. Idempotent. Only clears the interval; passes already in flight still run to completion. */
|
|
33
|
+
stop(): void;
|
|
34
|
+
};
|
|
35
|
+
/**
|
|
36
|
+
* Real-timer scheduler used when no `scheduler` is injected. Keeps each real
|
|
37
|
+
* `Timer` in a private map, keyed by an opaque token, so the public
|
|
38
|
+
* {@link WorkflowClaimRenewalIntervalScheduler} interface never needs to name
|
|
39
|
+
* the concrete timer type — `clearInterval` looks the real timer back up by
|
|
40
|
+
* token rather than requiring a narrowed `unknown` parameter.
|
|
41
|
+
*/
|
|
42
|
+
export declare function defaultWorkflowClaimRenewalScheduler(): WorkflowClaimRenewalIntervalScheduler;
|
|
43
|
+
/**
|
|
44
|
+
* Create the interval-mode driver: four independent single-flight sub-passes
|
|
45
|
+
* (renewal, reclaim, signal poll, update poll), each on its OWN cadence slot
|
|
46
|
+
* so a slow or hung sub-pass never delays a different one — see
|
|
47
|
+
* `workflow-claim-renewal-task.ts`'s module doc for the full rationale
|
|
48
|
+
* (WFT-79 Finding 2 and its extensions).
|
|
49
|
+
*/
|
|
50
|
+
export declare function createWorkflowClaimRenewalIntervalDriver(options: WorkflowClaimRenewalIntervalDriverOptions): WorkflowClaimRenewalIntervalDriver;
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
import {
|
|
2
|
+
runReclaimPass,
|
|
3
|
+
runRenewalSubPass,
|
|
4
|
+
runSignalPollSubPass,
|
|
5
|
+
runUpdatePollSubPass
|
|
6
|
+
} from "./workflow-claim-renewal-subpasses.js";
|
|
7
|
+
export function defaultWorkflowClaimRenewalScheduler() {
|
|
8
|
+
const timers = new Map;
|
|
9
|
+
let nextToken = 1;
|
|
10
|
+
return {
|
|
11
|
+
setInterval(callback, intervalMs) {
|
|
12
|
+
const token = nextToken;
|
|
13
|
+
nextToken += 1;
|
|
14
|
+
const handle = setInterval(callback, intervalMs);
|
|
15
|
+
handle.unref?.();
|
|
16
|
+
timers.set(token, handle);
|
|
17
|
+
return token;
|
|
18
|
+
},
|
|
19
|
+
clearInterval(handle) {
|
|
20
|
+
if (typeof handle !== "number")
|
|
21
|
+
return;
|
|
22
|
+
const timer = timers.get(handle);
|
|
23
|
+
if (timer === void 0)
|
|
24
|
+
return;
|
|
25
|
+
clearInterval(timer);
|
|
26
|
+
timers.delete(handle);
|
|
27
|
+
}
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
export function createWorkflowClaimRenewalIntervalDriver(options) {
|
|
31
|
+
const {
|
|
32
|
+
target,
|
|
33
|
+
getNow,
|
|
34
|
+
intervalMs,
|
|
35
|
+
scheduler,
|
|
36
|
+
reclaimTarget,
|
|
37
|
+
signalPollTarget,
|
|
38
|
+
updatePollTarget,
|
|
39
|
+
onPassComplete
|
|
40
|
+
} = options;
|
|
41
|
+
let intervalHandle = null, inFlightRenewal = null, inFlightReclaim = null, inFlightSignalPoll = null, inFlightUpdatePoll = null;
|
|
42
|
+
function emitPassComplete(result) {
|
|
43
|
+
try {
|
|
44
|
+
onPassComplete?.(result);
|
|
45
|
+
} catch (error) {
|
|
46
|
+
console.error("weft: workflow claim renewal onPassComplete sink threw", error);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
async function runRenewalTickPass() {
|
|
50
|
+
const startedAt = getNow(), workflowIds = [...target.listHeldWorkflowIds()], renewal = await runRenewalSubPass(target, workflowIds), finishedAt = getNow(), result = { startedAt, finishedAt, ...renewal };
|
|
51
|
+
emitPassComplete(result);
|
|
52
|
+
return result;
|
|
53
|
+
}
|
|
54
|
+
async function runReclaimTickPass() {
|
|
55
|
+
const startedAt = getNow(), reclaim = await runReclaimPass(reclaimTarget), finishedAt = getNow(), result = {
|
|
56
|
+
startedAt,
|
|
57
|
+
finishedAt,
|
|
58
|
+
outcomes: [],
|
|
59
|
+
renewedCount: 0,
|
|
60
|
+
failedCount: 0,
|
|
61
|
+
reclaim
|
|
62
|
+
};
|
|
63
|
+
emitPassComplete(result);
|
|
64
|
+
return result;
|
|
65
|
+
}
|
|
66
|
+
async function runSignalPollTickPass() {
|
|
67
|
+
const startedAt = getNow(), signalPoll = await runSignalPollSubPass(signalPollTarget, getNow), finishedAt = getNow(), result = {
|
|
68
|
+
startedAt,
|
|
69
|
+
finishedAt,
|
|
70
|
+
outcomes: [],
|
|
71
|
+
renewedCount: 0,
|
|
72
|
+
failedCount: 0,
|
|
73
|
+
signalPoll
|
|
74
|
+
};
|
|
75
|
+
emitPassComplete(result);
|
|
76
|
+
return result;
|
|
77
|
+
}
|
|
78
|
+
async function runUpdatePollTickPass() {
|
|
79
|
+
const startedAt = getNow(), updatePoll = await runUpdatePollSubPass(updatePollTarget, getNow), finishedAt = getNow(), result = {
|
|
80
|
+
startedAt,
|
|
81
|
+
finishedAt,
|
|
82
|
+
outcomes: [],
|
|
83
|
+
renewedCount: 0,
|
|
84
|
+
failedCount: 0,
|
|
85
|
+
updatePoll
|
|
86
|
+
};
|
|
87
|
+
emitPassComplete(result);
|
|
88
|
+
return result;
|
|
89
|
+
}
|
|
90
|
+
function tick() {
|
|
91
|
+
if (inFlightRenewal === null) {
|
|
92
|
+
const pass = runRenewalTickPass();
|
|
93
|
+
inFlightRenewal = pass;
|
|
94
|
+
pass.catch(() => {}).finally(() => {
|
|
95
|
+
inFlightRenewal = null;
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
if (inFlightReclaim === null && reclaimTarget !== void 0) {
|
|
99
|
+
const pass = runReclaimTickPass();
|
|
100
|
+
inFlightReclaim = pass;
|
|
101
|
+
pass.catch(() => {}).finally(() => {
|
|
102
|
+
inFlightReclaim = null;
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
if (inFlightSignalPoll === null && signalPollTarget !== void 0) {
|
|
106
|
+
const pass = runSignalPollTickPass();
|
|
107
|
+
inFlightSignalPoll = pass;
|
|
108
|
+
pass.catch(() => {}).finally(() => {
|
|
109
|
+
inFlightSignalPoll = null;
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
if (inFlightUpdatePoll === null && updatePollTarget !== void 0) {
|
|
113
|
+
const pass = runUpdatePollTickPass();
|
|
114
|
+
inFlightUpdatePoll = pass;
|
|
115
|
+
pass.catch(() => {}).finally(() => {
|
|
116
|
+
inFlightUpdatePoll = null;
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
function start() {
|
|
121
|
+
if (intervalHandle !== null)
|
|
122
|
+
return;
|
|
123
|
+
intervalHandle = scheduler.setInterval(tick, intervalMs);
|
|
124
|
+
}
|
|
125
|
+
function stop() {
|
|
126
|
+
if (intervalHandle === null)
|
|
127
|
+
return;
|
|
128
|
+
scheduler.clearInterval(intervalHandle);
|
|
129
|
+
intervalHandle = null;
|
|
130
|
+
}
|
|
131
|
+
return { start, stop };
|
|
132
|
+
}
|