@lostgradient/weft 0.19.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/cli/generated/operation-catalog.snapshot.json +1334 -0
- package/dist/cli/generated/operation-client.generated.d.ts +9 -2
- package/dist/cli/generated/operation-client.generated.js +2 -0
- package/dist/core/engine/activity-reconciliation.d.ts +2 -2
- package/dist/core/engine/activity-reconciliation.js +12 -8
- package/dist/core/engine/async-activity-completion.js +12 -4
- package/dist/core/engine/async-activity-records.d.ts +11 -1
- package/dist/core/engine/async-activity-records.js +4 -3
- package/dist/core/engine/attributes-tags.d.ts +11 -1
- package/dist/core/engine/attributes-tags.js +4 -3
- package/dist/core/engine/bulk-operations-purge.js +5 -2
- package/dist/core/engine/bulk-operations.js +17 -5
- package/dist/core/engine/callback-creators-bundles.js +1 -1
- package/dist/core/engine/callback-creators-core.js +4 -2
- package/dist/core/engine/checkpoint-io.js +1 -1
- package/dist/core/engine/child-workflow.js +2 -1
- package/dist/core/engine/completed-review-storage.js +1 -1
- package/dist/core/engine/condition-waiters.d.ts +45 -3
- package/dist/core/engine/condition-waiters.js +12 -3
- package/dist/core/engine/disposal.js +6 -0
- package/dist/core/engine/engine-internal-types.d.ts +12 -4
- package/dist/core/engine/fenced-write.d.ts +46 -15
- package/dist/core/engine/fenced-write.js +39 -6
- package/dist/core/engine/finalizer-state.d.ts +3 -1
- package/dist/core/engine/handle-result.d.ts +49 -1
- package/dist/core/engine/handle-result.js +95 -20
- package/dist/core/engine/index.d.ts +13 -5
- package/dist/core/engine/index.js +155 -5
- package/dist/core/engine/inline-launch-queue.js +9 -0
- package/dist/core/engine/internals.d.ts +73 -0
- package/dist/core/engine/invoke-update-handler.d.ts +19 -0
- package/dist/core/engine/invoke-update-handler.js +7 -0
- package/dist/core/engine/lease-deposition.d.ts +132 -0
- package/dist/core/engine/lease-deposition.js +31 -0
- package/dist/core/engine/lease-errors.d.ts +80 -9
- package/dist/core/engine/lease-errors.js +28 -2
- package/dist/core/engine/lifecycle/resume.js +3 -1
- package/dist/core/engine/lifecycle/standalone-claim-acquire.d.ts +33 -0
- package/dist/core/engine/lifecycle/standalone-claim-acquire.js +25 -0
- package/dist/core/engine/lifecycle/start-commit.js +20 -6
- package/dist/core/engine/lifecycle/transition.d.ts +15 -1
- package/dist/core/engine/lifecycle/transition.js +6 -3
- package/dist/core/engine/operations-activity.js +1 -1
- package/dist/core/engine/operations-time.js +21 -7
- package/dist/core/engine/owner-side-signal-poll.d.ts +132 -0
- package/dist/core/engine/owner-side-signal-poll.js +22 -0
- package/dist/core/engine/owner-side-update-poll.d.ts +116 -0
- package/dist/core/engine/owner-side-update-poll.js +22 -0
- package/dist/core/engine/ownership-bootstrap.d.ts +280 -0
- package/dist/core/engine/ownership-bootstrap.js +89 -0
- package/dist/core/engine/ownership-mode-marker.d.ts +82 -0
- package/dist/core/engine/ownership-mode-marker.js +52 -0
- package/dist/core/engine/ownership-options.d.ts +22 -6
- package/dist/core/engine/ownership-options.js +42 -12
- package/dist/core/engine/pending-updates.js +35 -24
- package/dist/core/engine/queries.d.ts +70 -0
- package/dist/core/engine/queries.js +58 -4
- package/dist/core/engine/schedules.js +1 -1
- package/dist/core/engine/signals.d.ts +11 -1
- package/dist/core/engine/signals.js +11 -7
- package/dist/core/engine/sleep-timer-acknowledgements.d.ts +68 -0
- package/dist/core/engine/sleep-timer-acknowledgements.js +19 -0
- package/dist/core/engine/storage-io.d.ts +58 -10
- package/dist/core/engine/storage-io.js +27 -9
- package/dist/core/engine/termination/cleanup.d.ts +5 -4
- package/dist/core/engine/termination/complete.js +23 -4
- package/dist/core/engine/termination/finalizer-claim.js +4 -4
- package/dist/core/engine/termination/state-commit-callbacks.d.ts +15 -0
- package/dist/core/engine/termination/state-commit-callbacks.js +0 -0
- package/dist/core/engine/termination/suspend.js +1 -1
- package/dist/core/engine/update-validation.d.ts +28 -0
- package/dist/core/engine/update-validation.js +37 -0
- package/dist/core/engine/updates.d.ts +17 -16
- package/dist/core/engine/updates.js +24 -47
- package/dist/core/engine/wake-ownership-check.d.ts +86 -0
- package/dist/core/engine/wake-ownership-check.js +19 -0
- package/dist/core/engine/wake-ownership-guard.d.ts +46 -0
- package/dist/core/engine/wake-ownership-guard.js +23 -0
- package/dist/core/engine/workflow-claim-codec.d.ts +64 -0
- package/dist/core/engine/workflow-claim-codec.js +44 -0
- package/dist/core/engine/workflow-claim-cooldown.d.ts +60 -0
- package/dist/core/engine/workflow-claim-cooldown.js +17 -0
- package/dist/core/engine/workflow-claim-fold.d.ts +82 -0
- package/dist/core/engine/workflow-claim-fold.js +53 -0
- package/dist/core/engine/workflow-claim-metrics.d.ts +131 -0
- package/dist/core/engine/workflow-claim-metrics.js +62 -0
- package/dist/core/engine/workflow-claim-reclaim-scan.d.ts +96 -0
- package/dist/core/engine/workflow-claim-reclaim-scan.js +66 -0
- package/dist/core/engine/workflow-claim-reclaim-target.d.ts +107 -0
- package/dist/core/engine/workflow-claim-reclaim-target.js +155 -0
- package/dist/core/engine/workflow-claim-registry.d.ts +227 -0
- package/dist/core/engine/workflow-claim-registry.js +198 -0
- package/dist/core/engine/workflow-claim-renewal-interval.d.ts +50 -0
- package/dist/core/engine/workflow-claim-renewal-interval.js +132 -0
- package/dist/core/engine/workflow-claim-renewal-subpasses.d.ts +265 -0
- package/dist/core/engine/workflow-claim-renewal-subpasses.js +104 -0
- package/dist/core/engine/workflow-claim-renewal-task.d.ts +159 -0
- package/dist/core/engine/workflow-claim-renewal-task.js +65 -0
- package/dist/core/engine/workflow-claim-transitions.d.ts +186 -0
- package/dist/core/engine/workflow-claim-transitions.js +120 -0
- package/dist/core/inline-execution-strategy.d.ts +20 -0
- package/dist/core/inline-execution-strategy.js +15 -4
- package/dist/core/scheduler/scheduler-class.js +3 -3
- package/dist/core/scheduler/timer-sources.d.ts +13 -0
- package/dist/core/scheduler/timer-sources.js +1 -1
- package/dist/core/types/options.d.ts +17 -1
- package/dist/core/weft-error.d.ts +1 -1
- package/dist/core/weft-error.js +2 -1
- package/dist/http.js +2 -2
- package/dist/index.d.ts +2 -1
- package/dist/index.js +8 -0
- package/dist/indexeddb.js +1 -1
- package/dist/json-schema.js +3 -3
- package/dist/observability/index.js +2 -2
- package/dist/server/operations/get-task-detail-schema.d.ts +341 -0
- package/dist/server/operations/get-task-detail-schema.js +118 -0
- package/dist/server/operations/get-task-detail.d.ts +346 -0
- package/dist/server/operations/get-task-detail.js +215 -0
- package/dist/server/operations/static-registrations.js +3 -0
- package/dist/storage/auto.js +1 -1
- package/dist/storage/bun-sql.js +113 -0
- package/dist/storage/compressed-storage.js +1 -1
- package/dist/storage/index.d.ts +38 -35
- package/dist/storage/interface.d.ts +87 -125
- package/dist/storage/interface.js +1 -1
- package/dist/storage/key-encoding.d.ts +51 -0
- package/dist/storage/key-encoding.js +13 -0
- package/dist/storage/key-prefixes.d.ts +1 -1
- package/dist/storage/key-prefixes.js +3 -0
- package/dist/storage/lmdb.js +1 -1
- package/dist/storage/memory.js +1 -1
- package/dist/storage/neon.js +2 -2
- package/dist/storage/node-sqlite.js +113 -0
- package/dist/storage/ownership-keys.d.ts +55 -0
- package/dist/storage/ownership-keys.js +6 -0
- package/dist/storage/postgres.js +2 -2
- package/dist/storage/resolve.js +1 -1
- package/dist/storage/scoped-storage.js +1 -1
- package/dist/storage/testing.js +1 -1
- package/dist/storage/turso.js +2 -2
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/web-extension.js +1 -1
- package/dist/worker/protocol.js +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Owner-side coordinated-update polling (WFT-79), mirroring
|
|
3
|
+
* `owner-side-signal-poll.ts`'s mechanism and rationale.
|
|
4
|
+
*
|
|
5
|
+
* **The problem.** `updates.ts`'s `deliverCoordinatedUpdateToWaiterIfAvailable`
|
|
6
|
+
* discards a delivery attempt when this engine no longer holds the claim
|
|
7
|
+
* generation it parked the waiter under (`confirmWakeOwnership`), leaving the
|
|
8
|
+
* durable coordinated-update request in place for the true owner to deliver.
|
|
9
|
+
* But nothing DRIVES the true owner to look: `schedulePendingInlineUpdateDrain`
|
|
10
|
+
* only fires a `setTimeout(0)` drain on the engine that RECEIVED the
|
|
11
|
+
* `engine.update()`/`submitCoordinatedUpdate()` call, and the owning engine's
|
|
12
|
+
* maintenance task polls the durable signal buffer and child-result state,
|
|
13
|
+
* never pending coordinated updates. If the true owner is parked on
|
|
14
|
+
* `ctx.waitForUpdate()`, or simply has no reason to re-check because nothing
|
|
15
|
+
* else is advancing it, the caller's request sits durable and undelivered
|
|
16
|
+
* until something unrelated happens to drive that workflow.
|
|
17
|
+
*
|
|
18
|
+
* **The fix.** The owning engine re-checks its own held workflows' pending
|
|
19
|
+
* coordinated-update queues on the same cadence it renews their claims, and
|
|
20
|
+
* drains any workflow with at least one buffered request — by any engine.
|
|
21
|
+
* This bounds cross-engine update delivery latency at one renewal interval,
|
|
22
|
+
* the same bound ADR 0002 already gives cross-engine signal delivery.
|
|
23
|
+
*
|
|
24
|
+
* **Scope and independence.** This module is the poll pass alone: given this
|
|
25
|
+
* engine's held workflow ids, probe each for pending coordinated updates, and
|
|
26
|
+
* drain the ones that have any. It does not read `pending-updates.ts` or
|
|
27
|
+
* `updates.ts` directly — the dependencies below are small structural types
|
|
28
|
+
* this module owns, so it stays independent of those modules' concrete
|
|
29
|
+
* shapes. Draining is expected to be satisfied by composing
|
|
30
|
+
* `processPendingUpdatesForHandlers` (drains inline `ctx.onUpdate()`
|
|
31
|
+
* handlers) and `deliverCoordinatedUpdateToWaiterIfAvailable`-per-pending-
|
|
32
|
+
* update (drains `ctx.waitForUpdate()` waiters) — both already idempotent and
|
|
33
|
+
* fenced on this engine's own claim generation, so a poll racing a
|
|
34
|
+
* concurrent delivery (the normal `setTimeout(0)` drain, or another poll
|
|
35
|
+
* tick) is always safe to re-run. This module is also NOT a recurring task
|
|
36
|
+
* by itself (compare `workflow-claim-renewal-task.ts`) — it is production-
|
|
37
|
+
* wired by `ownership-bootstrap.ts`, which composes `EngineInternals` into a
|
|
38
|
+
* real {@link OwnerSideUpdatePollTarget}, and by
|
|
39
|
+
* `workflow-claim-renewal-task.ts`, which calls {@link runOwnerSideUpdatePoll}
|
|
40
|
+
* against that target from the same lifecycle cadence that drives claim
|
|
41
|
+
* renewal and signal polling.
|
|
42
|
+
*
|
|
43
|
+
* @module core/engine/owner-side-update-poll
|
|
44
|
+
*/
|
|
45
|
+
/**
|
|
46
|
+
* The minimal structural shape this poll needs. Satisfied in production by
|
|
47
|
+
* `ownership-bootstrap.ts`'s adapter, composed from `EngineInternals` —
|
|
48
|
+
* without this module importing it.
|
|
49
|
+
*/
|
|
50
|
+
export type OwnerSideUpdatePollTarget = {
|
|
51
|
+
/**
|
|
52
|
+
* Every workflow id this engine currently holds a live claim for, active or
|
|
53
|
+
* parked. Read fresh at the start of every pass — implementations may
|
|
54
|
+
* return a live or a defensive-copy array; this poll never mutates it and
|
|
55
|
+
* takes its own snapshot before iterating.
|
|
56
|
+
*
|
|
57
|
+
* Expected to be satisfied by `WorkflowClaimRegistry.listHeldWorkflowIds`,
|
|
58
|
+
* the same source `workflow-claim-renewal-task.ts`'s own renewal sub-pass
|
|
59
|
+
* uses.
|
|
60
|
+
*/
|
|
61
|
+
listHeldWorkflowIds(): readonly string[];
|
|
62
|
+
/**
|
|
63
|
+
* Whether `workflowId` currently has at least one durably-buffered
|
|
64
|
+
* coordinated-update request, without consuming any of them.
|
|
65
|
+
*
|
|
66
|
+
* Expected to be satisfied by
|
|
67
|
+
* `internals.updateCoordinator.getPendingUpdates(workflowId).length > 0`.
|
|
68
|
+
*/
|
|
69
|
+
hasPendingUpdates(workflowId: string): Promise<boolean>;
|
|
70
|
+
/**
|
|
71
|
+
* Drain every pending coordinated update currently buffered for
|
|
72
|
+
* `workflowId`: deliver to a registered `ctx.onUpdate()` handler, or to a
|
|
73
|
+
* parked `ctx.waitForUpdate()` waiter, whichever applies. Idempotent and
|
|
74
|
+
* self-fenced on this engine's own current claim generation — safe to call
|
|
75
|
+
* even when a concurrent drain (the normal post-request `setTimeout(0)`
|
|
76
|
+
* trigger, or another poll tick) is racing it.
|
|
77
|
+
*/
|
|
78
|
+
drainPendingUpdates(workflowId: string): Promise<void>;
|
|
79
|
+
};
|
|
80
|
+
/** One held workflow's outcome within a single update-poll pass. */
|
|
81
|
+
export type OwnerSideUpdatePollOutcome = {
|
|
82
|
+
workflowId: string;
|
|
83
|
+
status: 'drained';
|
|
84
|
+
} | {
|
|
85
|
+
workflowId: string;
|
|
86
|
+
status: 'no-pending-updates';
|
|
87
|
+
} | {
|
|
88
|
+
workflowId: string;
|
|
89
|
+
status: 'drain-failed';
|
|
90
|
+
error: unknown;
|
|
91
|
+
};
|
|
92
|
+
/** The result of one full owner-side update poll pass ({@link runOwnerSideUpdatePoll}). */
|
|
93
|
+
export type OwnerSideUpdatePollResult = {
|
|
94
|
+
/** `getNow()` read at the start of the pass, before any probe. */
|
|
95
|
+
startedAt: number;
|
|
96
|
+
/** `getNow()` read after every probe/drain has settled. */
|
|
97
|
+
finishedAt: number;
|
|
98
|
+
/** One entry per held workflow id the pass attempted, in iteration order. */
|
|
99
|
+
outcomes: OwnerSideUpdatePollOutcome[];
|
|
100
|
+
/** `outcomes.filter(o => o.status === 'drained').length`, precomputed for observability consumers. */
|
|
101
|
+
drainedCount: number;
|
|
102
|
+
};
|
|
103
|
+
/** Options for {@link runOwnerSideUpdatePoll}. */
|
|
104
|
+
export type OwnerSideUpdatePollOptions = {
|
|
105
|
+
target: OwnerSideUpdatePollTarget;
|
|
106
|
+
/** Wall-clock source (ms), injected so tests never depend on real time. */
|
|
107
|
+
getNow: () => number;
|
|
108
|
+
};
|
|
109
|
+
/**
|
|
110
|
+
* Run exactly one owner-side update poll pass: snapshot this engine's
|
|
111
|
+
* currently-held workflow ids, probe each for pending coordinated updates,
|
|
112
|
+
* and drain the ones that have any. A drain failure for one workflow is
|
|
113
|
+
* captured as a `'drain-failed'` outcome and never stops the remaining held
|
|
114
|
+
* workflows in the same pass from being probed and, where pending, drained.
|
|
115
|
+
*/
|
|
116
|
+
export declare function runOwnerSideUpdatePoll(options: OwnerSideUpdatePollOptions): Promise<OwnerSideUpdatePollResult>;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
export async function runOwnerSideUpdatePoll(options) {
|
|
2
|
+
const { target, getNow } = options, startedAt = getNow(), heldWorkflowIds = target.listHeldWorkflowIds(), outcomes = [];
|
|
3
|
+
for (const workflowId of heldWorkflowIds) {
|
|
4
|
+
if (!await target.hasPendingUpdates(workflowId)) {
|
|
5
|
+
outcomes.push({ workflowId, status: "no-pending-updates" });
|
|
6
|
+
continue;
|
|
7
|
+
}
|
|
8
|
+
try {
|
|
9
|
+
await target.drainPendingUpdates(workflowId);
|
|
10
|
+
outcomes.push({ workflowId, status: "drained" });
|
|
11
|
+
} catch (error) {
|
|
12
|
+
outcomes.push({ workflowId, status: "drain-failed", error });
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
const finishedAt = getNow();
|
|
16
|
+
return {
|
|
17
|
+
startedAt,
|
|
18
|
+
finishedAt,
|
|
19
|
+
outcomes,
|
|
20
|
+
drainedCount: outcomes.filter((outcome) => outcome.status === "drained").length
|
|
21
|
+
};
|
|
22
|
+
}
|
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Engine-level bootstrap for `ownership: 'workflow-lease'`
|
|
3
|
+
* ([ADR 0002](../../../documentation/contributing/architecture-decisions/0002-multiengine-per-workflow-ownership.md)):
|
|
4
|
+
* runs the two construction-time capability gates ({@link bootstrapOwnershipGates})
|
|
5
|
+
* and, once they pass, constructs this engine's {@link WorkflowClaimRegistry}
|
|
6
|
+
* and its claim-renewal task. Standalone from `Engine`/`EngineInternals` so the
|
|
7
|
+
* idempotency, disposal-race, and background-task-mode wiring in
|
|
8
|
+
* `src/core/engine/index.ts` stay thin call sites over one tested unit.
|
|
9
|
+
*
|
|
10
|
+
* **Both fencing modes run the gates.** This module builds the
|
|
11
|
+
* `workflow-lease` machinery, but `Engine`'s ownership bootstrap also runs
|
|
12
|
+
* Gate 1 and Gate 2 for `ownership: 'lease'` before it acquires the global
|
|
13
|
+
* lease, so the store-wide mode marker rejects a mismatched pairing in either
|
|
14
|
+
* direction. `'lease'` acquisition, fencing, health, and disposal are
|
|
15
|
+
* otherwise unchanged.
|
|
16
|
+
*
|
|
17
|
+
* **The renewal task's reclaim scan is fully wired here**, via
|
|
18
|
+
* {@link createWorkflowClaimReclaimTarget}: candidate discovery
|
|
19
|
+
* (`workflow-claim-reclaim-scan.ts`, including its ownerless-but-running
|
|
20
|
+
* scan — ADR 0002 § "Reclaiming stranded claims") plus a bounded-retry
|
|
21
|
+
* `registry.takeover`/`registry.acquire` loop, per the ADR's "bounded at 5
|
|
22
|
+
* attempts" row. A reclaimed or freshly-acquired claim's `onReclaimed` drive
|
|
23
|
+
* is retried on every later pass if it throws — see
|
|
24
|
+
* {@link createWorkflowClaimReclaimTarget}'s doc for why release-on-failure
|
|
25
|
+
* is unsafe here.
|
|
26
|
+
*
|
|
27
|
+
* **Owner-side signal polling is composed here from small structural
|
|
28
|
+
* sources** ({@link buildOwnerSideSignalPollTarget}) rather than built
|
|
29
|
+
* against `EngineInternals` directly — this module stays independent of
|
|
30
|
+
* that module's concrete shape, matching `owner-side-signal-poll.ts`'s own
|
|
31
|
+
* decoupling discipline. `src/core/engine/index.ts` is the only expected
|
|
32
|
+
* caller: it already has `EngineInternals` in scope and supplies
|
|
33
|
+
* {@link OwnerSideSignalPollSources} as one-line closures over
|
|
34
|
+
* `getInternals(this)`, then passes the composed target through
|
|
35
|
+
* {@link WorkflowLeaseOwnershipBootstrapOptions.signalPollTarget}.
|
|
36
|
+
*
|
|
37
|
+
* @module core/engine/ownership-bootstrap
|
|
38
|
+
*/
|
|
39
|
+
import type { Storage } from '../../storage/interface.ts';
|
|
40
|
+
import type { OwnerSideSignalPollTarget } from './owner-side-signal-poll.ts';
|
|
41
|
+
import type { OwnerSideUpdatePollTarget } from './owner-side-update-poll.ts';
|
|
42
|
+
import { WorkflowClaimMetricsCollector } from './workflow-claim-metrics.ts';
|
|
43
|
+
import { WorkflowClaimRegistry } from './workflow-claim-registry.ts';
|
|
44
|
+
import type { WorkflowClaimRenewalTarget } from './workflow-claim-renewal-subpasses.ts';
|
|
45
|
+
import { type WorkflowClaimRenewalTask } from './workflow-claim-renewal-task.ts';
|
|
46
|
+
export { createWorkflowClaimReclaimTarget, WORKFLOW_CLAIM_TAKEOVER_MAX_ATTEMPTS, type WorkflowClaimReclaimTargetHandle, } from './workflow-claim-reclaim-target.ts';
|
|
47
|
+
/** Input to {@link bootstrapWorkflowLeaseOwnership}. */
|
|
48
|
+
export type WorkflowLeaseOwnershipBootstrapOptions = {
|
|
49
|
+
storage: Storage;
|
|
50
|
+
/** Engine-clock source (ms), injected so tests can drive it deterministically. */
|
|
51
|
+
getNow: () => number;
|
|
52
|
+
/** Resolved `workflowClaimTtl` (ms). */
|
|
53
|
+
claimTtlMs: number;
|
|
54
|
+
/** Resolved `workflowClaimRenewInterval` (ms). */
|
|
55
|
+
claimRenewIntervalMs: number;
|
|
56
|
+
/**
|
|
57
|
+
* Optional owner-side signal-poll target, built by a caller with access to
|
|
58
|
+
* `EngineInternals`/`inline-parking.ts` — typically
|
|
59
|
+
* {@link buildOwnerSideSignalPollTarget} composed from
|
|
60
|
+
* {@link OwnerSideSignalPollSources}. See the module doc's signal-poll
|
|
61
|
+
* paragraph. Omitted leaves `result.signalPoll` `undefined` on every pass.
|
|
62
|
+
*/
|
|
63
|
+
signalPollTarget?: OwnerSideSignalPollTarget;
|
|
64
|
+
/**
|
|
65
|
+
* Optional owner-side update-poll target (WFT-79), built by a caller with
|
|
66
|
+
* access to `EngineInternals` — typically `index.ts`'s
|
|
67
|
+
* `#buildOwnerSideUpdatePollTarget`. Omitted leaves `result.updatePoll`
|
|
68
|
+
* `undefined` on every pass. See `owner-side-update-poll.ts`'s module doc
|
|
69
|
+
* for why the owning engine must re-check its own held workflows' pending
|
|
70
|
+
* coordinated-update queues rather than relying solely on the
|
|
71
|
+
* `setTimeout(0)` drain the receiving engine fires.
|
|
72
|
+
*/
|
|
73
|
+
updatePollTarget?: OwnerSideUpdatePollTarget;
|
|
74
|
+
/**
|
|
75
|
+
* Invoked after this engine reclaims a stranded claim, to actually drive the
|
|
76
|
+
* workflow. Taking the claim only moves durable ownership keys; without this
|
|
77
|
+
* the reclaimed workflow makes no progress while this engine's renewal keeps
|
|
78
|
+
* its claim alive, shielding it from any engine that would have resumed it.
|
|
79
|
+
*/
|
|
80
|
+
onWorkflowClaimReclaimed?: (workflowId: string) => Promise<void>;
|
|
81
|
+
/**
|
|
82
|
+
* Optional workflow-type eligibility check, forwarded to
|
|
83
|
+
* {@link createWorkflowClaimReclaimTarget}'s `isTypeRegistered` parameter.
|
|
84
|
+
* Omitted skips the check, matching that function's own default. See its
|
|
85
|
+
* doc for why an engine that cannot execute a candidate's registered type
|
|
86
|
+
* must never win its claim in the first place.
|
|
87
|
+
*/
|
|
88
|
+
isWorkflowTypeRegistered?: (workflowType: string) => boolean;
|
|
89
|
+
};
|
|
90
|
+
/**
|
|
91
|
+
* Everything `Engine` needs to hold on `EngineInternals` once bootstrap
|
|
92
|
+
* succeeds. `metrics` is the concrete {@link WorkflowClaimMetricsCollector} —
|
|
93
|
+
* not just the structural `WorkflowClaimMetricsRecorder` interface it also
|
|
94
|
+
* satisfies — so a caller (tests, a later Prometheus-exporter bridge) can read
|
|
95
|
+
* `snapshot()` directly instead of needing a second recorder reference.
|
|
96
|
+
*/
|
|
97
|
+
export type WorkflowLeaseOwnershipBootstrapResult = {
|
|
98
|
+
registry: WorkflowClaimRegistry;
|
|
99
|
+
renewalTask: WorkflowClaimRenewalTask;
|
|
100
|
+
metrics: WorkflowClaimMetricsCollector;
|
|
101
|
+
};
|
|
102
|
+
/**
|
|
103
|
+
* Structural sources {@link buildOwnerSideSignalPollTarget} composes into a
|
|
104
|
+
* real {@link OwnerSideSignalPollTarget} (ADR 0002 § "Signal delivery needs
|
|
105
|
+
* more than a classification"). Defined as small closures rather than
|
|
106
|
+
* accepting `EngineInternals` directly — see this module's doc comment.
|
|
107
|
+
* `src/core/engine/index.ts` is the only expected caller.
|
|
108
|
+
*
|
|
109
|
+
* A parked `waitForSignal()` lands in one of two disjoint in-memory
|
|
110
|
+
* populations, and both need owner-side polling to survive a signal
|
|
111
|
+
* delivered on a non-owning engine:
|
|
112
|
+
*
|
|
113
|
+
* - **Checkpoint-parked** (`listParkedInlineWorkflowIds`/`parkedSignalName`):
|
|
114
|
+
* a top-level `waitForSignal()` with no live `ctx.onQuery`/update handler
|
|
115
|
+
* is evicted from memory (`inline-parking.ts`'s `parkInlineWorkflowAfterCheckpoint`)
|
|
116
|
+
* and tracked only as `EngineInternals.parkedInlineWorkflows` plus the
|
|
117
|
+
* checkpointed operation itself — recovered here via
|
|
118
|
+
* `EngineInternals.pendingTimelineEntries`, which every checkpoint commit
|
|
119
|
+
* sets to the just-persisted operation's timeline summary
|
|
120
|
+
* (`checkpoint-io.ts`) and only clears once that operation is finalized.
|
|
121
|
+
* For a still-parked `wait-signal` entry, `operationLabel` IS the awaited
|
|
122
|
+
* signal name (`state-utilities.ts`'s `getTimelineOperationLabel`).
|
|
123
|
+
* - **Live in-memory waiters** (`listSignalWaiterEntries`): a `ctx.race`/
|
|
124
|
+
* `ctx.all` wait-signal branch, or a top-level `waitForSignal()` on a
|
|
125
|
+
* workflow WITH live query/update handlers, registers into
|
|
126
|
+
* `EngineInternals.signalWaiters`/`signalWaitersByWorkflow` instead
|
|
127
|
+
* (`operations-coordination.ts`, `coordination-branch-executors.ts`) — the
|
|
128
|
+
* generator's turn stays alive, blocked on an unresolved in-process
|
|
129
|
+
* promise, rather than being evicted.
|
|
130
|
+
*/
|
|
131
|
+
export type OwnerSideSignalPollSources = {
|
|
132
|
+
/** `EngineInternals.parkedInlineWorkflows` — this engine's checkpoint-parked (memory-evicted) workflow ids. */
|
|
133
|
+
listParkedInlineWorkflowIds(): Iterable<string>;
|
|
134
|
+
/** Whether `workflowId` is currently checkpoint-parked. */
|
|
135
|
+
isParkedInlineWorkflow(workflowId: string): boolean;
|
|
136
|
+
/**
|
|
137
|
+
* The signal name a checkpoint-parked workflow is currently awaiting —
|
|
138
|
+
* `undefined` when `workflowId` is not parked on a `wait-signal`
|
|
139
|
+
* operation. See this type's doc comment for the `pendingTimelineEntries`
|
|
140
|
+
* source.
|
|
141
|
+
*/
|
|
142
|
+
parkedSignalName(workflowId: string): string | undefined;
|
|
143
|
+
/**
|
|
144
|
+
* Every `[workflowId, waiterKey]` pair from
|
|
145
|
+
* `EngineInternals.signalWaitersByWorkflow`. `waiterKey` is always
|
|
146
|
+
* `` `${workflowId}:${signalName}` ``.
|
|
147
|
+
*/
|
|
148
|
+
listSignalWaiterEntries(): Iterable<readonly [workflowId: string, waiterKey: string]>;
|
|
149
|
+
/** `signals.ts`'s `hasBufferedSignal`. */
|
|
150
|
+
hasBufferedSignal(workflowId: string, signalName: string): Promise<boolean>;
|
|
151
|
+
/** `inline-parking.ts`'s `resumeParkedInlineWorkflow`, bound to this engine. */
|
|
152
|
+
resumeParkedInlineWorkflow(workflowId: string): Promise<void>;
|
|
153
|
+
/**
|
|
154
|
+
* Wake exactly the in-memory signal waiter registered under `waiterKey` —
|
|
155
|
+
* mirrors `signals.ts`'s (unexported) `deliverBufferedSignals` waiter
|
|
156
|
+
* branch: release the waiter via the exported `releaseSignalWaiter`, then
|
|
157
|
+
* invoke it. A no-op when no waiter is currently registered under that
|
|
158
|
+
* exact key (already consumed by something else since discovery).
|
|
159
|
+
*/
|
|
160
|
+
wakeSignalWaiter(workflowId: string, waiterKey: string): void;
|
|
161
|
+
/**
|
|
162
|
+
* Optional pre-wake ownership confirmation for the LIVE in-memory waiter
|
|
163
|
+
* branch only (WFT-79 Finding 1). Mirrors `wake-ownership-guard.ts`'s
|
|
164
|
+
* `confirmWakeOwnership` three-way decision tree: `'discard'` means this
|
|
165
|
+
* engine no longer holds the claim generation it registered the waiter
|
|
166
|
+
* under, so `wakeSignalWaiter` must NOT be called — doing so would let a
|
|
167
|
+
* signal buffered for a `ctx.race()`/`ctx.all()` branch wake a generator
|
|
168
|
+
* this engine has since been deposed from, driving it concurrently with
|
|
169
|
+
* whichever engine now legitimately owns the workflow (the exact
|
|
170
|
+
* duplicate-execution hazard ADR 0002 exists to close). `'proceed'` (or
|
|
171
|
+
* this source being omitted entirely) preserves the pre-guard behavior:
|
|
172
|
+
* every confirmed-buffered live waiter is woken unconditionally.
|
|
173
|
+
*
|
|
174
|
+
* **Not called for the checkpoint-parked branch above.** A checkpoint-parked
|
|
175
|
+
* resume goes through `resumeParkedInlineWorkflow` →
|
|
176
|
+
* `resumeWorkflowFromStorage`'s `acquireStandaloneClaimBeforeResume`, which
|
|
177
|
+
* re-confirms this engine's cached claim generation against the durable
|
|
178
|
+
* holder (via `wakeOwnershipCheck`) before resuming, and throws
|
|
179
|
+
* `WorkflowClaimUnavailableError` rather than replaying against a stale
|
|
180
|
+
* generation when that check fails — guarding it here too would be
|
|
181
|
+
* redundant, not unsafe, but the finding is scoped to the live-waiter
|
|
182
|
+
* release this module makes directly.
|
|
183
|
+
*
|
|
184
|
+
* **Required, deliberately.** This module cannot call `confirmWakeOwnership`
|
|
185
|
+
* directly: that helper takes `EngineInternals`, and this module stays
|
|
186
|
+
* independent of that concrete shape (see this module's doc comment) so it
|
|
187
|
+
* never couples to `src/core/engine/index.ts`'s internals layout. The caller
|
|
188
|
+
* therefore supplies it — `src/core/engine/index.ts` passes
|
|
189
|
+
* `(workflowId) => confirmWakeOwnership(internals, workflowId, 'signal')`.
|
|
190
|
+
*
|
|
191
|
+
* It is NOT optional, because an optional fence is one that silently does
|
|
192
|
+
* nothing when a call site forgets it — which is exactly how the owner-side
|
|
193
|
+
* signal poll shipped unwired in the first place. Making it required turns a
|
|
194
|
+
* missing fence into a compile error rather than a duplicate-execution bug
|
|
195
|
+
* that only a reviewer can catch. A caller that genuinely needs no fence
|
|
196
|
+
* (`ownership: 'none'`/`'lease'`) still passes one; `confirmWakeOwnership`
|
|
197
|
+
* returns `'proceed'` when no claim registry is installed.
|
|
198
|
+
*/
|
|
199
|
+
confirmSignalWakeOwnership(workflowId: string): Promise<'proceed' | 'discard'>;
|
|
200
|
+
};
|
|
201
|
+
/**
|
|
202
|
+
* Compose {@link OwnerSideSignalPollSources} into a real
|
|
203
|
+
* {@link OwnerSideSignalPollTarget}.
|
|
204
|
+
*
|
|
205
|
+
* `wakeWorkflow` re-confirms `hasBufferedSignal` for each live waiter key it
|
|
206
|
+
* owns before firing it, rather than trusting the outer poll's per-entry
|
|
207
|
+
* probe: `runOwnerSideSignalPoll` calls `wakeWorkflow(workflowId)` with no
|
|
208
|
+
* `signalName` (`owner-side-signal-poll.ts`'s fixed contract), and one
|
|
209
|
+
* workflow can have multiple live waiter keys at once (distinct `ctx.race`
|
|
210
|
+
* branches waiting on different signals). Firing all of them because ONE was
|
|
211
|
+
* confirmed buffered would spuriously "win" a branch whose signal never
|
|
212
|
+
* arrived — `coordination-branch-executors.ts`'s race branch resolves
|
|
213
|
+
* unconditionally once its waiter fires, with no re-check.
|
|
214
|
+
*/
|
|
215
|
+
export declare function buildOwnerSideSignalPollTarget(sources: OwnerSideSignalPollSources): OwnerSideSignalPollTarget;
|
|
216
|
+
/**
|
|
217
|
+
* Adapt a {@link WorkflowClaimRegistry} to the renewal task's
|
|
218
|
+
* {@link WorkflowClaimRenewalTarget} contract. `registry.renew()`'s
|
|
219
|
+
* discriminated result never throws on a lost CAS — background scanning must
|
|
220
|
+
* isolate a per-workflow failure and continue, so every `WorkflowClaimRegistry`
|
|
221
|
+
* method returns a result instead of throwing. This adapter is where that gets
|
|
222
|
+
* converted to the renewal task's reject-on-failure contract, without adding a
|
|
223
|
+
* throwing method to the registry itself, so the registry's "none throw"
|
|
224
|
+
* contract stays intact for every other caller.
|
|
225
|
+
*
|
|
226
|
+
* `'renewed'` resolves. `'not-held'` also resolves rather than rejects: it is
|
|
227
|
+
* the benign race against this engine's own `release()` (the registry stops
|
|
228
|
+
* new renewals for a workflow before it reads the bytes `release()` conditions
|
|
229
|
+
* on — see `workflow-claim-registry.ts`'s module doc), not a lost claim, and
|
|
230
|
+
* counting it as a renewal failure would inflate
|
|
231
|
+
* `weft_workflow_claim_renewal_failures_total` on ordinary terminal shutdown.
|
|
232
|
+
* Only `'lost'` rejects. The registry's own `renew()` already emits
|
|
233
|
+
* `WeftWorkflowClaimLostWarning` on a `'lost'` result, so this adapter does not
|
|
234
|
+
* double-emit.
|
|
235
|
+
*
|
|
236
|
+
* Exported (only) so its three-branch mapping can be pinned directly against a
|
|
237
|
+
* canned `WorkflowClaimRenewResult`, without needing a storage-timing race to
|
|
238
|
+
* reach `'not-held'` through a real registry.
|
|
239
|
+
*/
|
|
240
|
+
export declare function createWorkflowClaimRenewalTarget(registry: WorkflowClaimRegistry, metrics?: WorkflowClaimMetricsCollector): WorkflowClaimRenewalTarget;
|
|
241
|
+
/**
|
|
242
|
+
* Run Gate 1 + Gate 2 for `ownership: 'workflow-lease'`, then construct this
|
|
243
|
+
* engine's {@link WorkflowClaimRegistry} and claim-renewal task. Does not
|
|
244
|
+
* start the renewal task's interval — the caller decides interval-mode vs.
|
|
245
|
+
* `runMaintenance()`-driven mode, since only it knows `backgroundTaskMode`.
|
|
246
|
+
* Throws whatever {@link bootstrapOwnershipGates} throws (a bare `Error` for
|
|
247
|
+
* Gate 1, `OwnershipModeMismatchError` for Gate 2) before constructing
|
|
248
|
+
* anything durable-adjacent — a failed gate leaves no registry, no renewal
|
|
249
|
+
* task, and no metrics recorder behind.
|
|
250
|
+
*
|
|
251
|
+
* The renewal task's `onPassComplete` seam bridges every pass into the
|
|
252
|
+
* returned {@link WorkflowClaimMetricsRecorder}: each `'failed'` renewal
|
|
253
|
+
* outcome records one `weft_workflow_claim_renewal_failures_total`, and the
|
|
254
|
+
* registry's own currently-held-id count (read fresh after the pass, since a
|
|
255
|
+
* failed renewal can drop a claim mid-pass) sets `weft_workflow_claims_active`.
|
|
256
|
+
*
|
|
257
|
+
* **`renewalTask.stop()` also quiesces the reclaim target (WFT-79 Finding 2).**
|
|
258
|
+
* `createWorkflowClaimRenewalTask`'s own `stop()` only clears the interval —
|
|
259
|
+
* a reclaim pass already in flight (started by the last tick before `stop()`)
|
|
260
|
+
* still runs to completion, per that module's own doc. Left alone, a
|
|
261
|
+
* disposal that races an in-flight interval-driven pass could let the
|
|
262
|
+
* {@link createWorkflowClaimReclaimTarget reclaim target} land a fresh
|
|
263
|
+
* takeover/acquire CAS, or drive a freshly reclaimed workflow's `onReclaimed`
|
|
264
|
+
* against a host that is mid-teardown, AFTER `Engine`'s disposal path has
|
|
265
|
+
* already snapshotted the registry for `WorkflowClaimRegistry.releaseAll()`.
|
|
266
|
+
* The `renewalTask` this function returns wraps the raw task's `stop` so it
|
|
267
|
+
* synchronously calls {@link WorkflowClaimReclaimTargetHandle.markDisposing}
|
|
268
|
+
* first: since JS has no preemption, that flag flip happens strictly before
|
|
269
|
+
* any other code runs, so every reclaim-target checkpoint after an `await`
|
|
270
|
+
* (including one a suspended continuation resumes into after `stop()` was
|
|
271
|
+
* called) observes it. A checkpoint that finds itself disposing self-releases
|
|
272
|
+
* any claim it just landed rather than driving it — see that module's doc —
|
|
273
|
+
* so the hazard closes without `Engine`'s disposal path (`index.ts`, out of
|
|
274
|
+
* this module's scope) needing to await anything new. This wrapping makes
|
|
275
|
+
* `stop()` permanent: a later `start()` on the returned task still clears
|
|
276
|
+
* `disposing` for renewal-interval purposes, but the reclaim target itself
|
|
277
|
+
* never un-disposes. That is intentional — `index.ts` never restarts a
|
|
278
|
+
* renewal task after detaching it — and is pinned by a test.
|
|
279
|
+
*/
|
|
280
|
+
export declare function bootstrapWorkflowLeaseOwnership(options: WorkflowLeaseOwnershipBootstrapOptions): Promise<WorkflowLeaseOwnershipBootstrapResult>;
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { bootstrapOwnershipGates } from "./ownership-mode-marker.js";
|
|
2
|
+
import { WorkflowClaimMetricsCollector } from "./workflow-claim-metrics.js";
|
|
3
|
+
import { createWorkflowClaimReclaimTarget } from "./workflow-claim-reclaim-target.js";
|
|
4
|
+
import { WorkflowClaimRegistry } from "./workflow-claim-registry.js";
|
|
5
|
+
import {
|
|
6
|
+
createWorkflowClaimRenewalTask
|
|
7
|
+
} from "./workflow-claim-renewal-task.js";
|
|
8
|
+
export {
|
|
9
|
+
createWorkflowClaimReclaimTarget,
|
|
10
|
+
WORKFLOW_CLAIM_TAKEOVER_MAX_ATTEMPTS
|
|
11
|
+
} from "./workflow-claim-reclaim-target.js";
|
|
12
|
+
export function buildOwnerSideSignalPollTarget(sources) {
|
|
13
|
+
return {
|
|
14
|
+
listParkedSignalWaits() {
|
|
15
|
+
const waits = [];
|
|
16
|
+
for (const workflowId of sources.listParkedInlineWorkflowIds()) {
|
|
17
|
+
const signalName = sources.parkedSignalName(workflowId);
|
|
18
|
+
if (signalName !== void 0)
|
|
19
|
+
waits.push({ workflowId, signalName });
|
|
20
|
+
}
|
|
21
|
+
for (const [workflowId, waiterKey] of sources.listSignalWaiterEntries())
|
|
22
|
+
waits.push({ workflowId, signalName: waiterKey.slice(workflowId.length + 1) });
|
|
23
|
+
return waits;
|
|
24
|
+
},
|
|
25
|
+
hasBufferedSignal: (workflowId, signalName) => sources.hasBufferedSignal(workflowId, signalName),
|
|
26
|
+
async wakeWorkflow(workflowId) {
|
|
27
|
+
if (sources.isParkedInlineWorkflow(workflowId)) {
|
|
28
|
+
await sources.resumeParkedInlineWorkflow(workflowId);
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
31
|
+
for (const [waiterWorkflowId, waiterKey] of sources.listSignalWaiterEntries()) {
|
|
32
|
+
if (waiterWorkflowId !== workflowId)
|
|
33
|
+
continue;
|
|
34
|
+
const signalName = waiterKey.slice(workflowId.length + 1);
|
|
35
|
+
if (await sources.hasBufferedSignal(workflowId, signalName)) {
|
|
36
|
+
if (await sources.confirmSignalWakeOwnership(workflowId) === "discard")
|
|
37
|
+
continue;
|
|
38
|
+
sources.wakeSignalWaiter(workflowId, waiterKey);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
export function createWorkflowClaimRenewalTarget(registry, metrics) {
|
|
45
|
+
return {
|
|
46
|
+
listHeldWorkflowIds: () => registry.listHeldWorkflowIds(),
|
|
47
|
+
async renewWorkflowClaim(workflowId) {
|
|
48
|
+
if ((await registry.renew(workflowId)).status === "lost") {
|
|
49
|
+
metrics?.recordClaimAttempt("deposed");
|
|
50
|
+
throw Error(`workflow "${workflowId}" lost its ownership claim during renewal`);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
export async function bootstrapWorkflowLeaseOwnership(options) {
|
|
56
|
+
await bootstrapOwnershipGates({
|
|
57
|
+
storage: options.storage,
|
|
58
|
+
ownershipMode: "workflow-lease",
|
|
59
|
+
getNow: options.getNow
|
|
60
|
+
});
|
|
61
|
+
const registry = new WorkflowClaimRegistry({
|
|
62
|
+
storage: options.storage,
|
|
63
|
+
engineId: crypto.randomUUID(),
|
|
64
|
+
getNow: options.getNow,
|
|
65
|
+
claimTtlMs: options.claimTtlMs,
|
|
66
|
+
claimRenewIntervalMs: options.claimRenewIntervalMs
|
|
67
|
+
}), metrics = new WorkflowClaimMetricsCollector, reclaimTarget = createWorkflowClaimReclaimTarget(registry, options.storage, metrics, options.onWorkflowClaimReclaimed, options.isWorkflowTypeRegistered), rawRenewalTask = createWorkflowClaimRenewalTask({
|
|
68
|
+
target: createWorkflowClaimRenewalTarget(registry, metrics),
|
|
69
|
+
reclaimTarget,
|
|
70
|
+
...options.signalPollTarget === void 0 ? {} : { signalPollTarget: options.signalPollTarget },
|
|
71
|
+
...options.updatePollTarget === void 0 ? {} : { updatePollTarget: options.updatePollTarget },
|
|
72
|
+
getNow: options.getNow,
|
|
73
|
+
intervalMs: options.claimRenewIntervalMs,
|
|
74
|
+
onPassComplete: (result) => {
|
|
75
|
+
for (const outcome of result.outcomes)
|
|
76
|
+
if (outcome.status === "failed")
|
|
77
|
+
metrics.recordClaimRenewalFailure();
|
|
78
|
+
metrics.setActiveClaims(registry.listHeldWorkflowIds().length);
|
|
79
|
+
}
|
|
80
|
+
});
|
|
81
|
+
return { registry, renewalTask: {
|
|
82
|
+
runOnce: () => rawRenewalTask.runOnce(),
|
|
83
|
+
start: () => rawRenewalTask.start(),
|
|
84
|
+
stop: () => {
|
|
85
|
+
reclaimTarget.markDisposing();
|
|
86
|
+
rawRenewalTask.stop();
|
|
87
|
+
}
|
|
88
|
+
}, metrics };
|
|
89
|
+
}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The two construction-time capability gates described in
|
|
3
|
+
* [ADR 0002 § Construction-time capability gates](../../../documentation/contributing/architecture-decisions/0002-multiengine-per-workflow-ownership.md#construction-time-capability-gates):
|
|
4
|
+
*
|
|
5
|
+
* - **Gate 1 — storage capability.** Fails fast when the configured storage
|
|
6
|
+
* backend does not support `conditionalBatch`, naming the *configured*
|
|
7
|
+
* fencing mode in the diagnostic rather than a hardcoded one, so an operator
|
|
8
|
+
* troubleshooting the shipped `ownership: 'lease'` is not sent chasing
|
|
9
|
+
* `workflow-lease`, which is unimplemented until a later stage.
|
|
10
|
+
* - **Gate 2 — ownership-mode marker.** Stamps or verifies the store-wide
|
|
11
|
+
* `ownership-mode-marker` singleton so `ownership: 'lease'` and
|
|
12
|
+
* `ownership: 'workflow-lease'` remain mutually exclusive across engine
|
|
13
|
+
* *processes*, not merely within one process.
|
|
14
|
+
*
|
|
15
|
+
* Both gates fire only for the two fencing modes (`'lease'` and
|
|
16
|
+
* `'workflow-lease'`); `ownership: 'none'` never touches the marker or checks
|
|
17
|
+
* the capability. {@link bootstrapOwnershipGates} runs them in the ADR's
|
|
18
|
+
* required order (Gate 1 then Gate 2) and is the unit later wired in front of
|
|
19
|
+
* claim acquisition, recovery scanning, scheduler polling, and task polling —
|
|
20
|
+
* that wiring is a later stage and is deliberately not done here.
|
|
21
|
+
*
|
|
22
|
+
* @module core/engine/ownership-mode-marker
|
|
23
|
+
*/
|
|
24
|
+
import { type Storage } from '../../storage/interface.ts';
|
|
25
|
+
import { type FencingOwnershipMode } from './workflow-claim-codec.ts';
|
|
26
|
+
/**
|
|
27
|
+
* Mirrors the full `ownership` discriminant (`EngineConstructorOptions.ownership`
|
|
28
|
+
* / `ResolvedOptions.ownershipMode`). Duplicated here as a literal union rather
|
|
29
|
+
* than imported, because neither call site exports a standalone name for it —
|
|
30
|
+
* importing `ResolvedOptions` would pull in the whole engine-internal-types
|
|
31
|
+
* surface for one field's type.
|
|
32
|
+
*/
|
|
33
|
+
export type EngineOwnershipMode = 'none' | 'lease' | 'workflow-lease';
|
|
34
|
+
/**
|
|
35
|
+
* Gate 1 — storage capability. Fires for `ownership: 'lease'` or
|
|
36
|
+
* `ownership: 'workflow-lease'`; reuses the existing, untyped `Error` that
|
|
37
|
+
* {@link requireStorageCapability} already throws. The diagnostic names the
|
|
38
|
+
* mode that is actually configured — hardcoding `'workflow-lease'` into the
|
|
39
|
+
* message would misdirect an operator troubleshooting the shipped global
|
|
40
|
+
* lease toward an unimplemented feature.
|
|
41
|
+
*/
|
|
42
|
+
export declare function assertOwnershipStorageCapability(storage: Storage, configuredMode: FencingOwnershipMode): void;
|
|
43
|
+
/** Input to {@link assertOwnershipModeMarker}. */
|
|
44
|
+
export type AssertOwnershipModeMarkerInput = {
|
|
45
|
+
storage: Storage;
|
|
46
|
+
/** This engine's configured fencing mode. */
|
|
47
|
+
configuredMode: FencingOwnershipMode;
|
|
48
|
+
/** Engine-clock source (ms), injected so tests can control `establishedAt` deterministically. */
|
|
49
|
+
getNow: () => number;
|
|
50
|
+
};
|
|
51
|
+
/**
|
|
52
|
+
* Gate 2 — ownership-mode marker. Fires immediately after Gate 1 passes, for
|
|
53
|
+
* the same trigger (`ownership: 'lease'` or `ownership: 'workflow-lease'`;
|
|
54
|
+
* `ownership: 'none'` never calls this).
|
|
55
|
+
*
|
|
56
|
+
* Reads `KEYS.ownershipModeMarker()`. If absent, `conditionalBatch`-puts
|
|
57
|
+
* `{ mode: configuredMode, establishedAt: now() }` with an expected value of
|
|
58
|
+
* `null` — the first fencing-mode engine against a fresh store establishes
|
|
59
|
+
* the mode every later one must agree with. On a CAS loss (another engine won
|
|
60
|
+
* the race to stamp it), re-reads and compares against that engine's mode
|
|
61
|
+
* instead, since it is now authoritative. If the stored mode — from either
|
|
62
|
+
* the initial read or the post-CAS-loss re-read — differs from this engine's
|
|
63
|
+
* configured mode, throws {@link OwnershipModeMismatchError} before any
|
|
64
|
+
* further construction proceeds.
|
|
65
|
+
*/
|
|
66
|
+
export declare function assertOwnershipModeMarker(input: AssertOwnershipModeMarkerInput): Promise<void>;
|
|
67
|
+
/** Input to {@link bootstrapOwnershipGates}. */
|
|
68
|
+
export type BootstrapOwnershipGatesInput = {
|
|
69
|
+
storage: Storage;
|
|
70
|
+
ownershipMode: EngineOwnershipMode;
|
|
71
|
+
/** Engine-clock source (ms), injected so tests can control `establishedAt` deterministically. */
|
|
72
|
+
getNow: () => number;
|
|
73
|
+
};
|
|
74
|
+
/**
|
|
75
|
+
* Run Gate 1 then Gate 2, in that order, as ADR 0002 requires. No-op for
|
|
76
|
+
* `ownership: 'none'`, which never touches storage-capability enforcement or
|
|
77
|
+
* the mode marker. Both gates must complete successfully before any claim
|
|
78
|
+
* acquisition, recovery scan, scheduler poll, or task poll proceeds — but
|
|
79
|
+
* wiring this into `Engine` construction or any of those call sites is a
|
|
80
|
+
* later stage; this function is the standalone, testable unit only.
|
|
81
|
+
*/
|
|
82
|
+
export declare function bootstrapOwnershipGates(input: BootstrapOwnershipGatesInput): Promise<void>;
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import {
|
|
2
|
+
KEYS,
|
|
3
|
+
requireStorageCapability,
|
|
4
|
+
storageConditionalBatch
|
|
5
|
+
} from "../../storage/interface.js";
|
|
6
|
+
import { OwnershipModeMismatchError } from "./lease-errors.js";
|
|
7
|
+
import {
|
|
8
|
+
decodeOwnershipModeMarker,
|
|
9
|
+
encodeOwnershipModeMarker
|
|
10
|
+
} from "./workflow-claim-codec.js";
|
|
11
|
+
export function assertOwnershipStorageCapability(storage, configuredMode) {
|
|
12
|
+
requireStorageCapability(storage, "conditionalBatch", `ownership: '${configuredMode}'`);
|
|
13
|
+
}
|
|
14
|
+
async function readOwnershipModeMarkerOrThrowIfCorrupt(storage, markerKey) {
|
|
15
|
+
const bytes = await storage.get(markerKey);
|
|
16
|
+
if (bytes === null)
|
|
17
|
+
return null;
|
|
18
|
+
const decoded = decodeOwnershipModeMarker(bytes);
|
|
19
|
+
if (decoded === null)
|
|
20
|
+
throw Error(`The store's ownership-mode-marker ("${markerKey}") exists but does not decode as a valid { mode, establishedAt } record. Treating an undecodable marker as absent would let this engine silently overwrite whatever mode the store's real prior engines agreed on, defeating the marker's purpose of making a mixed fencing-mode deployment detectable. Resolve by operator repair: inspect the stored bytes and, only if certain no other engine relies on them, delete the key so a fresh marker can be established.`);
|
|
21
|
+
return decoded;
|
|
22
|
+
}
|
|
23
|
+
function assertOwnershipModeMatches(configuredMode, storedRecord) {
|
|
24
|
+
if (storedRecord.mode !== configuredMode)
|
|
25
|
+
throw new OwnershipModeMismatchError(configuredMode, storedRecord.mode, storedRecord.establishedAt);
|
|
26
|
+
}
|
|
27
|
+
export async function assertOwnershipModeMarker(input) {
|
|
28
|
+
const { storage, configuredMode, getNow } = input, markerKey = KEYS.ownershipModeMarker(), initial = await readOwnershipModeMarkerOrThrowIfCorrupt(storage, markerKey);
|
|
29
|
+
if (initial !== null) {
|
|
30
|
+
assertOwnershipModeMatches(configuredMode, initial);
|
|
31
|
+
return;
|
|
32
|
+
}
|
|
33
|
+
if (await storageConditionalBatch(storage, [{ key: markerKey, expectedValue: null }], [
|
|
34
|
+
{
|
|
35
|
+
type: "put",
|
|
36
|
+
key: markerKey,
|
|
37
|
+
value: encodeOwnershipModeMarker({ mode: configuredMode, establishedAt: getNow() })
|
|
38
|
+
}
|
|
39
|
+
]))
|
|
40
|
+
return;
|
|
41
|
+
const reread = await readOwnershipModeMarkerOrThrowIfCorrupt(storage, markerKey);
|
|
42
|
+
if (reread === null)
|
|
43
|
+
throw Error(`Lost the compare-and-swap while establishing the store's ownership-mode-marker ("${markerKey}"), and a re-read then found it absent again. This indicates the marker was deleted concurrently during construction rather than raced by another engine's write (which would have left it present). Retry construction once the concurrent deletion has settled.`);
|
|
44
|
+
assertOwnershipModeMatches(configuredMode, reread);
|
|
45
|
+
}
|
|
46
|
+
export async function bootstrapOwnershipGates(input) {
|
|
47
|
+
const { storage, ownershipMode, getNow } = input;
|
|
48
|
+
if (ownershipMode === "none")
|
|
49
|
+
return;
|
|
50
|
+
assertOwnershipStorageCapability(storage, ownershipMode);
|
|
51
|
+
await assertOwnershipModeMarker({ storage, configuredMode: ownershipMode, getNow });
|
|
52
|
+
}
|