@lostgradient/weft 0.19.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/cli/generated/operation-catalog.snapshot.json +1334 -0
- package/dist/cli/generated/operation-client.generated.d.ts +9 -2
- package/dist/cli/generated/operation-client.generated.js +2 -0
- package/dist/core/engine/activity-reconciliation.d.ts +2 -2
- package/dist/core/engine/activity-reconciliation.js +12 -8
- package/dist/core/engine/async-activity-completion.js +12 -4
- package/dist/core/engine/async-activity-records.d.ts +11 -1
- package/dist/core/engine/async-activity-records.js +4 -3
- package/dist/core/engine/attributes-tags.d.ts +11 -1
- package/dist/core/engine/attributes-tags.js +4 -3
- package/dist/core/engine/bulk-operations-purge.js +5 -2
- package/dist/core/engine/bulk-operations.js +17 -5
- package/dist/core/engine/callback-creators-bundles.js +1 -1
- package/dist/core/engine/callback-creators-core.js +4 -2
- package/dist/core/engine/checkpoint-io.js +1 -1
- package/dist/core/engine/child-workflow.js +2 -1
- package/dist/core/engine/completed-review-storage.js +1 -1
- package/dist/core/engine/condition-waiters.d.ts +45 -3
- package/dist/core/engine/condition-waiters.js +12 -3
- package/dist/core/engine/disposal.js +6 -0
- package/dist/core/engine/engine-internal-types.d.ts +12 -4
- package/dist/core/engine/fenced-write.d.ts +46 -15
- package/dist/core/engine/fenced-write.js +39 -6
- package/dist/core/engine/finalizer-state.d.ts +3 -1
- package/dist/core/engine/handle-result.d.ts +49 -1
- package/dist/core/engine/handle-result.js +95 -20
- package/dist/core/engine/index.d.ts +13 -5
- package/dist/core/engine/index.js +155 -5
- package/dist/core/engine/inline-launch-queue.js +9 -0
- package/dist/core/engine/internals.d.ts +73 -0
- package/dist/core/engine/invoke-update-handler.d.ts +19 -0
- package/dist/core/engine/invoke-update-handler.js +7 -0
- package/dist/core/engine/lease-deposition.d.ts +132 -0
- package/dist/core/engine/lease-deposition.js +31 -0
- package/dist/core/engine/lease-errors.d.ts +80 -9
- package/dist/core/engine/lease-errors.js +28 -2
- package/dist/core/engine/lifecycle/resume.js +3 -1
- package/dist/core/engine/lifecycle/standalone-claim-acquire.d.ts +33 -0
- package/dist/core/engine/lifecycle/standalone-claim-acquire.js +25 -0
- package/dist/core/engine/lifecycle/start-commit.js +20 -6
- package/dist/core/engine/lifecycle/transition.d.ts +15 -1
- package/dist/core/engine/lifecycle/transition.js +6 -3
- package/dist/core/engine/operations-activity.js +1 -1
- package/dist/core/engine/operations-time.js +21 -7
- package/dist/core/engine/owner-side-signal-poll.d.ts +132 -0
- package/dist/core/engine/owner-side-signal-poll.js +22 -0
- package/dist/core/engine/owner-side-update-poll.d.ts +116 -0
- package/dist/core/engine/owner-side-update-poll.js +22 -0
- package/dist/core/engine/ownership-bootstrap.d.ts +280 -0
- package/dist/core/engine/ownership-bootstrap.js +89 -0
- package/dist/core/engine/ownership-mode-marker.d.ts +82 -0
- package/dist/core/engine/ownership-mode-marker.js +52 -0
- package/dist/core/engine/ownership-options.d.ts +22 -6
- package/dist/core/engine/ownership-options.js +42 -12
- package/dist/core/engine/pending-updates.js +35 -24
- package/dist/core/engine/queries.d.ts +70 -0
- package/dist/core/engine/queries.js +58 -4
- package/dist/core/engine/schedules.js +1 -1
- package/dist/core/engine/signals.d.ts +11 -1
- package/dist/core/engine/signals.js +11 -7
- package/dist/core/engine/sleep-timer-acknowledgements.d.ts +68 -0
- package/dist/core/engine/sleep-timer-acknowledgements.js +19 -0
- package/dist/core/engine/storage-io.d.ts +58 -10
- package/dist/core/engine/storage-io.js +27 -9
- package/dist/core/engine/termination/cleanup.d.ts +5 -4
- package/dist/core/engine/termination/complete.js +23 -4
- package/dist/core/engine/termination/finalizer-claim.js +4 -4
- package/dist/core/engine/termination/state-commit-callbacks.d.ts +15 -0
- package/dist/core/engine/termination/state-commit-callbacks.js +0 -0
- package/dist/core/engine/termination/suspend.js +1 -1
- package/dist/core/engine/update-validation.d.ts +28 -0
- package/dist/core/engine/update-validation.js +37 -0
- package/dist/core/engine/updates.d.ts +17 -16
- package/dist/core/engine/updates.js +24 -47
- package/dist/core/engine/wake-ownership-check.d.ts +86 -0
- package/dist/core/engine/wake-ownership-check.js +19 -0
- package/dist/core/engine/wake-ownership-guard.d.ts +46 -0
- package/dist/core/engine/wake-ownership-guard.js +23 -0
- package/dist/core/engine/workflow-claim-codec.d.ts +64 -0
- package/dist/core/engine/workflow-claim-codec.js +44 -0
- package/dist/core/engine/workflow-claim-cooldown.d.ts +60 -0
- package/dist/core/engine/workflow-claim-cooldown.js +17 -0
- package/dist/core/engine/workflow-claim-fold.d.ts +82 -0
- package/dist/core/engine/workflow-claim-fold.js +53 -0
- package/dist/core/engine/workflow-claim-metrics.d.ts +131 -0
- package/dist/core/engine/workflow-claim-metrics.js +62 -0
- package/dist/core/engine/workflow-claim-reclaim-scan.d.ts +96 -0
- package/dist/core/engine/workflow-claim-reclaim-scan.js +66 -0
- package/dist/core/engine/workflow-claim-reclaim-target.d.ts +107 -0
- package/dist/core/engine/workflow-claim-reclaim-target.js +155 -0
- package/dist/core/engine/workflow-claim-registry.d.ts +227 -0
- package/dist/core/engine/workflow-claim-registry.js +198 -0
- package/dist/core/engine/workflow-claim-renewal-interval.d.ts +50 -0
- package/dist/core/engine/workflow-claim-renewal-interval.js +132 -0
- package/dist/core/engine/workflow-claim-renewal-subpasses.d.ts +265 -0
- package/dist/core/engine/workflow-claim-renewal-subpasses.js +104 -0
- package/dist/core/engine/workflow-claim-renewal-task.d.ts +159 -0
- package/dist/core/engine/workflow-claim-renewal-task.js +65 -0
- package/dist/core/engine/workflow-claim-transitions.d.ts +186 -0
- package/dist/core/engine/workflow-claim-transitions.js +120 -0
- package/dist/core/inline-execution-strategy.d.ts +20 -0
- package/dist/core/inline-execution-strategy.js +15 -4
- package/dist/core/scheduler/scheduler-class.js +3 -3
- package/dist/core/scheduler/timer-sources.d.ts +13 -0
- package/dist/core/scheduler/timer-sources.js +1 -1
- package/dist/core/types/options.d.ts +17 -1
- package/dist/core/weft-error.d.ts +1 -1
- package/dist/core/weft-error.js +2 -1
- package/dist/http.js +2 -2
- package/dist/index.d.ts +2 -1
- package/dist/index.js +8 -0
- package/dist/indexeddb.js +1 -1
- package/dist/json-schema.js +3 -3
- package/dist/observability/index.js +2 -2
- package/dist/server/operations/get-task-detail-schema.d.ts +341 -0
- package/dist/server/operations/get-task-detail-schema.js +118 -0
- package/dist/server/operations/get-task-detail.d.ts +346 -0
- package/dist/server/operations/get-task-detail.js +215 -0
- package/dist/server/operations/static-registrations.js +3 -0
- package/dist/storage/auto.js +1 -1
- package/dist/storage/bun-sql.js +113 -0
- package/dist/storage/compressed-storage.js +1 -1
- package/dist/storage/index.d.ts +38 -35
- package/dist/storage/interface.d.ts +87 -125
- package/dist/storage/interface.js +1 -1
- package/dist/storage/key-encoding.d.ts +51 -0
- package/dist/storage/key-encoding.js +13 -0
- package/dist/storage/key-prefixes.d.ts +1 -1
- package/dist/storage/key-prefixes.js +3 -0
- package/dist/storage/lmdb.js +1 -1
- package/dist/storage/memory.js +1 -1
- package/dist/storage/neon.js +2 -2
- package/dist/storage/node-sqlite.js +113 -0
- package/dist/storage/ownership-keys.d.ts +55 -0
- package/dist/storage/ownership-keys.js +6 -0
- package/dist/storage/postgres.js +2 -2
- package/dist/storage/resolve.js +1 -1
- package/dist/storage/scoped-storage.js +1 -1
- package/dist/storage/testing.js +1 -1
- package/dist/storage/turso.js +2 -2
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/web-extension.js +1 -1
- package/dist/worker/protocol.js +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
export const WORKFLOW_CLAIM_ATTEMPT_OUTCOMES = [
|
|
2
|
+
"acquired",
|
|
3
|
+
"takeover",
|
|
4
|
+
"lost_race",
|
|
5
|
+
"deposed",
|
|
6
|
+
"backoff_skipped"
|
|
7
|
+
];
|
|
8
|
+
function assertSafeNonNegativeInteger(value, label) {
|
|
9
|
+
if (!Number.isInteger(value) || !Number.isSafeInteger(value) || value < 0)
|
|
10
|
+
throw RangeError(`${label} must be a safe, non-negative integer; received ${String(value)}`);
|
|
11
|
+
}
|
|
12
|
+
function zeroFilledAttemptCounts() {
|
|
13
|
+
const counts = {};
|
|
14
|
+
for (const outcome of WORKFLOW_CLAIM_ATTEMPT_OUTCOMES)
|
|
15
|
+
counts[outcome] = 0;
|
|
16
|
+
return counts;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export class WorkflowClaimMetricsCollector {
|
|
20
|
+
#attempts;
|
|
21
|
+
#activeClaims;
|
|
22
|
+
#renewalFailures;
|
|
23
|
+
constructor() {
|
|
24
|
+
this.#attempts = zeroFilledAttemptCounts();
|
|
25
|
+
this.#activeClaims = 0;
|
|
26
|
+
this.#renewalFailures = 0;
|
|
27
|
+
}
|
|
28
|
+
recordClaimAttempt(outcome) {
|
|
29
|
+
this.#attempts[outcome] += 1;
|
|
30
|
+
}
|
|
31
|
+
setActiveClaims(count) {
|
|
32
|
+
assertSafeNonNegativeInteger(count, "activeClaims");
|
|
33
|
+
this.#activeClaims = count;
|
|
34
|
+
}
|
|
35
|
+
recordClaimRenewalFailure() {
|
|
36
|
+
this.#renewalFailures += 1;
|
|
37
|
+
}
|
|
38
|
+
snapshot() {
|
|
39
|
+
return {
|
|
40
|
+
attempts: { ...this.#attempts },
|
|
41
|
+
activeClaims: this.#activeClaims,
|
|
42
|
+
renewalFailures: this.#renewalFailures
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
reset() {
|
|
46
|
+
this.#attempts = zeroFilledAttemptCounts();
|
|
47
|
+
this.#activeClaims = 0;
|
|
48
|
+
this.#renewalFailures = 0;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
export function serializeWorkflowClaimMetricsForPrometheus(snapshot) {
|
|
52
|
+
const lines = [
|
|
53
|
+
"# HELP weft_workflow_claim_attempts_total Total per-workflow ownership claim attempts, by outcome",
|
|
54
|
+
"# TYPE weft_workflow_claim_attempts_total counter"
|
|
55
|
+
];
|
|
56
|
+
for (const outcome of WORKFLOW_CLAIM_ATTEMPT_OUTCOMES)
|
|
57
|
+
lines.push(`weft_workflow_claim_attempts_total{outcome="${outcome}"} ${snapshot.attempts[outcome]}`);
|
|
58
|
+
lines.push("# HELP weft_workflow_claims_active Workflows this engine currently holds a claim for", "# TYPE weft_workflow_claims_active gauge", `weft_workflow_claims_active ${snapshot.activeClaims}`, "# HELP weft_workflow_claim_renewal_failures_total Total per-workflow claim renewal failures", "# TYPE weft_workflow_claim_renewal_failures_total counter", `weft_workflow_claim_renewal_failures_total ${snapshot.renewalFailures}`);
|
|
59
|
+
return lines.join(`
|
|
60
|
+
`) + `
|
|
61
|
+
`;
|
|
62
|
+
}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Storage-scan candidate discovery for the recurring reclaim pass described
|
|
3
|
+
* in [ADR 0002 § Reclaiming stranded claims](../../../documentation/contributing/architecture-decisions/0002-multiengine-per-workflow-ownership.md#reclaiming-stranded-claims).
|
|
4
|
+
*
|
|
5
|
+
* A single boot-time `recoverAll()` sweep only catches a stranded claim that
|
|
6
|
+
* happens to already be past its grace-adjusted expiry at boot. An engine
|
|
7
|
+
* that crashes (or whose graceful-shutdown release fails) later leaves its
|
|
8
|
+
* `wf-owner-holder:<id>` record behind for every OTHER engine to discover;
|
|
9
|
+
* nothing rescans for that unless something recurring does. This module is
|
|
10
|
+
* the discovery half of that recurring scan: enumerate every workflow id
|
|
11
|
+
* with a currently-persisted holder record, store-wide, so a caller can
|
|
12
|
+
* attempt `WorkflowClaimRegistry.takeover` against each one.
|
|
13
|
+
*
|
|
14
|
+
* **Deliberately not a liveness filter.** This module does not decode holder
|
|
15
|
+
* bytes or judge staleness — `WorkflowClaimRegistry.takeover` already does
|
|
16
|
+
* that with a fresh read at the moment of the attempt (see its own
|
|
17
|
+
* grace-adjusted `isWorkflowClaimExpired` check), and re-deciding it here
|
|
18
|
+
* from a possibly-stale scan read would be redundant at best and a source of
|
|
19
|
+
* a second, drifting judgment at worst. This is discovery only.
|
|
20
|
+
*
|
|
21
|
+
* **Ownerless-but-running workflows.** The holder-keyed scan above misses a
|
|
22
|
+
* real rolling-deploy shape: an incoming engine boots and runs its
|
|
23
|
+
* `recoverAll()` sweep WHILE an outgoing engine is still the live holder of
|
|
24
|
+
* some workflow — recovery correctly skips it, since the holder is not yet
|
|
25
|
+
* expired. If the outgoing engine then disposes gracefully, its
|
|
26
|
+
* `releaseAll()` DELETES that workflow's `wf-owner-holder:<id>` record
|
|
27
|
+
* (per the ADR's `release` row — the epoch is retained, only the holder
|
|
28
|
+
* goes). The workflow is now `running` in `WorkflowState` with no holder at
|
|
29
|
+
* all, and the holder-keyed scan above will never find it again — there is
|
|
30
|
+
* no `wf-owner-holder:<id>` key left to enumerate. The already-running
|
|
31
|
+
* incoming engine has nothing further to trigger a re-scan, so the workflow
|
|
32
|
+
* is stranded until an operator explicitly calls `recoverAll()`/`resume()`.
|
|
33
|
+
* This module closes that gap with a second scan over the `running`-status
|
|
34
|
+
* workflow-visibility index (`wf-idx-status:running:`), checking each
|
|
35
|
+
* candidate's holder key fresh and including only the genuinely holderless
|
|
36
|
+
* ones — bounded to the same cost class as the holder-keyed scan (one extra
|
|
37
|
+
* index prefix scan per pass, plus one point read per running-and-holderless
|
|
38
|
+
* candidate).
|
|
39
|
+
*
|
|
40
|
+
* **Workflows with no visibility-index entry at all (WFT-79 Finding 2).**
|
|
41
|
+
* The index-based scan above is itself incomplete on a Bun SQLite deployment
|
|
42
|
+
* that predates the workflow visibility indexes and has not yet run the
|
|
43
|
+
* one-time backfill (see
|
|
44
|
+
* [Workflow Visibility Backfill](../../../documentation/guides/workflow-visibility-backfill.md)):
|
|
45
|
+
* such a workflow has no `wf-idx-status:running:<id>` row to enumerate. If
|
|
46
|
+
* that workflow is the outgoing engine's live holder at the moment an
|
|
47
|
+
* incoming engine's `recoverAll()` sweep runs, recovery correctly skips it
|
|
48
|
+
* (the holder is not yet expired); if the outgoing engine then disposes
|
|
49
|
+
* gracefully, `releaseAll()` deletes the holder record and the workflow is
|
|
50
|
+
* left `running` with no holder AND no visibility-index entry — invisible to
|
|
51
|
+
* both scans above. Left there, it is stranded indefinitely: nothing else
|
|
52
|
+
* re-scans for it.
|
|
53
|
+
*
|
|
54
|
+
* This module closes that second gap with a bounded, cursor-rotated fallback
|
|
55
|
+
* scan directly over the authoritative `wf:<id>` workflow records (the same
|
|
56
|
+
* source `recoverAll()`'s own preflight already scans at boot — see
|
|
57
|
+
* `lifecycle/transition.ts`'s `preflightRecoverAll`), decoding each record
|
|
58
|
+
* and including only genuinely holderless `running` ones not already found
|
|
59
|
+
* by either scan above. This IS a store-wide operation and therefore more
|
|
60
|
+
* expensive than the index-based scans, so it is bounded per pass to
|
|
61
|
+
* {@link WORKFLOW_CLAIM_RECLAIM_AUTHORITATIVE_SCAN_LIMIT} records rather than
|
|
62
|
+
* scanning the entire keyspace on every call. A per-storage cursor (advanced
|
|
63
|
+
* past the last key read, and wrapped back to the start once a pass reaches
|
|
64
|
+
* the end of the keyspace) rotates the scanned window across passes, so a
|
|
65
|
+
* store with more un-backfilled workflows than the per-pass limit still gets
|
|
66
|
+
* full coverage over several reclaim-scan passes rather than only ever
|
|
67
|
+
* re-scanning the same lexicographically-first window. This fallback is
|
|
68
|
+
* expected to do genuine work only on deployments that have not yet run the
|
|
69
|
+
* visibility backfill; run that backfill to eliminate this scan's ongoing
|
|
70
|
+
* cost entirely.
|
|
71
|
+
*
|
|
72
|
+
* @module core/engine/workflow-claim-reclaim-scan
|
|
73
|
+
*/
|
|
74
|
+
import { type Storage } from '../../storage/interface.ts';
|
|
75
|
+
/**
|
|
76
|
+
* Per-pass bound on the authoritative-record fallback scan
|
|
77
|
+
* ({@link listOwnerlessRunningCandidatesFromAuthoritativeRecords}). This
|
|
78
|
+
* fallback is a store-wide scan, unlike the two index-based scans above, so
|
|
79
|
+
* it is capped rather than run to exhaustion on every pass — see the module
|
|
80
|
+
* doc's "Workflows with no visibility-index entry at all" section for why
|
|
81
|
+
* the cap is safe (a per-storage cursor rotates the scanned window across
|
|
82
|
+
* passes, so a store with more un-backfilled workflows than this limit still
|
|
83
|
+
* gets full coverage over several passes).
|
|
84
|
+
*/
|
|
85
|
+
export declare const WORKFLOW_CLAIM_RECLAIM_AUTHORITATIVE_SCAN_LIMIT = 500;
|
|
86
|
+
/**
|
|
87
|
+
* Full candidate discovery for one reclaim-scan pass: every workflow id with
|
|
88
|
+
* a currently-persisted holder record, every `running`-status workflow id
|
|
89
|
+
* with NO holder record at all found via the visibility index (see the
|
|
90
|
+
* module doc's "Ownerless-but-running workflows" section), and every
|
|
91
|
+
* `running`-status, holderless workflow id found by the bounded
|
|
92
|
+
* authoritative-record fallback for workflows with no visibility-index entry
|
|
93
|
+
* at all (see the module doc's "Workflows with no visibility-index entry at
|
|
94
|
+
* all" section) — excluding `excludeWorkflowIds` from all three.
|
|
95
|
+
*/
|
|
96
|
+
export declare function listWorkflowClaimReclaimCandidates(storage: Storage, excludeWorkflowIds: ReadonlySet<string>): Promise<string[]>;
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { KEYS, tryDecodeStorageKeyComponent } from "../../storage/interface.js";
|
|
2
|
+
import { decodeWorkflowState } from "./validation.js";
|
|
3
|
+
const RUNNING_STATUS = "running";
|
|
4
|
+
export const WORKFLOW_CLAIM_RECLAIM_AUTHORITATIVE_SCAN_LIMIT = 500;
|
|
5
|
+
const authoritativeScanCursors = new WeakMap;
|
|
6
|
+
async function listHolderScanCandidates(storage, excludeWorkflowIds) {
|
|
7
|
+
const prefix = KEYS.workflowOwnerHolder(""), candidates = [];
|
|
8
|
+
for await (const [key] of storage.scan(prefix)) {
|
|
9
|
+
const workflowId = tryDecodeStorageKeyComponent(key.slice(prefix.length));
|
|
10
|
+
if (workflowId === null || excludeWorkflowIds.has(workflowId))
|
|
11
|
+
continue;
|
|
12
|
+
candidates.push(workflowId);
|
|
13
|
+
}
|
|
14
|
+
return candidates;
|
|
15
|
+
}
|
|
16
|
+
async function listOwnerlessRunningCandidates(storage, excludeWorkflowIds, seen) {
|
|
17
|
+
const prefix = KEYS.workflowVisibilityStatus(RUNNING_STATUS, ""), candidates = [];
|
|
18
|
+
for await (const [key] of storage.scan(prefix)) {
|
|
19
|
+
const workflowId = tryDecodeStorageKeyComponent(key.slice(prefix.length));
|
|
20
|
+
if (workflowId === null || excludeWorkflowIds.has(workflowId) || seen.has(workflowId) || candidates.includes(workflowId))
|
|
21
|
+
continue;
|
|
22
|
+
if (await storage.get(KEYS.workflowOwnerHolder(workflowId)) !== null)
|
|
23
|
+
continue;
|
|
24
|
+
candidates.push(workflowId);
|
|
25
|
+
}
|
|
26
|
+
return candidates;
|
|
27
|
+
}
|
|
28
|
+
async function classifyAuthoritativeScanEntry(storage, prefix, key, value, excludeWorkflowIds, seen, alreadyCollected) {
|
|
29
|
+
const workflowId = tryDecodeStorageKeyComponent(key.slice(prefix.length));
|
|
30
|
+
if (workflowId === null)
|
|
31
|
+
return null;
|
|
32
|
+
if (KEYS.workflow(workflowId) !== key)
|
|
33
|
+
return null;
|
|
34
|
+
if (excludeWorkflowIds.has(workflowId) || seen.has(workflowId) || alreadyCollected.has(workflowId))
|
|
35
|
+
return null;
|
|
36
|
+
let status;
|
|
37
|
+
try {
|
|
38
|
+
status = decodeWorkflowState(value).status;
|
|
39
|
+
} catch {
|
|
40
|
+
return null;
|
|
41
|
+
}
|
|
42
|
+
if (status !== RUNNING_STATUS)
|
|
43
|
+
return null;
|
|
44
|
+
if (await storage.get(KEYS.workflowOwnerHolder(workflowId)) !== null)
|
|
45
|
+
return null;
|
|
46
|
+
return workflowId;
|
|
47
|
+
}
|
|
48
|
+
async function listOwnerlessRunningCandidatesFromAuthoritativeRecords(storage, excludeWorkflowIds, seen) {
|
|
49
|
+
const prefix = KEYS.workflow(""), cursor = authoritativeScanCursors.get(storage), scanOptions = cursor === void 0 ? { limit: WORKFLOW_CLAIM_RECLAIM_AUTHORITATIVE_SCAN_LIMIT } : { limit: WORKFLOW_CLAIM_RECLAIM_AUTHORITATIVE_SCAN_LIMIT, gt: cursor }, candidates = [], collected = new Set;
|
|
50
|
+
let scannedCount = 0, lastKey;
|
|
51
|
+
for await (const [key, value] of storage.scan(prefix, scanOptions)) {
|
|
52
|
+
scannedCount += 1;
|
|
53
|
+
lastKey = key;
|
|
54
|
+
const workflowId = await classifyAuthoritativeScanEntry(storage, prefix, key, value, excludeWorkflowIds, seen, collected);
|
|
55
|
+
if (workflowId === null)
|
|
56
|
+
continue;
|
|
57
|
+
collected.add(workflowId);
|
|
58
|
+
candidates.push(workflowId);
|
|
59
|
+
}
|
|
60
|
+
authoritativeScanCursors.set(storage, scannedCount < WORKFLOW_CLAIM_RECLAIM_AUTHORITATIVE_SCAN_LIMIT ? void 0 : lastKey);
|
|
61
|
+
return candidates;
|
|
62
|
+
}
|
|
63
|
+
export async function listWorkflowClaimReclaimCandidates(storage, excludeWorkflowIds) {
|
|
64
|
+
const holderCandidates = await listHolderScanCandidates(storage, excludeWorkflowIds), ownerlessCandidates = await listOwnerlessRunningCandidates(storage, excludeWorkflowIds, new Set(holderCandidates)), authoritativeFallbackCandidates = await listOwnerlessRunningCandidatesFromAuthoritativeRecords(storage, excludeWorkflowIds, new Set([...holderCandidates, ...ownerlessCandidates]));
|
|
65
|
+
return [...holderCandidates, ...ownerlessCandidates, ...authoritativeFallbackCandidates];
|
|
66
|
+
}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The reclaim-scan candidate-discovery and takeover-attempt machinery for
|
|
3
|
+
* `ownership: 'workflow-lease'` ([ADR 0002](../../../documentation/contributing/architecture-decisions/0002-multiengine-per-workflow-ownership.md)).
|
|
4
|
+
* Extracted from `ownership-bootstrap.ts` — which still composes this module's
|
|
5
|
+
* {@link createWorkflowClaimReclaimTarget} into the renewal task's
|
|
6
|
+
* `reclaimTarget` option — so that file stays under the repository's
|
|
7
|
+
* implementation-file-size ceiling as this seam grows disposal-quiescence and
|
|
8
|
+
* epoch-fenced-release handling on top of the original takeover-retry loop.
|
|
9
|
+
* This is a real responsibility boundary, not an arbitrary split: everything
|
|
10
|
+
* here is "how one engine discovers and attempts to reclaim a stranded
|
|
11
|
+
* workflow's claim," while `ownership-bootstrap.ts` keeps gate execution,
|
|
12
|
+
* registry/renewal-task construction, and the owner-side signal-poll seam.
|
|
13
|
+
*
|
|
14
|
+
* @module core/engine/workflow-claim-reclaim-target
|
|
15
|
+
*/
|
|
16
|
+
import type { Storage } from '../../storage/interface.ts';
|
|
17
|
+
import type { WorkflowClaimMetricsCollector } from './workflow-claim-metrics.ts';
|
|
18
|
+
import type { WorkflowClaimRegistry } from './workflow-claim-registry.ts';
|
|
19
|
+
import type { WorkflowClaimReclaimTarget } from './workflow-claim-renewal-subpasses.ts';
|
|
20
|
+
/** Bound on retrying a lost-race `takeover` CAS for one reclaim candidate within one pass — ADR 0002's `takeover` row. */
|
|
21
|
+
export declare const WORKFLOW_CLAIM_TAKEOVER_MAX_ATTEMPTS = 5;
|
|
22
|
+
/**
|
|
23
|
+
* A {@link WorkflowClaimReclaimTarget} with one additional, non-interface
|
|
24
|
+
* method: {@link markDisposing}. Structurally still a valid
|
|
25
|
+
* `WorkflowClaimReclaimTarget` (every caller that only knows that narrower
|
|
26
|
+
* type — e.g. `createWorkflowClaimRenewalTask`'s `reclaimTarget` option —
|
|
27
|
+
* keeps working unchanged), so existing tests that only exercise
|
|
28
|
+
* `listReclaimCandidateWorkflowIds`/`attemptWorkflowClaimTakeover` are
|
|
29
|
+
* unaffected by this addition.
|
|
30
|
+
*/
|
|
31
|
+
export type WorkflowClaimReclaimTargetHandle = WorkflowClaimReclaimTarget & {
|
|
32
|
+
/**
|
|
33
|
+
* Synchronously flip this target into "disposing" mode: every future
|
|
34
|
+
* `attemptWorkflowClaimTakeover` call becomes an immediate `'not-eligible'`
|
|
35
|
+
* no-op (no CAS attempted, no `onReclaimed` drive invoked), and
|
|
36
|
+
* `listReclaimCandidateWorkflowIds` returns `[]`. A reclaim attempt already
|
|
37
|
+
* past this checkpoint when disposal begins keeps running to its next
|
|
38
|
+
* checkpoint — every checkpoint after an `await` re-checks the flag — and,
|
|
39
|
+
* if it lands a takeover/acquire CAS after disposal was signaled, releases
|
|
40
|
+
* that claim immediately instead of driving it or leaving it held. See this
|
|
41
|
+
* module's doc and `ownership-bootstrap.ts`'s `bootstrapWorkflowLeaseOwnership`
|
|
42
|
+
* for how this is wired to `WorkflowClaimRenewalTask.stop()`.
|
|
43
|
+
*
|
|
44
|
+
* Idempotent. Calling this before any pass has started simply prevents one
|
|
45
|
+
* from ever discovering or attempting a candidate.
|
|
46
|
+
*/
|
|
47
|
+
markDisposing(): void;
|
|
48
|
+
};
|
|
49
|
+
/**
|
|
50
|
+
* Adapt a {@link WorkflowClaimRegistry} plus `storage` to the renewal task's
|
|
51
|
+
* {@link WorkflowClaimReclaimTarget} contract. Candidate discovery excludes
|
|
52
|
+
* this engine's own currently-held ids (`registry.listHeldWorkflowIds()`) —
|
|
53
|
+
* see `workflow-claim-reclaim-scan.ts`'s doc for why — then adds back any
|
|
54
|
+
* workflow this engine holds but whose `onReclaimed` drive previously
|
|
55
|
+
* failed (see `driveReclaimedWorkflow` below). `attemptWorkflowClaimTakeover`
|
|
56
|
+
* retries a `'lost-race'` CAS, bounded at {@link WORKFLOW_CLAIM_TAKEOVER_MAX_ATTEMPTS}
|
|
57
|
+
* per the ADR, and records `weft_workflow_claim_attempts_total{outcome="backoff_skipped"}`
|
|
58
|
+
* the moment the registry's own anti-thrash cooldown suppresses an attempt —
|
|
59
|
+
* the one `WorkflowClaimAttemptOutcome` this stage wires; the other four
|
|
60
|
+
* remain unrecorded by design (see the ADR's Observability section for the
|
|
61
|
+
* full set — that wiring is a later stage's work).
|
|
62
|
+
*
|
|
63
|
+
* **A failed `onReclaimed` drive is retried in place, never released.**
|
|
64
|
+
* Releasing on failure was considered and rejected: `onReclaimed` (bound to
|
|
65
|
+
* `resumeWorkflowFromStorage` in production) can throw AFTER
|
|
66
|
+
* `relaunchInlineWorkflowAfterResume` has already adopted the generator —
|
|
67
|
+
* `InlineExecutionStrategy#continueWorkflow` fires the drive and returns
|
|
68
|
+
* without awaiting it, so a caught error here does not prove no local user
|
|
69
|
+
* code started. Releasing the claim in that state would let another engine
|
|
70
|
+
* `acquire` it while this engine may still be mid-turn — the exact
|
|
71
|
+
* duplicate-execution hazard ADR 0002 exists to close, just re-opened via
|
|
72
|
+
* the failure path instead of the happy path. Retrying in place keeps the
|
|
73
|
+
* claim (and its write fence) intact and simply asks `onReclaimed` again on
|
|
74
|
+
* a later pass, via `pendingRedriveWorkflowIds` below.
|
|
75
|
+
*
|
|
76
|
+
* **Discovered and pending-redrive candidates are merged through a `Set`
|
|
77
|
+
* (WFT-79 Finding 4).** After a failed redrive loses this engine's local
|
|
78
|
+
* claim (a renewal loss between the failed drive and the next pass), the
|
|
79
|
+
* same workflow id can surface BOTH through `listWorkflowClaimReclaimCandidates`
|
|
80
|
+
* (as a foreign holder, since `registry.listHeldWorkflowIds()` no longer
|
|
81
|
+
* excludes it) AND through `pendingRedriveWorkflowIds`. Without deduping,
|
|
82
|
+
* one renewal pass would call `attemptWorkflowClaimTakeover` for that id
|
|
83
|
+
* twice, and each call independently retries up to
|
|
84
|
+
* {@link WORKFLOW_CLAIM_TAKEOVER_MAX_ATTEMPTS} — doubling the advertised
|
|
85
|
+
* per-pass bound to 10 attempts for that workflow, exactly when contention is
|
|
86
|
+
* already highest (deposition churn).
|
|
87
|
+
*
|
|
88
|
+
* **Disposal quiescence (WFT-79 Finding 2).** {@link WorkflowClaimReclaimTargetHandle.markDisposing}
|
|
89
|
+
* closes two related hazards when disposal overlaps an in-flight
|
|
90
|
+
* interval-driven pass: a late-landing takeover/acquire CAS stranding a claim
|
|
91
|
+
* this engine will never renew again, and a late `onReclaimed` drive running
|
|
92
|
+
* against a torn-down host. Every checkpoint that follows an `await` inside
|
|
93
|
+
* this target re-checks the flag; a CAS that lands after disposal was
|
|
94
|
+
* signaled is released immediately rather than driven, so the claim never
|
|
95
|
+
* outlives this engine's own best-effort `WorkflowClaimRegistry.releaseAll()`
|
|
96
|
+
* call regardless of the ordering race between them.
|
|
97
|
+
*/
|
|
98
|
+
export declare function createWorkflowClaimReclaimTarget(registry: WorkflowClaimRegistry, storage: Storage, metrics: WorkflowClaimMetricsCollector, onReclaimed?: (workflowId: string) => Promise<void>,
|
|
99
|
+
/**
|
|
100
|
+
* Optional workflow-type eligibility check, consulted before this engine
|
|
101
|
+
* ever attempts a fresh takeover/acquire CAS for a candidate (never for a
|
|
102
|
+
* `redriveAlreadyHeldClaim` retry, since that claim already passed this
|
|
103
|
+
* check when it was first taken). Omitted (the default) skips the check —
|
|
104
|
+
* every existing caller/test that does not care about mixed workflow-type
|
|
105
|
+
* fleets keeps working unchanged.
|
|
106
|
+
*/
|
|
107
|
+
isTypeRegistered?: (workflowType: string) => boolean): WorkflowClaimReclaimTargetHandle;
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
import { KEYS } from "../../storage/interface.js";
|
|
2
|
+
import { decodeWorkflowState } from "./validation.js";
|
|
3
|
+
import { listWorkflowClaimReclaimCandidates } from "./workflow-claim-reclaim-scan.js";
|
|
4
|
+
export const WORKFLOW_CLAIM_TAKEOVER_MAX_ATTEMPTS = 5;
|
|
5
|
+
async function isWorkflowStillRunning(storage, workflowId) {
|
|
6
|
+
const bytes = await storage.get(KEYS.workflow(workflowId));
|
|
7
|
+
if (bytes === null)
|
|
8
|
+
return !1;
|
|
9
|
+
try {
|
|
10
|
+
return decodeWorkflowState(bytes).status === "running";
|
|
11
|
+
} catch {
|
|
12
|
+
return !1;
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
async function isWorkflowTypeRegistered(storage, workflowId, isTypeRegistered) {
|
|
16
|
+
const bytes = await storage.get(KEYS.workflow(workflowId));
|
|
17
|
+
if (bytes === null)
|
|
18
|
+
return !1;
|
|
19
|
+
try {
|
|
20
|
+
return isTypeRegistered(decodeWorkflowState(bytes).type);
|
|
21
|
+
} catch {
|
|
22
|
+
return !1;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
export function createWorkflowClaimReclaimTarget(registry, storage, metrics, onReclaimed, isTypeRegistered) {
|
|
26
|
+
const pendingRedriveWorkflowIds = new Map;
|
|
27
|
+
let disposing = !1;
|
|
28
|
+
async function driveReclaimedWorkflow(workflowId, epoch) {
|
|
29
|
+
if (onReclaimed === void 0) {
|
|
30
|
+
pendingRedriveWorkflowIds.delete(workflowId);
|
|
31
|
+
return { status: "reclaimed" };
|
|
32
|
+
}
|
|
33
|
+
try {
|
|
34
|
+
await onReclaimed(workflowId);
|
|
35
|
+
pendingRedriveWorkflowIds.delete(workflowId);
|
|
36
|
+
return { status: "reclaimed" };
|
|
37
|
+
} catch (error) {
|
|
38
|
+
pendingRedriveWorkflowIds.set(workflowId, epoch);
|
|
39
|
+
throw error;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
async function releaseClaimAcquiredWhileDisposing(workflowId) {
|
|
43
|
+
await registry.release(workflowId);
|
|
44
|
+
}
|
|
45
|
+
async function confirmStillRunningOrReleaseFreshClaim(workflowId, acquiredEpoch) {
|
|
46
|
+
if (await isWorkflowStillRunning(storage, workflowId))
|
|
47
|
+
return !0;
|
|
48
|
+
if (registry.currentEpoch(workflowId) === acquiredEpoch)
|
|
49
|
+
await registry.release(workflowId);
|
|
50
|
+
return !1;
|
|
51
|
+
}
|
|
52
|
+
async function redriveAlreadyHeldClaim(workflowId, expectedEpoch) {
|
|
53
|
+
if (disposing)
|
|
54
|
+
return { status: "not-eligible" };
|
|
55
|
+
if (!await isWorkflowStillRunning(storage, workflowId)) {
|
|
56
|
+
pendingRedriveWorkflowIds.delete(workflowId);
|
|
57
|
+
if (registry.currentEpoch(workflowId) === expectedEpoch)
|
|
58
|
+
await registry.release(workflowId);
|
|
59
|
+
return { status: "not-eligible" };
|
|
60
|
+
}
|
|
61
|
+
if (disposing)
|
|
62
|
+
return { status: "not-eligible" };
|
|
63
|
+
return await driveReclaimedWorkflow(workflowId, expectedEpoch);
|
|
64
|
+
}
|
|
65
|
+
async function acquireOwnerlessRunningClaim(workflowId) {
|
|
66
|
+
if (disposing)
|
|
67
|
+
return { status: "not-eligible" };
|
|
68
|
+
if (!await isWorkflowStillRunning(storage, workflowId))
|
|
69
|
+
return { status: "not-eligible" };
|
|
70
|
+
if (disposing)
|
|
71
|
+
return { status: "not-eligible" };
|
|
72
|
+
const acquireResult = await registry.acquire(workflowId);
|
|
73
|
+
if (acquireResult.status === "lost-race") {
|
|
74
|
+
metrics.recordClaimAttempt("lost_race");
|
|
75
|
+
return "retry";
|
|
76
|
+
}
|
|
77
|
+
metrics.recordClaimAttempt("acquired");
|
|
78
|
+
if (disposing) {
|
|
79
|
+
await releaseClaimAcquiredWhileDisposing(workflowId);
|
|
80
|
+
return { status: "not-eligible" };
|
|
81
|
+
}
|
|
82
|
+
if (!await confirmStillRunningOrReleaseFreshClaim(workflowId, acquireResult.epoch))
|
|
83
|
+
return { status: "not-eligible" };
|
|
84
|
+
return await driveReclaimedWorkflow(workflowId, acquireResult.epoch);
|
|
85
|
+
}
|
|
86
|
+
async function handleTakeoverAcquired(workflowId, acquiredEpoch) {
|
|
87
|
+
metrics.recordClaimAttempt("takeover");
|
|
88
|
+
if (disposing) {
|
|
89
|
+
await releaseClaimAcquiredWhileDisposing(workflowId);
|
|
90
|
+
return { status: "not-eligible" };
|
|
91
|
+
}
|
|
92
|
+
if (!await confirmStillRunningOrReleaseFreshClaim(workflowId, acquiredEpoch))
|
|
93
|
+
return { status: "not-eligible" };
|
|
94
|
+
return await driveReclaimedWorkflow(workflowId, acquiredEpoch);
|
|
95
|
+
}
|
|
96
|
+
async function isEligibleForFreshTakeover(workflowId) {
|
|
97
|
+
if (isTypeRegistered === void 0)
|
|
98
|
+
return !0;
|
|
99
|
+
return await isWorkflowTypeRegistered(storage, workflowId, isTypeRegistered);
|
|
100
|
+
}
|
|
101
|
+
async function takeoverWithRetries(workflowId) {
|
|
102
|
+
if (!await isEligibleForFreshTakeover(workflowId))
|
|
103
|
+
return { status: "not-eligible" };
|
|
104
|
+
for (let attempt = 0;attempt < WORKFLOW_CLAIM_TAKEOVER_MAX_ATTEMPTS; attempt += 1) {
|
|
105
|
+
if (disposing)
|
|
106
|
+
return { status: "not-eligible" };
|
|
107
|
+
const result = await registry.takeover(workflowId);
|
|
108
|
+
switch (result.status) {
|
|
109
|
+
case "acquired":
|
|
110
|
+
return await handleTakeoverAcquired(workflowId, result.epoch);
|
|
111
|
+
case "backoff-skipped":
|
|
112
|
+
metrics.recordClaimAttempt("backoff_skipped");
|
|
113
|
+
return { status: "backoff-skipped" };
|
|
114
|
+
case "not-expired":
|
|
115
|
+
return { status: "not-eligible" };
|
|
116
|
+
case "no-claim": {
|
|
117
|
+
const outcome = await acquireOwnerlessRunningClaim(workflowId);
|
|
118
|
+
if (outcome === "retry")
|
|
119
|
+
continue;
|
|
120
|
+
return outcome;
|
|
121
|
+
}
|
|
122
|
+
case "lost-race":
|
|
123
|
+
metrics.recordClaimAttempt("lost_race");
|
|
124
|
+
continue;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
return { status: "lost-race" };
|
|
128
|
+
}
|
|
129
|
+
return {
|
|
130
|
+
async listReclaimCandidateWorkflowIds() {
|
|
131
|
+
if (disposing)
|
|
132
|
+
return [];
|
|
133
|
+
const discovered = await listWorkflowClaimReclaimCandidates(storage, new Set(registry.listHeldWorkflowIds()));
|
|
134
|
+
return [...new Set([...discovered, ...pendingRedriveWorkflowIds.keys()])];
|
|
135
|
+
},
|
|
136
|
+
async attemptWorkflowClaimTakeover(workflowId) {
|
|
137
|
+
if (disposing)
|
|
138
|
+
return { status: "not-eligible" };
|
|
139
|
+
const currentEpoch = registry.currentEpoch(workflowId);
|
|
140
|
+
if (currentEpoch !== null) {
|
|
141
|
+
const pendingRedriveEpoch = pendingRedriveWorkflowIds.get(workflowId);
|
|
142
|
+
if (pendingRedriveEpoch !== void 0 && pendingRedriveEpoch !== currentEpoch) {
|
|
143
|
+
pendingRedriveWorkflowIds.delete(workflowId);
|
|
144
|
+
return await takeoverWithRetries(workflowId);
|
|
145
|
+
}
|
|
146
|
+
return await redriveAlreadyHeldClaim(workflowId, currentEpoch);
|
|
147
|
+
}
|
|
148
|
+
pendingRedriveWorkflowIds.delete(workflowId);
|
|
149
|
+
return await takeoverWithRetries(workflowId);
|
|
150
|
+
},
|
|
151
|
+
markDisposing() {
|
|
152
|
+
disposing = !0;
|
|
153
|
+
}
|
|
154
|
+
};
|
|
155
|
+
}
|