@adonis-agora/durable 0.58.3 → 0.60.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +68 -0
- package/dist/assets/spa/assets/index-70hGzOnT.css +1 -0
- package/dist/assets/spa/assets/index-BI1EANJO.js +16 -0
- package/dist/assets/spa/assets/index-BI1EANJO.js.map +1 -0
- package/dist/assets/spa/index.html +2 -2
- package/dist/commands/lifecycle_republish.d.ts +23 -0
- package/dist/commands/lifecycle_republish.d.ts.map +1 -0
- package/dist/commands/lifecycle_republish.js +64 -0
- package/dist/commands/lifecycle_republish.js.map +1 -0
- package/dist/commands/main.d.ts +4 -3
- package/dist/commands/main.d.ts.map +1 -1
- package/dist/commands/main.js +3 -1
- package/dist/commands/main.js.map +1 -1
- package/dist/providers/dashboard_provider.d.ts +2 -0
- package/dist/providers/dashboard_provider.d.ts.map +1 -1
- package/dist/providers/dashboard_provider.js +9 -1
- package/dist/providers/dashboard_provider.js.map +1 -1
- package/dist/providers/durable_provider.d.ts.map +1 -1
- package/dist/providers/durable_provider.js +3 -0
- package/dist/providers/durable_provider.js.map +1 -1
- package/dist/src/codec-state-store.d.ts +10 -1
- package/dist/src/codec-state-store.d.ts.map +1 -1
- package/dist/src/codec-state-store.js +25 -0
- package/dist/src/codec-state-store.js.map +1 -1
- package/dist/src/commands/index.d.ts +1 -1
- package/dist/src/commands/index.d.ts.map +1 -1
- package/dist/src/commands/index.js +1 -1
- package/dist/src/commands/index.js.map +1 -1
- package/dist/src/commands/runs.d.ts +24 -1
- package/dist/src/commands/runs.d.ts.map +1 -1
- package/dist/src/commands/runs.js +50 -0
- package/dist/src/commands/runs.js.map +1 -1
- package/dist/src/commands/worker.d.ts.map +1 -1
- package/dist/src/commands/worker.js +3 -0
- package/dist/src/commands/worker.js.map +1 -1
- package/dist/src/dashboard/gateway-adapter.d.ts +5 -0
- package/dist/src/dashboard/gateway-adapter.d.ts.map +1 -1
- package/dist/src/dashboard/gateway-adapter.js +20 -0
- package/dist/src/dashboard/gateway-adapter.js.map +1 -1
- package/dist/src/dashboard/handlers.d.ts +29 -1
- package/dist/src/dashboard/handlers.d.ts.map +1 -1
- package/dist/src/dashboard/handlers.js +103 -0
- package/dist/src/dashboard/handlers.js.map +1 -1
- package/dist/src/dashboard/index.d.ts +2 -2
- package/dist/src/dashboard/index.d.ts.map +1 -1
- package/dist/src/dashboard/index.js +2 -2
- package/dist/src/dashboard/index.js.map +1 -1
- package/dist/src/dashboard/openapi.d.ts.map +1 -1
- package/dist/src/dashboard/openapi.js +39 -0
- package/dist/src/dashboard/openapi.js.map +1 -1
- package/dist/src/define_config.d.ts +13 -0
- package/dist/src/define_config.d.ts.map +1 -1
- package/dist/src/define_config.js.map +1 -1
- package/dist/src/diagnostics-bridge.d.ts +5 -1
- package/dist/src/diagnostics-bridge.d.ts.map +1 -1
- package/dist/src/diagnostics-bridge.js +16 -2
- package/dist/src/diagnostics-bridge.js.map +1 -1
- package/dist/src/engine.d.ts +426 -6
- package/dist/src/engine.d.ts.map +1 -1
- package/dist/src/engine.js +1951 -101
- package/dist/src/engine.js.map +1 -1
- package/dist/src/errors.d.ts.map +1 -1
- package/dist/src/errors.js +4 -0
- package/dist/src/errors.js.map +1 -1
- package/dist/src/event-accumulators.d.ts +28 -5
- package/dist/src/event-accumulators.d.ts.map +1 -1
- package/dist/src/event-accumulators.js +102 -21
- package/dist/src/event-accumulators.js.map +1 -1
- package/dist/src/events.d.ts +25 -0
- package/dist/src/events.d.ts.map +1 -1
- package/dist/src/events.js +24 -0
- package/dist/src/events.js.map +1 -1
- package/dist/src/handshake/announced.d.ts +6 -0
- package/dist/src/handshake/announced.d.ts.map +1 -1
- package/dist/src/handshake/announced.js +20 -0
- package/dist/src/handshake/announced.js.map +1 -1
- package/dist/src/handshake/descriptor.d.ts +6 -0
- package/dist/src/handshake/descriptor.d.ts.map +1 -1
- package/dist/src/handshake/descriptor.js +1 -0
- package/dist/src/handshake/descriptor.js.map +1 -1
- package/dist/src/index.d.ts +3 -2
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +3 -2
- package/dist/src/index.js.map +1 -1
- package/dist/src/interfaces.d.ts +139 -3
- package/dist/src/interfaces.d.ts.map +1 -1
- package/dist/src/interfaces.js +11 -0
- package/dist/src/interfaces.js.map +1 -1
- package/dist/src/lifecycle-events.d.ts +372 -0
- package/dist/src/lifecycle-events.d.ts.map +1 -0
- package/dist/src/lifecycle-events.js +383 -0
- package/dist/src/lifecycle-events.js.map +1 -0
- package/dist/src/metrics.d.ts +15 -0
- package/dist/src/metrics.d.ts.map +1 -1
- package/dist/src/metrics.js +36 -1
- package/dist/src/metrics.js.map +1 -1
- package/dist/src/otel/durable-metrics.d.ts +9 -0
- package/dist/src/otel/durable-metrics.d.ts.map +1 -1
- package/dist/src/otel/durable-metrics.js +17 -0
- package/dist/src/otel/durable-metrics.js.map +1 -1
- package/dist/src/otel/durable-otel.d.ts +7 -0
- package/dist/src/otel/durable-otel.d.ts.map +1 -1
- package/dist/src/otel/durable-otel.js +69 -3
- package/dist/src/otel/durable-otel.js.map +1 -1
- package/dist/src/run-detail-summary.d.ts +48 -1
- package/dist/src/run-detail-summary.d.ts.map +1 -1
- package/dist/src/run-detail-summary.js +30 -2
- package/dist/src/run-detail-summary.js.map +1 -1
- package/dist/src/run-gateway/interface.d.ts +16 -0
- package/dist/src/run-gateway/interface.d.ts.map +1 -1
- package/dist/src/run-gateway/proxy-run-gateway.d.ts +31 -1
- package/dist/src/run-gateway/proxy-run-gateway.d.ts.map +1 -1
- package/dist/src/run-gateway/proxy-run-gateway.js +59 -7
- package/dist/src/run-gateway/proxy-run-gateway.js.map +1 -1
- package/dist/src/run-gateway/run-request-responder.d.ts +4 -29
- package/dist/src/run-gateway/run-request-responder.d.ts.map +1 -1
- package/dist/src/run-gateway/run-request-responder.js +183 -6
- package/dist/src/run-gateway/run-request-responder.js.map +1 -1
- package/dist/src/run-gateway/store-run-gateway.d.ts +9 -0
- package/dist/src/run-gateway/store-run-gateway.d.ts.map +1 -1
- package/dist/src/run-gateway/store-run-gateway.js +8 -0
- package/dist/src/run-gateway/store-run-gateway.js.map +1 -1
- package/dist/src/run-waiting.d.ts +1 -0
- package/dist/src/run-waiting.d.ts.map +1 -1
- package/dist/src/run-waiting.js +6 -1
- package/dist/src/run-waiting.js.map +1 -1
- package/dist/src/sticky-continuation.d.ts +141 -0
- package/dist/src/sticky-continuation.d.ts.map +1 -0
- package/dist/src/sticky-continuation.js +231 -0
- package/dist/src/sticky-continuation.js.map +1 -0
- package/dist/src/stores/factory.d.ts +6 -0
- package/dist/src/stores/factory.d.ts.map +1 -1
- package/dist/src/stores/factory.js +3 -0
- package/dist/src/stores/factory.js.map +1 -1
- package/dist/src/stores/lucid-pg-hot-path.d.ts +30 -0
- package/dist/src/stores/lucid-pg-hot-path.d.ts.map +1 -0
- package/dist/src/stores/lucid-pg-hot-path.js +67 -0
- package/dist/src/stores/lucid-pg-hot-path.js.map +1 -0
- package/dist/src/stores/lucid.d.ts +30 -1
- package/dist/src/stores/lucid.d.ts.map +1 -1
- package/dist/src/stores/lucid.js +460 -17
- package/dist/src/stores/lucid.js.map +1 -1
- package/dist/src/telescope/dashboard.d.ts.map +1 -1
- package/dist/src/telescope/dashboard.js +50 -0
- package/dist/src/telescope/dashboard.js.map +1 -1
- package/dist/src/telescope/data-providers.d.ts +19 -0
- package/dist/src/telescope/data-providers.d.ts.map +1 -1
- package/dist/src/telescope/data-providers.js +128 -2
- package/dist/src/telescope/data-providers.js.map +1 -1
- package/dist/src/telescope/extension.d.ts.map +1 -1
- package/dist/src/telescope/extension.js +4 -1
- package/dist/src/telescope/extension.js.map +1 -1
- package/dist/src/telescope/index.d.ts +3 -2
- package/dist/src/telescope/index.d.ts.map +1 -1
- package/dist/src/telescope/index.js +2 -1
- package/dist/src/telescope/index.js.map +1 -1
- package/dist/src/telescope/telescope-sdk.d.ts +16 -0
- package/dist/src/telescope/telescope-sdk.d.ts.map +1 -1
- package/dist/src/telescope/watcher.d.ts +66 -0
- package/dist/src/telescope/watcher.d.ts.map +1 -0
- package/dist/src/telescope/watcher.js +185 -0
- package/dist/src/telescope/watcher.js.map +1 -0
- package/dist/src/testing/in-memory-state-store.d.ts +12 -1
- package/dist/src/testing/in-memory-state-store.d.ts.map +1 -1
- package/dist/src/testing/in-memory-state-store.js +109 -1
- package/dist/src/testing/in-memory-state-store.js.map +1 -1
- package/dist/src/testing-kit/harness.d.ts +1 -1
- package/dist/src/testing-kit/harness.d.ts.map +1 -1
- package/dist/src/testing-kit/harness.js +3 -1
- package/dist/src/testing-kit/harness.js.map +1 -1
- package/dist/src/testing-kit/state-store-conformance.d.ts.map +1 -1
- package/dist/src/testing-kit/state-store-conformance.js +258 -0
- package/dist/src/testing-kit/state-store-conformance.js.map +1 -1
- package/dist/src/transport-pool.d.ts +10 -0
- package/dist/src/transport-pool.d.ts.map +1 -1
- package/dist/src/transport-pool.js +23 -2
- package/dist/src/transport-pool.js.map +1 -1
- package/dist/src/transports/bullmq/bullmq-transport.d.ts +41 -1
- package/dist/src/transports/bullmq/bullmq-transport.d.ts.map +1 -1
- package/dist/src/transports/bullmq/bullmq-transport.js +283 -5
- package/dist/src/transports/bullmq/bullmq-transport.js.map +1 -1
- package/dist/src/transports/bullmq/deps.d.ts +12 -1
- package/dist/src/transports/bullmq/deps.d.ts.map +1 -1
- package/dist/src/transports/bullmq/deps.js.map +1 -1
- package/dist/src/transports/bullmq/naming.d.ts +10 -0
- package/dist/src/transports/bullmq/naming.d.ts.map +1 -1
- package/dist/src/transports/bullmq/naming.js +16 -0
- package/dist/src/transports/bullmq/naming.js.map +1 -1
- package/dist/src/transports/db.d.ts +3 -1
- package/dist/src/transports/db.d.ts.map +1 -1
- package/dist/src/transports/db.js +4 -4
- package/dist/src/transports/db.js.map +1 -1
- package/dist/src/transports/queue.d.ts +3 -1
- package/dist/src/transports/queue.d.ts.map +1 -1
- package/dist/src/transports/queue.js +2 -2
- package/dist/src/transports/queue.js.map +1 -1
- package/dist/src/transports/worker-keys.d.ts +2 -1
- package/dist/src/transports/worker-keys.d.ts.map +1 -1
- package/dist/src/transports/worker-keys.js +3 -2
- package/dist/src/transports/worker-keys.js.map +1 -1
- package/dist/src/workflow-ctx.d.ts +7 -0
- package/dist/src/workflow-ctx.d.ts.map +1 -1
- package/dist/src/workflow-ctx.js +90 -1
- package/dist/src/workflow-ctx.js.map +1 -1
- package/dist/src/workflow-discovery.d.ts.map +1 -1
- package/dist/src/workflow-discovery.js +1 -0
- package/dist/src/workflow-discovery.js.map +1 -1
- package/dist/src/workflow-events.d.ts +101 -0
- package/dist/src/workflow-events.d.ts.map +1 -1
- package/dist/src/workflow-events.js +79 -1
- package/dist/src/workflow-events.js.map +1 -1
- package/dist/src/workflow-ref.d.ts +17 -2
- package/dist/src/workflow-ref.d.ts.map +1 -1
- package/dist/src/workflow-ref.js.map +1 -1
- package/package.json +5 -5
- package/skills/durable-cluster/SKILL.md +4 -2
- package/dist/assets/spa/assets/index-6nmSTfT7.css +0 -1
- package/dist/assets/spa/assets/index-Dj0X-IAu.js +0 -16
- package/dist/assets/spa/assets/index-Dj0X-IAu.js.map +0 -1
package/dist/src/engine.js
CHANGED
|
@@ -12,28 +12,30 @@ import { controlPlaneDescriptor, planDispatch, WorkflowBlocked, } from './dispat
|
|
|
12
12
|
import { parseDuration } from './duration.js';
|
|
13
13
|
import { Entities } from './entities.js';
|
|
14
14
|
import { ContinueAsNew, GatherError, NonDeterminismError, RemoteStepTimeout, RemoteWorkflowTimeout, WorkflowSuspended, } from './errors.js';
|
|
15
|
-
import { EventAccumulators } from './event-accumulators.js';
|
|
16
|
-
import { eventMatches, eventMatchOf, eventPrefix } from './events.js';
|
|
15
|
+
import { accumulatorRunId, EventAccumulators, } from './event-accumulators.js';
|
|
16
|
+
import { acceptingSubscription, eventMatches, eventMatchOf, eventPrefix, toEventSubscription, } from './events.js';
|
|
17
17
|
import { childIdsOf, FOLDED_PREFIX, fanOfWaiter, parentRunIdOf, parseStepFanGroup, stepFanGroup, } from './fan-join.js';
|
|
18
|
-
import { aggregateAnnouncements } from './handshake/announced.js';
|
|
18
|
+
import { aggregateAnnouncements, announcementsOf, } from './handshake/announced.js';
|
|
19
19
|
import { buildWorkflowDirectory } from './handshake/directory.js';
|
|
20
20
|
import { isNonEmptyRunScope, TERMINAL_RUN_STATUSES } from './interfaces.js';
|
|
21
|
-
import {
|
|
21
|
+
import { chainPredecessorsOf, coalescedSubscribersOf, deliveredSubscribersOf, LIFECYCLE_HEAL_WINDOW_MS, LIFECYCLE_ORPHAN_MS, LIFECYCLE_OWED_TAG_PREFIX, LIFECYCLE_PENDING_TAG, LIFECYCLE_REGISTRY_HEARTBEAT_MS, LIFECYCLE_REGISTRY_STALE_MS, LIFECYCLE_REGISTRY_TAG, LIFECYCLE_REGISTRY_TTL_MS, LIFECYCLE_REGISTRY_WORKFLOW, LIFECYCLE_TRACE_TAG_PREFIX, LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX, LIFECYCLE_TRIGGERED_TAG_PREFIX, lastRunOfChain, lifecycleCoalescedTag, lifecycleDeliveriesOf, lifecycleEventName, lifecycleEventOf, lifecycleKindOf, lifecycleOwedTag, lifecycleRegistryNamespace, lifecycleRegistryRunId, lifecycleRegistrySubscriptions, lifecycleRunId, lifecycleSourceOf, lifecycleTraceparentOf, lifecycleTriggerFailedTag, nextContinuationId, owedSubscribersOf, parseLifecycleEvent, RUN_WAIT_TOKEN_PREFIX, retentionKeeps, runWaitPrefix, runWaitTargetOf, triggeredRunsOf, } from './lifecycle-events.js';
|
|
22
|
+
import { DEFAULT_NAMESPACE, normalizeNamespace } from './namespaces.js';
|
|
22
23
|
import { asHeartbeat } from './observability-scope.js';
|
|
23
24
|
import { assertRoutes, codeAndDetails, HANDOFF_SEQ_BASE, handoffHandlerId, handoffName, liveHandoff, onErrorRunId, retiredHandoffName, routeFor, } from './on-error.js';
|
|
24
25
|
import { ORIGIN_RESOLVED, originOfCaller } from './origin.js';
|
|
25
26
|
import { breakpointToken, stepId } from './protocol.js';
|
|
26
27
|
import { markRemoteEvent } from './remote-event.js';
|
|
27
28
|
import { RemoteWorkflowExecutor } from './remote-workflow-executor.js';
|
|
28
|
-
import { loadFanMembers, loadRunDetail, } from './run-detail-summary.js';
|
|
29
|
+
import { etagMatches, extendEtag, loadFanMembers, loadRunDetail, } from './run-detail-summary.js';
|
|
29
30
|
import { runFacetsOf } from './run-facets.js';
|
|
30
31
|
import { scanRunValueFacets } from './run-value-facets.js';
|
|
31
|
-
import { listSignalWaitersOfRuns } from './run-waiting.js';
|
|
32
|
+
import { getRunsByIds, listSignalWaitersOfRuns } from './run-waiting.js';
|
|
32
33
|
import { resolveSchedule, scheduleWindow } from './scheduler.js';
|
|
33
34
|
import { ScheduleClient } from './schedules.js';
|
|
34
35
|
import { SINGLETON_RESERVED_TAGS, withoutAdmission, withSingletonTracked, } from './singleton-admission.js';
|
|
35
36
|
import { SingletonGate } from './singleton-gate.js';
|
|
36
37
|
import { validateStandard, validateStandardSync, } from './standard-schema.js';
|
|
38
|
+
import { StickyCache, StickyContinuation } from './sticky-continuation.js';
|
|
37
39
|
import { sanitizeQueueToken, tenantGroup } from './tenant-group.js';
|
|
38
40
|
import { TransportPool } from './transport-pool.js';
|
|
39
41
|
import { workflowAls } from './workflow-als.js';
|
|
@@ -48,6 +50,10 @@ function mergeTags(staticTags, runTags) {
|
|
|
48
50
|
/** A breakpoint checkpoint's `name` is `breakpoint` (or `breakpoint:<label>`). This name — not the
|
|
49
51
|
* reused `signal` kind — is the explicit marker the dashboard and `continue()` detect it by. */
|
|
50
52
|
const BREAKPOINT = 'breakpoint';
|
|
53
|
+
/** Default size of the sticky continuation cache (see {@link WorkflowEngineDeps.stickyRuns}). */
|
|
54
|
+
const DEFAULT_STICKY_RUNS = 2000;
|
|
55
|
+
/** Default idle limit of a parked body in the sticky cache. */
|
|
56
|
+
const DEFAULT_STICKY_IDLE_MS = 300_000;
|
|
51
57
|
const isBreakpoint = (cp) => cp.status === 'pending' && cp.name.startsWith(BREAKPOINT);
|
|
52
58
|
/**
|
|
53
59
|
* Every `RunStatus` from which a turn's outcome (`completed` / `failed` / `suspended` / `blocked`) is a
|
|
@@ -161,6 +167,8 @@ const leaseRefused = new WeakMap();
|
|
|
161
167
|
* parent's next turn.
|
|
162
168
|
*/
|
|
163
169
|
const LOCAL_RUN_CONCURRENCY = 256;
|
|
170
|
+
/** Pending runs one {@link WorkflowEngine.runPending} poll takes at most. */
|
|
171
|
+
const PENDING_POLL_BATCH = 100;
|
|
164
172
|
/** How many dispatched runs this instance takes from the transport's run queue at once. */
|
|
165
173
|
const RUN_CONSUMER_CONCURRENCY = 64;
|
|
166
174
|
/** The most runs handed to {@link Transport.dispatchRuns} in one call. */
|
|
@@ -168,6 +176,8 @@ const RUN_DISPATCH_BATCH = 500;
|
|
|
168
176
|
/** Tries, and the first backoff, for telling a parent that its child finished (see `deliverToParent`). */
|
|
169
177
|
const PARENT_NOTIFY_ATTEMPTS = 4;
|
|
170
178
|
const PARENT_NOTIFY_BACKOFF_MS = 250;
|
|
179
|
+
/** What a lifecycle publish's run lease owner carries: `<instanceId>#lifecycle#<uuid>`. */
|
|
180
|
+
const LIFECYCLE_LEASE_MARK = '#lifecycle#';
|
|
171
181
|
/** How often, at most, and how many times a wake dropped on a held lease looks again (see `recheckAfterLease`). */
|
|
172
182
|
const LEASE_RECHECK_MAX_MS = 2_000;
|
|
173
183
|
const LEASE_RECHECK_SLACK_MS = 100;
|
|
@@ -217,11 +227,6 @@ function shuffle(items) {
|
|
|
217
227
|
function stepGroup(step, runNamespace) {
|
|
218
228
|
return tenantGroup(sanitizeQueueToken(step.name), step.partition ?? runNamespace);
|
|
219
229
|
}
|
|
220
|
-
/** The id for the next continuation of a run: `r` → `r~1` → `r~2` … (stable, traceable lineage). */
|
|
221
|
-
function nextContinuationId(runId) {
|
|
222
|
-
const m = runId.match(/^(.*)~(\d+)$/);
|
|
223
|
-
return m ? `${m[1]}~${Number(m[2]) + 1}` : `${runId}~1`;
|
|
224
|
-
}
|
|
225
230
|
/** True when version `a` is newer than `b` (numeric when both parse as numbers, else natural sort). */
|
|
226
231
|
function isNewerVersion(a, b) {
|
|
227
232
|
const na = Number(a);
|
|
@@ -287,6 +292,16 @@ export class WorkflowEngine {
|
|
|
287
292
|
maxRecoveryAttempts;
|
|
288
293
|
unregisteredWorkflowGraceMs;
|
|
289
294
|
reconcileMs;
|
|
295
|
+
/** Parked workflow bodies kept between turns (see {@link WorkflowEngineDeps.stickyRuns}). */
|
|
296
|
+
sticky;
|
|
297
|
+
/** Pending runs claimed by {@link runPending} for the turn about to run them, as read by the claim. */
|
|
298
|
+
claimedRuns = new Map();
|
|
299
|
+
/** The continuation each turn executing here runs in, by run id, while the turn runs. */
|
|
300
|
+
turnContinuations = new Map();
|
|
301
|
+
/** Whether a transport routes this instance's step results back to it (see `routeReplies`). */
|
|
302
|
+
repliesRouted = false;
|
|
303
|
+
/** The run row stamp (`updatedAt`, epoch ms) a turn here wrote when it settled `suspended`. */
|
|
304
|
+
suspendStamps = new Map();
|
|
290
305
|
heartbeatPersistThrottleMs;
|
|
291
306
|
timerBatchSize;
|
|
292
307
|
eventBufferTtlMs;
|
|
@@ -335,7 +350,7 @@ export class WorkflowEngine {
|
|
|
335
350
|
workflows = new Map();
|
|
336
351
|
/** The newest registered version per workflow name — used to `start` new runs. */
|
|
337
352
|
latest = new Map();
|
|
338
|
-
/** Event name → workflow
|
|
353
|
+
/** Event name → subscribing workflow name → that workflow's subscriptions to it. See `onEvent`. */
|
|
339
354
|
eventTriggers = new Map();
|
|
340
355
|
/** Durable-entity subsystem (registers the `__entity` runner; see `registerEntity`). */
|
|
341
356
|
entities;
|
|
@@ -546,6 +561,9 @@ export class WorkflowEngine {
|
|
|
546
561
|
reconcile === undefined || !Number.isFinite(reconcile) || reconcile < 0
|
|
547
562
|
? 300_000
|
|
548
563
|
: reconcile || undefined;
|
|
564
|
+
const stickyRuns = deps.stickyRuns ?? DEFAULT_STICKY_RUNS;
|
|
565
|
+
const stickyIdle = deps.stickyIdleMs ?? DEFAULT_STICKY_IDLE_MS;
|
|
566
|
+
this.sticky = new StickyCache(Number.isFinite(stickyRuns) ? Math.max(0, Math.floor(stickyRuns)) : DEFAULT_STICKY_RUNS, Number.isFinite(stickyIdle) && stickyIdle >= 0 ? stickyIdle : DEFAULT_STICKY_IDLE_MS);
|
|
549
567
|
// Validated like `timerBatchSize`: a NaN/negative throttle would silently disable the
|
|
550
568
|
// cross-instance liveness salvage (`timeoutMs + NaN` never compares true). `0` stays valid.
|
|
551
569
|
const throttle = deps.heartbeatPersistThrottleMs;
|
|
@@ -675,6 +693,11 @@ export class WorkflowEngine {
|
|
|
675
693
|
async (event) => {
|
|
676
694
|
await this.persistStepEvent(event);
|
|
677
695
|
});
|
|
696
|
+
// Step results come back to the instance holding the body parked on the step, where a transport
|
|
697
|
+
// can route them (see WorkflowEngineDeps.routeReplies).
|
|
698
|
+
if ((deps.routeReplies ?? true) && this.sticky.enabled) {
|
|
699
|
+
this.repliesRouted = this.pool.routeReplies(this.instanceId);
|
|
700
|
+
}
|
|
678
701
|
// Control plane: re-broadcast lifecycle events from OTHER instances to this instance's
|
|
679
702
|
// subscribers (cross-pod live-tail), and act on cancellations issued elsewhere. A broker may
|
|
680
703
|
// echo our own publish back — ignore those (we already handled them locally) to avoid duplicates.
|
|
@@ -1109,10 +1132,11 @@ export class WorkflowEngine {
|
|
|
1109
1132
|
const current = this.latest.get(name);
|
|
1110
1133
|
if (!current || isNewerVersion(version, current.version))
|
|
1111
1134
|
this.latest.set(name, registered);
|
|
1112
|
-
for (const
|
|
1113
|
-
const
|
|
1114
|
-
subscribers.
|
|
1115
|
-
|
|
1135
|
+
for (const entry of opts?.onEvent ?? []) {
|
|
1136
|
+
const subscription = toEventSubscription(entry);
|
|
1137
|
+
const subscribers = this.eventTriggers.get(subscription.event) ?? new Map();
|
|
1138
|
+
subscribers.set(name, [...(subscribers.get(name) ?? []), subscription]);
|
|
1139
|
+
this.eventTriggers.set(subscription.event, subscribers);
|
|
1116
1140
|
}
|
|
1117
1141
|
}
|
|
1118
1142
|
/**
|
|
@@ -1409,17 +1433,1226 @@ export class WorkflowEngine {
|
|
|
1409
1433
|
console.warn(`[adonis-durable] could not start dead-letter workflow ${target} for run ${run.id}: ${err instanceof Error ? err.message : String(err)}`);
|
|
1410
1434
|
}));
|
|
1411
1435
|
}
|
|
1412
|
-
/**
|
|
1413
|
-
*
|
|
1414
|
-
|
|
1436
|
+
/**
|
|
1437
|
+
* Emit a locally-produced lifecycle event: deliver to subscribers AND broadcast it on the control
|
|
1438
|
+
* plane so other instances (e.g. a dashboard pod) can live-tail this run. A `run.completed` or
|
|
1439
|
+
* `run.failed` also wakes the `ctx.waitForRun` waiters on the run and publishes its lifecycle
|
|
1440
|
+
* event (see {@link publishLifecycle}); `lifecycle: false` marks a `run.completed` that is not a
|
|
1441
|
+
* real completion (a continue-as-new handoff), which publishes no lifecycle event.
|
|
1442
|
+
*/
|
|
1443
|
+
emit(event, opts) {
|
|
1415
1444
|
const full = { ...event, at: new Date() };
|
|
1416
1445
|
this.deliver(full);
|
|
1446
|
+
if (full.type === 'run.completed' || full.type === 'run.failed') {
|
|
1447
|
+
this.notifyRunWaiters(full.runId, full.workflow);
|
|
1448
|
+
}
|
|
1449
|
+
if (opts?.lifecycle !== false)
|
|
1450
|
+
this.publishLifecycle(full);
|
|
1417
1451
|
if (!this.controlPlane)
|
|
1418
1452
|
return;
|
|
1419
1453
|
this.eventsToBroadcast.push(full);
|
|
1420
1454
|
if (!this.broadcastingEvents)
|
|
1421
1455
|
void this.broadcastEvents();
|
|
1422
1456
|
}
|
|
1457
|
+
/**
|
|
1458
|
+
* Start every workflow subscribed to `name` that takes `payload` (see {@link acceptingSubscription}),
|
|
1459
|
+
* each under `runIdOf(workflow)` and with its subscription's `input` applied; only the ones in
|
|
1460
|
+
* `only` when given. Returns the subscribers reached, with the run each started (none when the
|
|
1461
|
+
* payload went into its debounce/batch accumulator) and whether that run is new, and the ones that
|
|
1462
|
+
* could not be started, with why.
|
|
1463
|
+
*
|
|
1464
|
+
* `remote` names subscribers known only from the fleet's announcements (see
|
|
1465
|
+
* {@link announcedRemoteSubscribers}): each is started by name with the payload as its input, with
|
|
1466
|
+
* no `where` or `input` mapping, unless it is also registered here.
|
|
1467
|
+
*/
|
|
1468
|
+
async startSubscribers(name, payload, runIdOf, only, remote = []) {
|
|
1469
|
+
const reached = [];
|
|
1470
|
+
const failed = [];
|
|
1471
|
+
for (const [workflow, subscriptions] of this.eventTriggers.get(name) ?? []) {
|
|
1472
|
+
if (only && !only.has(workflow))
|
|
1473
|
+
continue;
|
|
1474
|
+
const { accepted, errors } = acceptingSubscription(subscriptions, payload);
|
|
1475
|
+
if (!accepted) {
|
|
1476
|
+
if (errors.length > 0)
|
|
1477
|
+
failed.push({ workflow, error: errors[0] });
|
|
1478
|
+
continue;
|
|
1479
|
+
}
|
|
1480
|
+
// A subscriber that rejects the payload (validateInput / inputSchema) must not block the
|
|
1481
|
+
// others or the waiters: its run never starts, and the caller decides how to report it.
|
|
1482
|
+
try {
|
|
1483
|
+
const input = accepted.input ? accepted.input(payload) : payload;
|
|
1484
|
+
const batch = this.latest.get(workflow)?.eventBatch;
|
|
1485
|
+
if (batch) {
|
|
1486
|
+
// Coalesce: route the event into the workflow's accumulator (one long-lived run per
|
|
1487
|
+
// target and key that debounces/batches and then starts the target), named by the run
|
|
1488
|
+
// id it would start, so a second route of the same event is dropped there.
|
|
1489
|
+
await this.accumulators.route(workflow, batch, input, runIdOf(workflow));
|
|
1490
|
+
reached.push({ workflow });
|
|
1491
|
+
}
|
|
1492
|
+
else {
|
|
1493
|
+
const runId = runIdOf(workflow);
|
|
1494
|
+
const existed = (await this.store.getRun(runId)) !== null;
|
|
1495
|
+
await this.start(workflow, input, runId);
|
|
1496
|
+
reached.push({ workflow, runId, created: !existed });
|
|
1497
|
+
}
|
|
1498
|
+
}
|
|
1499
|
+
catch (error) {
|
|
1500
|
+
failed.push({ workflow, error });
|
|
1501
|
+
}
|
|
1502
|
+
}
|
|
1503
|
+
const local = this.eventTriggers.get(name);
|
|
1504
|
+
for (const workflow of remote) {
|
|
1505
|
+
if (local?.has(workflow) || (only && !only.has(workflow)))
|
|
1506
|
+
continue;
|
|
1507
|
+
try {
|
|
1508
|
+
const runId = runIdOf(workflow);
|
|
1509
|
+
const existed = (await this.store.getRun(runId)) !== null;
|
|
1510
|
+
await this.start(workflow, payload, runId);
|
|
1511
|
+
reached.push({ workflow, runId, created: !existed });
|
|
1512
|
+
}
|
|
1513
|
+
catch (error) {
|
|
1514
|
+
failed.push({ workflow, error });
|
|
1515
|
+
}
|
|
1516
|
+
}
|
|
1517
|
+
return { reached, failed };
|
|
1518
|
+
}
|
|
1519
|
+
/** The workflows registered on this instance that subscribe to `event`. */
|
|
1520
|
+
localSubscribers(event) {
|
|
1521
|
+
return [...(this.eventTriggers.get(event)?.keys() ?? [])];
|
|
1522
|
+
}
|
|
1523
|
+
/**
|
|
1524
|
+
* Whether this engine's own subscribers are owed the lifecycle events of `namespace`'s runs: always
|
|
1525
|
+
* for an engine with no namespace (its sweep reads every namespace's runs), else only its own
|
|
1526
|
+
* namespace's. `undefined` asks without a namespace and is always true.
|
|
1527
|
+
*/
|
|
1528
|
+
servesNamespace(namespace) {
|
|
1529
|
+
return (namespace === undefined ||
|
|
1530
|
+
this.namespace === undefined ||
|
|
1531
|
+
normalizeNamespace(this.namespace) === normalizeNamespace(namespace));
|
|
1532
|
+
}
|
|
1533
|
+
/** The workflows registered here that subscribe to `event` and serve `namespace`'s runs. */
|
|
1534
|
+
localSubscribersFor(event, namespace) {
|
|
1535
|
+
return this.servesNamespace(namespace) ? this.localSubscribers(event) : [];
|
|
1536
|
+
}
|
|
1537
|
+
/**
|
|
1538
|
+
* The subscribers of `event` this engine starts by name although it has no `onEvent` for them:
|
|
1539
|
+
* announced only by non-Node workers (a Python `@worker.workflow(..., on_workflow_completed=...)`),
|
|
1540
|
+
* and reachable from here, either registered (`registerRemote`) or served by convention on a live
|
|
1541
|
+
* worker group of the same name. A subscriber some Node engine announces is left to that engine,
|
|
1542
|
+
* which applies its own `where` and `input`; one this engine cannot reach stays owed, for an
|
|
1543
|
+
* instance that can. Read from the snapshot {@link announcedSubscribers} keeps. With `namespace`,
|
|
1544
|
+
* only the subscriptions announced for that namespace's runs (see {@link announcedFor}).
|
|
1545
|
+
*/
|
|
1546
|
+
async announcedRemoteSubscribers(event, namespace) {
|
|
1547
|
+
const announced = this.announcedFor(event, namespace);
|
|
1548
|
+
if (announced.size === 0)
|
|
1549
|
+
return [];
|
|
1550
|
+
const local = new Set(this.localSubscribers(event));
|
|
1551
|
+
const byEngine = this.announcedFor(event, namespace, { node: true });
|
|
1552
|
+
const startable = [];
|
|
1553
|
+
for (const workflow of [...announced].sort()) {
|
|
1554
|
+
if (local.has(workflow) || byEngine?.has(workflow) || workflow.startsWith('__'))
|
|
1555
|
+
continue;
|
|
1556
|
+
const registered = this.latest.get(workflow);
|
|
1557
|
+
if (registered && !registered.convention) {
|
|
1558
|
+
startable.push(workflow);
|
|
1559
|
+
continue;
|
|
1560
|
+
}
|
|
1561
|
+
try {
|
|
1562
|
+
if (await this.pool.transportWithLiveGroup(conventionGroup(workflow, this.namespace))) {
|
|
1563
|
+
startable.push(workflow);
|
|
1564
|
+
}
|
|
1565
|
+
}
|
|
1566
|
+
catch {
|
|
1567
|
+
// An unreadable fleet leaves the subscriber owed, for the next sweep.
|
|
1568
|
+
}
|
|
1569
|
+
}
|
|
1570
|
+
return startable;
|
|
1571
|
+
}
|
|
1572
|
+
/**
|
|
1573
|
+
* The subscribers of `event` this engine can deliver to for a run of `namespace`: the ones
|
|
1574
|
+
* registered here that serve it, and the announced ones it starts by name (see
|
|
1575
|
+
* {@link announcedRemoteSubscribers}).
|
|
1576
|
+
*/
|
|
1577
|
+
async deliverableSubscribers(event, namespace) {
|
|
1578
|
+
const local = new Set(this.localSubscribersFor(event, namespace));
|
|
1579
|
+
const remote = await this.announcedRemoteSubscribers(event, namespace);
|
|
1580
|
+
return { local, remote, all: new Set([...local, ...remote]) };
|
|
1581
|
+
}
|
|
1582
|
+
/**
|
|
1583
|
+
* Every workflow a run of `workflow` settling as `kind` owes its lifecycle event to: the ones
|
|
1584
|
+
* registered here plus the ones the fleet announces (from a snapshot of the lifecycle subscription
|
|
1585
|
+
* registry and the worker descriptors at most 5s old), sorted.
|
|
1586
|
+
*
|
|
1587
|
+
* With `namespace` (the settled run's), only the subscribers that serve that namespace's runs: the
|
|
1588
|
+
* ones registered here when this engine has no namespace or has that one, the registry rows of
|
|
1589
|
+
* engines in that namespace or with none, and the worker descriptors serving it. What the
|
|
1590
|
+
* terminal status write marks the run owed to.
|
|
1591
|
+
*/
|
|
1592
|
+
lifecycleSubscribers(workflow, kind, namespace) {
|
|
1593
|
+
if (workflow.startsWith('__'))
|
|
1594
|
+
return [];
|
|
1595
|
+
const event = lifecycleEventName(workflow, kind);
|
|
1596
|
+
return [
|
|
1597
|
+
...new Set([
|
|
1598
|
+
...this.localSubscribersFor(event, namespace),
|
|
1599
|
+
...this.announcedSubscribers(event, namespace),
|
|
1600
|
+
]),
|
|
1601
|
+
].sort();
|
|
1602
|
+
}
|
|
1603
|
+
/**
|
|
1604
|
+
* Who starts whom through lifecycle events, for `workflow`: the workflows that start when it
|
|
1605
|
+
* settles, and the workflows whose settling starts it. Built from this engine's registry and the
|
|
1606
|
+
* subscriptions live workers announce (read again when the snapshot is older than 5s); a failed
|
|
1607
|
+
* fleet read leaves the local registry alone. What the dashboard's workflow page shows.
|
|
1608
|
+
*
|
|
1609
|
+
* With `namespace`, only the subscriptions of that namespace: the ones announced by registry runs
|
|
1610
|
+
* and worker descriptors serving it, and this engine's own when it is that namespace's engine (an
|
|
1611
|
+
* engine with no namespace is `default`'s). What a tenant is shown.
|
|
1612
|
+
*/
|
|
1613
|
+
async workflowLifecycle(workflow, opts) {
|
|
1614
|
+
await this.freshAnnouncedSubscriptions();
|
|
1615
|
+
const snapshot = this.announcedSubscriptionsSnapshot;
|
|
1616
|
+
const scope = opts?.namespace;
|
|
1617
|
+
let fleet = snapshot?.byEvent ?? new Map();
|
|
1618
|
+
if (scope !== undefined) {
|
|
1619
|
+
fleet = new Map();
|
|
1620
|
+
for (const entry of snapshot?.entries ?? []) {
|
|
1621
|
+
if ((entry.namespace ?? DEFAULT_NAMESPACE) !== scope)
|
|
1622
|
+
continue;
|
|
1623
|
+
const subscribers = fleet.get(entry.event) ?? new Set();
|
|
1624
|
+
subscribers.add(entry.workflow);
|
|
1625
|
+
fleet.set(entry.event, subscribers);
|
|
1626
|
+
}
|
|
1627
|
+
}
|
|
1628
|
+
const triggers = scope === undefined || (this.namespace || 'default') === scope
|
|
1629
|
+
? this.eventTriggers
|
|
1630
|
+
: new Map();
|
|
1631
|
+
const startsOnSettle = new Map();
|
|
1632
|
+
const startedBy = new Map();
|
|
1633
|
+
const link = (into, other, kind, local) => {
|
|
1634
|
+
const key = `${other}\u0000${kind}`;
|
|
1635
|
+
const entry = into.get(key) ?? {
|
|
1636
|
+
workflow: other,
|
|
1637
|
+
kind,
|
|
1638
|
+
local: false,
|
|
1639
|
+
announced: false,
|
|
1640
|
+
where: false,
|
|
1641
|
+
input: false,
|
|
1642
|
+
};
|
|
1643
|
+
if (local) {
|
|
1644
|
+
entry.local = true;
|
|
1645
|
+
entry.where ||= local.subscriptions.some((sub) => sub.where !== undefined);
|
|
1646
|
+
entry.input ||= local.subscriptions.some((sub) => sub.input !== undefined);
|
|
1647
|
+
const batch = this.latest.get(local.subscriber)?.eventBatch;
|
|
1648
|
+
if (batch)
|
|
1649
|
+
entry.coalesce = { mode: batch.mode, keyed: batch.key !== undefined };
|
|
1650
|
+
}
|
|
1651
|
+
else {
|
|
1652
|
+
entry.announced = true;
|
|
1653
|
+
}
|
|
1654
|
+
into.set(key, entry);
|
|
1655
|
+
};
|
|
1656
|
+
if (!workflow.startsWith('__')) {
|
|
1657
|
+
for (const kind of ['completed', 'failed']) {
|
|
1658
|
+
const event = lifecycleEventName(workflow, kind);
|
|
1659
|
+
for (const [subscriber, subscriptions] of triggers.get(event) ?? []) {
|
|
1660
|
+
link(startsOnSettle, subscriber, kind, { subscriber, subscriptions });
|
|
1661
|
+
}
|
|
1662
|
+
for (const subscriber of fleet.get(event) ?? []) {
|
|
1663
|
+
link(startsOnSettle, subscriber, kind, undefined);
|
|
1664
|
+
}
|
|
1665
|
+
}
|
|
1666
|
+
}
|
|
1667
|
+
for (const [event, subscribers] of triggers) {
|
|
1668
|
+
const source = parseLifecycleEvent(event);
|
|
1669
|
+
const subscriptions = subscribers.get(workflow);
|
|
1670
|
+
if (source && subscriptions) {
|
|
1671
|
+
link(startedBy, source.workflow, source.kind, { subscriber: workflow, subscriptions });
|
|
1672
|
+
}
|
|
1673
|
+
}
|
|
1674
|
+
for (const [event, subscribers] of fleet) {
|
|
1675
|
+
const source = parseLifecycleEvent(event);
|
|
1676
|
+
if (source && subscribers.has(workflow)) {
|
|
1677
|
+
link(startedBy, source.workflow, source.kind, undefined);
|
|
1678
|
+
}
|
|
1679
|
+
}
|
|
1680
|
+
const sorted = (links) => [...links.values()].sort((a, b) => a.workflow.localeCompare(b.workflow) || a.kind.localeCompare(b.kind));
|
|
1681
|
+
return { workflow, startsOnSettle: sorted(startsOnSettle), startedBy: sorted(startedBy) };
|
|
1682
|
+
}
|
|
1683
|
+
/**
|
|
1684
|
+
* `patch` (a terminal status write of `run`) plus the {@link LIFECYCLE_PENDING_TAG} marker and one
|
|
1685
|
+
* {@link LIFECYCLE_OWED_TAG_PREFIX} tag per subscriber of that ending, and the run's span as a
|
|
1686
|
+
* {@link LIFECYCLE_TRACE_TAG_PREFIX} tag when a run trace provider knows it. Written in the same
|
|
1687
|
+
* update as the status, so a run can never be settled without a record of whom its lifecycle event
|
|
1688
|
+
* is owed to.
|
|
1689
|
+
*
|
|
1690
|
+
* While this instance has no successful read of the fleet's announced subscriptions (none yet, or
|
|
1691
|
+
* the last one failed), it cannot know who else subscribes, so the marker is the pending tag alone:
|
|
1692
|
+
* the run's owed set is unresolved, and the publish that next reads the fleet resolves it (see
|
|
1693
|
+
* {@link publishSettled}). Never waits on that read.
|
|
1694
|
+
*/
|
|
1695
|
+
withLifecycleMarker(run, patch) {
|
|
1696
|
+
const kind = lifecycleKindOf(patch.status);
|
|
1697
|
+
if (!kind || run.workflow.startsWith('__'))
|
|
1698
|
+
return patch;
|
|
1699
|
+
const subscribers = this.lifecycleSubscribers(run.workflow, kind, normalizeNamespace(run.namespace));
|
|
1700
|
+
const unresolved = !this.announcedSubscriptionsSnapshot?.ok;
|
|
1701
|
+
if (subscribers.length === 0 && !unresolved)
|
|
1702
|
+
return patch;
|
|
1703
|
+
const tags = patch.tags ?? run.tags ?? [];
|
|
1704
|
+
let traceparent;
|
|
1705
|
+
try {
|
|
1706
|
+
traceparent = this.runTraceparent?.(run.id);
|
|
1707
|
+
}
|
|
1708
|
+
catch {
|
|
1709
|
+
traceparent = undefined;
|
|
1710
|
+
}
|
|
1711
|
+
const recordsTrace = traceparent !== undefined &&
|
|
1712
|
+
subscribers.length > 0 &&
|
|
1713
|
+
!tags.some((t) => t.startsWith(LIFECYCLE_TRACE_TAG_PREFIX));
|
|
1714
|
+
const marker = [
|
|
1715
|
+
LIFECYCLE_PENDING_TAG,
|
|
1716
|
+
...(unresolved ? [] : subscribers.map(lifecycleOwedTag)),
|
|
1717
|
+
...(recordsTrace ? [`${LIFECYCLE_TRACE_TAG_PREFIX}${traceparent}`] : []),
|
|
1718
|
+
];
|
|
1719
|
+
if (marker.every((tag) => tags.includes(tag)))
|
|
1720
|
+
return patch;
|
|
1721
|
+
return { ...patch, tags: [...new Set([...tags, ...marker])] };
|
|
1722
|
+
}
|
|
1723
|
+
/** Answers the traceparent of a run's span in this process; see {@link setRunTraceparent}. */
|
|
1724
|
+
runTraceparent;
|
|
1725
|
+
/**
|
|
1726
|
+
* Install the function that answers the W3C `traceparent` of a run's span in this process, or
|
|
1727
|
+
* `undefined` to remove it. `attachDurableOtel` installs one. With it, a run some workflow
|
|
1728
|
+
* subscribes to records its traceparent (a {@link LIFECYCLE_TRACE_TAG_PREFIX} tag) with its first
|
|
1729
|
+
* suspend (see {@link withRunTrace}), or else with its terminal write, and the `run.started` of a
|
|
1730
|
+
* run a lifecycle event started carries its source run's as `sourceTraceparent`, so the
|
|
1731
|
+
* subscriber's trace links to the source's whichever process ran either.
|
|
1732
|
+
*/
|
|
1733
|
+
setRunTraceparent(provider) {
|
|
1734
|
+
this.runTraceparent = provider;
|
|
1735
|
+
}
|
|
1736
|
+
/**
|
|
1737
|
+
* `patch` (a suspend write of `run`) plus the run's {@link LIFECYCLE_TRACE_TAG_PREFIX} tag, when a
|
|
1738
|
+
* run trace provider is installed (see {@link setRunTraceparent}), some workflow subscribes to the
|
|
1739
|
+
* run's ending, the provider answers for the run and the run does not carry the tag yet. The first
|
|
1740
|
+
* turn's suspend records the run's root span this way, so a run that settles in a process that
|
|
1741
|
+
* never saw its first turn still links its subscribers' traces; a workflow nobody subscribes to
|
|
1742
|
+
* gets no tag and no extra read or write. The tags are read again before they are written, so a
|
|
1743
|
+
* tag the turn added is kept.
|
|
1744
|
+
*/
|
|
1745
|
+
async withRunTrace(run, patch) {
|
|
1746
|
+
const provider = this.runTraceparent;
|
|
1747
|
+
if (!provider || run.workflow.startsWith('__'))
|
|
1748
|
+
return patch;
|
|
1749
|
+
if ((run.tags ?? []).some((t) => t.startsWith(LIFECYCLE_TRACE_TAG_PREFIX)))
|
|
1750
|
+
return patch;
|
|
1751
|
+
const namespace = normalizeNamespace(run.namespace);
|
|
1752
|
+
if (this.lifecycleSubscribers(run.workflow, 'completed', namespace).length === 0 &&
|
|
1753
|
+
this.lifecycleSubscribers(run.workflow, 'failed', namespace).length === 0) {
|
|
1754
|
+
return patch;
|
|
1755
|
+
}
|
|
1756
|
+
let traceparent;
|
|
1757
|
+
try {
|
|
1758
|
+
traceparent = provider(run.id);
|
|
1759
|
+
}
|
|
1760
|
+
catch {
|
|
1761
|
+
return patch;
|
|
1762
|
+
}
|
|
1763
|
+
if (!traceparent)
|
|
1764
|
+
return patch;
|
|
1765
|
+
const tags = (await this.store.getRun(run.id))?.tags ?? run.tags ?? [];
|
|
1766
|
+
if (tags.some((t) => t.startsWith(LIFECYCLE_TRACE_TAG_PREFIX)))
|
|
1767
|
+
return patch;
|
|
1768
|
+
return { ...patch, tags: [...tags, `${LIFECYCLE_TRACE_TAG_PREFIX}${traceparent}`] };
|
|
1769
|
+
}
|
|
1770
|
+
/**
|
|
1771
|
+
* `{ sourceTraceparent }` for the `run.started` of `run`, when a lifecycle event started it and a
|
|
1772
|
+
* run trace provider is installed: the source run's recorded traceparent, else this process's.
|
|
1773
|
+
*/
|
|
1774
|
+
async sourceTraceOf(run) {
|
|
1775
|
+
const provider = this.runTraceparent;
|
|
1776
|
+
if (!provider)
|
|
1777
|
+
return {};
|
|
1778
|
+
const source = lifecycleSourceOf(run);
|
|
1779
|
+
if (!source)
|
|
1780
|
+
return {};
|
|
1781
|
+
try {
|
|
1782
|
+
const recorded = lifecycleTraceparentOf((await this.store.getRun(source.runId))?.tags);
|
|
1783
|
+
const traceparent = recorded ?? provider(source.runId);
|
|
1784
|
+
return traceparent ? { sourceTraceparent: traceparent } : {};
|
|
1785
|
+
}
|
|
1786
|
+
catch {
|
|
1787
|
+
return {};
|
|
1788
|
+
}
|
|
1789
|
+
}
|
|
1790
|
+
/** How many registry rows one page of a read of the lifecycle subscription registry lists. */
|
|
1791
|
+
static LIFECYCLE_REGISTRY_PAGE_SIZE = 1000;
|
|
1792
|
+
/** How long a read of the subscriptions the fleet announces is reused. */
|
|
1793
|
+
static ANNOUNCED_SUBSCRIPTIONS_TTL_MS = 5_000;
|
|
1794
|
+
/** The last read of the fleet's announced subscriptions; see {@link announcedSubscribers}. */
|
|
1795
|
+
announcedSubscriptionsSnapshot;
|
|
1796
|
+
announcedSubscriptionsRead;
|
|
1797
|
+
/**
|
|
1798
|
+
* The workflows the fleet announces as subscribers of `event`: the live rows of the lifecycle
|
|
1799
|
+
* subscription registry (see {@link LIFECYCLE_REGISTRY_WORKFLOW}) and the live worker descriptors'
|
|
1800
|
+
* `registrations[].subscribes`, from a snapshot refreshed in the background once it is older than
|
|
1801
|
+
* {@link ANNOUNCED_SUBSCRIPTIONS_TTL_MS}. Synchronous, so a terminal status write never waits on
|
|
1802
|
+
* the fleet; before the first read completes it answers with nothing.
|
|
1803
|
+
*/
|
|
1804
|
+
announcedSubscribers(event, namespace) {
|
|
1805
|
+
const snapshot = this.announcedSubscriptionsSnapshot;
|
|
1806
|
+
if (!snapshot || this.clock() - snapshot.at > WorkflowEngine.ANNOUNCED_SUBSCRIPTIONS_TTL_MS) {
|
|
1807
|
+
void this.refreshAnnouncedSubscriptions();
|
|
1808
|
+
}
|
|
1809
|
+
return this.announcedFor(event, namespace);
|
|
1810
|
+
}
|
|
1811
|
+
/**
|
|
1812
|
+
* The subscribers of `event` in the current snapshot that serve `namespace`'s runs: announced for
|
|
1813
|
+
* that namespace, or by an engine with no namespace (see {@link LifecycleRegistryEntry}). Every
|
|
1814
|
+
* announced subscriber when `namespace` is undefined. `node` keeps only the ones Node engines
|
|
1815
|
+
* announce (registry rows and Node worker descriptors), which start their own subscribers.
|
|
1816
|
+
*/
|
|
1817
|
+
announcedFor(event, namespace, opts) {
|
|
1818
|
+
const snapshot = this.announcedSubscriptionsSnapshot;
|
|
1819
|
+
if (!snapshot?.byEvent.has(event))
|
|
1820
|
+
return new Set();
|
|
1821
|
+
const scope = namespace === undefined ? undefined : normalizeNamespace(namespace);
|
|
1822
|
+
const out = new Set();
|
|
1823
|
+
for (const entry of snapshot.entries) {
|
|
1824
|
+
if (entry.event !== event || (opts?.node && !entry.node))
|
|
1825
|
+
continue;
|
|
1826
|
+
if (scope !== undefined && entry.namespace !== null && entry.namespace !== scope)
|
|
1827
|
+
continue;
|
|
1828
|
+
out.add(entry.workflow);
|
|
1829
|
+
}
|
|
1830
|
+
return out;
|
|
1831
|
+
}
|
|
1832
|
+
/**
|
|
1833
|
+
* Read the fleet's announced subscriptions into the snapshot: the registry rows whose heartbeat
|
|
1834
|
+
* is at most {@link LIFECYCLE_REGISTRY_TTL_MS} old (a range of the store's `created_at` index, read
|
|
1835
|
+
* page by page until a short page, so a large fleet is never cut off), and
|
|
1836
|
+
* the worker descriptors on a transport that can read them fleet-wide. Never throws: a failed read
|
|
1837
|
+
* of either leaves an empty snapshot marked not `ok`, which counts as the local registry only and
|
|
1838
|
+
* never orphans a subscriber. Concurrent callers share one read.
|
|
1839
|
+
*/
|
|
1840
|
+
refreshAnnouncedSubscriptions() {
|
|
1841
|
+
this.announcedSubscriptionsRead ??= (async () => {
|
|
1842
|
+
const at = this.clock();
|
|
1843
|
+
const byEvent = new Map();
|
|
1844
|
+
const entries = [];
|
|
1845
|
+
const add = (event, workflow, node, namespace) => {
|
|
1846
|
+
entries.push({ event, workflow, namespace, node });
|
|
1847
|
+
const subscribers = byEvent.get(event) ?? new Set();
|
|
1848
|
+
subscribers.add(workflow);
|
|
1849
|
+
byEvent.set(event, subscribers);
|
|
1850
|
+
};
|
|
1851
|
+
const readRegistry = async () => {
|
|
1852
|
+
const size = WorkflowEngine.LIFECYCLE_REGISTRY_PAGE_SIZE;
|
|
1853
|
+
const out = [];
|
|
1854
|
+
for (let page = 1;; page += 1) {
|
|
1855
|
+
const rows = await asHeartbeat(() => this.store.listRuns({
|
|
1856
|
+
workflow: LIFECYCLE_REGISTRY_WORKFLOW,
|
|
1857
|
+
createdAfter: at - LIFECYCLE_REGISTRY_TTL_MS,
|
|
1858
|
+
// A namespaced engine also reads `default`, where an engine with no namespace writes
|
|
1859
|
+
// its row: those subscribers are owed this namespace's runs too.
|
|
1860
|
+
...(this.namespace !== undefined
|
|
1861
|
+
? { namespaces: [...new Set([this.namespace, 'default'])] }
|
|
1862
|
+
: {}),
|
|
1863
|
+
page,
|
|
1864
|
+
size,
|
|
1865
|
+
}));
|
|
1866
|
+
out.push(...rows);
|
|
1867
|
+
if (rows.length < size)
|
|
1868
|
+
return out;
|
|
1869
|
+
}
|
|
1870
|
+
};
|
|
1871
|
+
try {
|
|
1872
|
+
const [descriptors, rows] = await Promise.all([
|
|
1873
|
+
this.pool.introspectsFleet ? this.pool.readAllWorkerDescriptors() : [],
|
|
1874
|
+
readRegistry(),
|
|
1875
|
+
]);
|
|
1876
|
+
for (const row of rows) {
|
|
1877
|
+
if (row.workflow !== LIFECYCLE_REGISTRY_WORKFLOW)
|
|
1878
|
+
continue;
|
|
1879
|
+
if (at - row.updatedAt.getTime() > LIFECYCLE_REGISTRY_TTL_MS)
|
|
1880
|
+
continue;
|
|
1881
|
+
const namespace = lifecycleRegistryNamespace(row.input, row.namespace);
|
|
1882
|
+
for (const { workflow, events } of lifecycleRegistrySubscriptions(row.input)) {
|
|
1883
|
+
for (const event of events)
|
|
1884
|
+
add(event, workflow, true, namespace);
|
|
1885
|
+
}
|
|
1886
|
+
}
|
|
1887
|
+
for (const descriptor of descriptors) {
|
|
1888
|
+
for (const announced of announcementsOf(descriptor)) {
|
|
1889
|
+
const namespace = servedNamespace(descriptor.namespace, announced.group);
|
|
1890
|
+
for (const event of announced.subscribes ?? []) {
|
|
1891
|
+
add(event, announced.name, descriptor.runtime === 'node', namespace);
|
|
1892
|
+
}
|
|
1893
|
+
}
|
|
1894
|
+
}
|
|
1895
|
+
this.announcedSubscriptionsSnapshot = { at, ok: true, byEvent, entries };
|
|
1896
|
+
}
|
|
1897
|
+
catch {
|
|
1898
|
+
this.announcedSubscriptionsSnapshot = { at, ok: false, byEvent: new Map(), entries: [] };
|
|
1899
|
+
}
|
|
1900
|
+
})().finally(() => {
|
|
1901
|
+
this.announcedSubscriptionsRead = undefined;
|
|
1902
|
+
});
|
|
1903
|
+
return this.announcedSubscriptionsRead;
|
|
1904
|
+
}
|
|
1905
|
+
/** Read the fleet's announced subscriptions again when the snapshot is older than the TTL. */
|
|
1906
|
+
async freshAnnouncedSubscriptions() {
|
|
1907
|
+
const snapshot = this.announcedSubscriptionsSnapshot;
|
|
1908
|
+
if (!snapshot || this.clock() - snapshot.at > WorkflowEngine.ANNOUNCED_SUBSCRIPTIONS_TTL_MS) {
|
|
1909
|
+
await this.refreshAnnouncedSubscriptions();
|
|
1910
|
+
}
|
|
1911
|
+
}
|
|
1912
|
+
/** When this engine last wrote its registry row; see {@link heartbeatLifecycleRegistry}. */
|
|
1913
|
+
lifecycleRegistryBeatAt;
|
|
1914
|
+
/** When this engine last deleted stale registry rows; undefined before its first pass. */
|
|
1915
|
+
lifecycleRegistryCleanedAt;
|
|
1916
|
+
/** How often an engine with a registry row looks for stale rows to delete. */
|
|
1917
|
+
static LIFECYCLE_REGISTRY_CLEANUP_MS = 60_000;
|
|
1918
|
+
/**
|
|
1919
|
+
* This engine's lifecycle registry entry: each workflow registered here that subscribes to an
|
|
1920
|
+
* event, with the events it subscribes to. Undefined when none does.
|
|
1921
|
+
*/
|
|
1922
|
+
lifecycleRegistryEntry() {
|
|
1923
|
+
const subscribes = new Map();
|
|
1924
|
+
for (const [event, subscribers] of this.eventTriggers) {
|
|
1925
|
+
for (const workflow of subscribers.keys()) {
|
|
1926
|
+
if (workflow.startsWith('__'))
|
|
1927
|
+
continue;
|
|
1928
|
+
const events = subscribes.get(workflow) ?? new Set();
|
|
1929
|
+
events.add(event);
|
|
1930
|
+
subscribes.set(workflow, events);
|
|
1931
|
+
}
|
|
1932
|
+
}
|
|
1933
|
+
if (subscribes.size === 0)
|
|
1934
|
+
return undefined;
|
|
1935
|
+
return {
|
|
1936
|
+
instanceId: this.instanceId,
|
|
1937
|
+
runtime: 'node',
|
|
1938
|
+
engineNamespace: this.namespace ?? null,
|
|
1939
|
+
subscriptions: [...subscribes.keys()].sort().map((workflow) => ({
|
|
1940
|
+
workflow,
|
|
1941
|
+
events: [...(subscribes.get(workflow) ?? [])].sort(),
|
|
1942
|
+
})),
|
|
1943
|
+
};
|
|
1944
|
+
}
|
|
1945
|
+
/**
|
|
1946
|
+
* Write this engine's row of the lifecycle subscription registry (see
|
|
1947
|
+
* {@link LIFECYCLE_REGISTRY_WORKFLOW}), at most once per {@link LIFECYCLE_REGISTRY_HEARTBEAT_MS}:
|
|
1948
|
+
* an update of the existing row, or a create when it is missing (first beat, or deleted as stale
|
|
1949
|
+
* after a long pause). Once a minute it also deletes rows whose heartbeat stopped
|
|
1950
|
+
* {@link LIFECYCLE_REGISTRY_STALE_MS} ago: the ones that went stale in the last
|
|
1951
|
+
* {@link LIFECYCLE_REGISTRY_STALE_MS}, a bounded range of the `created_at` index, and on its first
|
|
1952
|
+
* pass every stale row, for the ones left while no engine ran. The row is created tagged
|
|
1953
|
+
* {@link LIFECYCLE_REGISTRY_TAG}, which retention keeps. A no-op for an engine without
|
|
1954
|
+
* subscriptions. Its store calls are marked as heartbeats for observability tools. Never throws.
|
|
1955
|
+
*/
|
|
1956
|
+
async heartbeatLifecycleRegistry() {
|
|
1957
|
+
const nowMs = this.clock();
|
|
1958
|
+
if (this.lifecycleRegistryBeatAt !== undefined &&
|
|
1959
|
+
nowMs - this.lifecycleRegistryBeatAt < LIFECYCLE_REGISTRY_HEARTBEAT_MS) {
|
|
1960
|
+
return;
|
|
1961
|
+
}
|
|
1962
|
+
const entry = this.lifecycleRegistryEntry();
|
|
1963
|
+
if (!entry)
|
|
1964
|
+
return;
|
|
1965
|
+
this.lifecycleRegistryBeatAt = nowMs;
|
|
1966
|
+
const id = lifecycleRegistryRunId(this.instanceId);
|
|
1967
|
+
const at = new Date(nowMs);
|
|
1968
|
+
try {
|
|
1969
|
+
await asHeartbeat(async () => {
|
|
1970
|
+
const beat = { input: entry, createdAt: at, updatedAt: at };
|
|
1971
|
+
if (await this.store.updateRunIf(id, ['completed'], beat))
|
|
1972
|
+
return;
|
|
1973
|
+
try {
|
|
1974
|
+
await this.store.createRun({
|
|
1975
|
+
id,
|
|
1976
|
+
workflow: LIFECYCLE_REGISTRY_WORKFLOW,
|
|
1977
|
+
workflowVersion: '1',
|
|
1978
|
+
status: 'completed',
|
|
1979
|
+
input: entry,
|
|
1980
|
+
tags: [LIFECYCLE_REGISTRY_TAG],
|
|
1981
|
+
...(this.namespace !== undefined ? { namespace: this.namespace } : {}),
|
|
1982
|
+
createdAt: at,
|
|
1983
|
+
updatedAt: at,
|
|
1984
|
+
});
|
|
1985
|
+
}
|
|
1986
|
+
catch {
|
|
1987
|
+
// Created by a concurrent beat under the same instance id.
|
|
1988
|
+
await this.store.updateRun(id, beat);
|
|
1989
|
+
}
|
|
1990
|
+
});
|
|
1991
|
+
}
|
|
1992
|
+
catch (error) {
|
|
1993
|
+
console.error('[adonis-durable] could not write the lifecycle subscription registry:', error);
|
|
1994
|
+
}
|
|
1995
|
+
const cleanedAt = this.lifecycleRegistryCleanedAt;
|
|
1996
|
+
if (cleanedAt !== undefined &&
|
|
1997
|
+
nowMs - cleanedAt < WorkflowEngine.LIFECYCLE_REGISTRY_CLEANUP_MS) {
|
|
1998
|
+
return;
|
|
1999
|
+
}
|
|
2000
|
+
this.lifecycleRegistryCleanedAt = nowMs;
|
|
2001
|
+
const staleBefore = nowMs - LIFECYCLE_REGISTRY_STALE_MS;
|
|
2002
|
+
try {
|
|
2003
|
+
await asHeartbeat(async () => {
|
|
2004
|
+
const stale = await this.store.listRuns({
|
|
2005
|
+
workflow: LIFECYCLE_REGISTRY_WORKFLOW,
|
|
2006
|
+
...(this.namespace !== undefined ? { namespace: this.namespace } : {}),
|
|
2007
|
+
createdBefore: staleBefore,
|
|
2008
|
+
...(cleanedAt !== undefined
|
|
2009
|
+
? { createdAfter: staleBefore - LIFECYCLE_REGISTRY_STALE_MS }
|
|
2010
|
+
: {}),
|
|
2011
|
+
size: 100,
|
|
2012
|
+
});
|
|
2013
|
+
for (const row of stale) {
|
|
2014
|
+
if (row.workflow === LIFECYCLE_REGISTRY_WORKFLOW && row.id !== id) {
|
|
2015
|
+
await this.store.deleteRun(row.id);
|
|
2016
|
+
}
|
|
2017
|
+
}
|
|
2018
|
+
});
|
|
2019
|
+
}
|
|
2020
|
+
catch (error) {
|
|
2021
|
+
console.error('[adonis-durable] could not prune the lifecycle subscription registry:', error);
|
|
2022
|
+
}
|
|
2023
|
+
}
|
|
2024
|
+
/**
|
|
2025
|
+
* The runs started by `runId` settling (see {@link publishLifecycle}), read from the run's own
|
|
2026
|
+
* tags, so it needs no subscriber registry. Empty for a run that has not settled or started
|
|
2027
|
+
* nobody. Pass the run itself when it is already loaded, to skip reading it again.
|
|
2028
|
+
*/
|
|
2029
|
+
async triggeredRuns(source) {
|
|
2030
|
+
const run = typeof source === 'string' ? await this.store.getRun(source) : source;
|
|
2031
|
+
return run ? triggeredRunsOf(run) : [];
|
|
2032
|
+
}
|
|
2033
|
+
/** The live path of {@link publishSettled}: right after this instance settled the run. */
|
|
2034
|
+
publishLifecycle(event) {
|
|
2035
|
+
if (event.type !== 'run.completed' && event.type !== 'run.failed')
|
|
2036
|
+
return;
|
|
2037
|
+
const kind = event.type === 'run.completed' ? 'completed' : 'failed';
|
|
2038
|
+
if (!event.workflow || event.workflow.startsWith('__'))
|
|
2039
|
+
return;
|
|
2040
|
+
const name = lifecycleEventName(event.workflow, kind);
|
|
2041
|
+
const snapshot = this.announcedSubscriptionsSnapshot;
|
|
2042
|
+
// Without a successful read of the fleet the terminal write left the owed set unresolved, and
|
|
2043
|
+
// this publish resolves it.
|
|
2044
|
+
if (snapshot?.ok &&
|
|
2045
|
+
this.localSubscribers(name).length === 0 &&
|
|
2046
|
+
!snapshot.byEvent.get(name)?.size) {
|
|
2047
|
+
return;
|
|
2048
|
+
}
|
|
2049
|
+
// The turn that settled the run still holds its lease: publish once it let go.
|
|
2050
|
+
const turns = [...this.inflight].filter(([, runId]) => runId === event.runId).map(([p]) => p);
|
|
2051
|
+
void (async () => {
|
|
2052
|
+
await Promise.allSettled(turns);
|
|
2053
|
+
const run = await this.store.getRun(event.runId);
|
|
2054
|
+
if (run)
|
|
2055
|
+
await this.publishSettled(run);
|
|
2056
|
+
})().catch(() => undefined);
|
|
2057
|
+
}
|
|
2058
|
+
/**
|
|
2059
|
+
* Take the run lease for this instance's turn, as `tryLockRun` at `nowMs`; when the lease is held
|
|
2060
|
+
* by a lifecycle publish (see {@link withLifecycleLease}), which only ever holds a settled run for
|
|
2061
|
+
* the length of its publish (an explicit retry can land in it), wait for that publish to let go,
|
|
2062
|
+
* at most `leaseMs`.
|
|
2063
|
+
*/
|
|
2064
|
+
async lockRunPastLifecyclePublish(runId, nowMs) {
|
|
2065
|
+
if (await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs)) {
|
|
2066
|
+
return true;
|
|
2067
|
+
}
|
|
2068
|
+
// Real time, not `this.clock`: a test clock may stand still.
|
|
2069
|
+
const giveUpAt = Date.now() + this.leaseMs;
|
|
2070
|
+
let delayMs = 10;
|
|
2071
|
+
for (;;) {
|
|
2072
|
+
const held = await this.store.getRun(runId);
|
|
2073
|
+
if (!held?.lockedBy?.includes(LIFECYCLE_LEASE_MARK) || Date.now() >= giveUpAt)
|
|
2074
|
+
return false;
|
|
2075
|
+
await new Promise((r) => setTimeout(r, delayMs));
|
|
2076
|
+
delayMs = Math.min(delayMs * 2, 250);
|
|
2077
|
+
const now = this.clock();
|
|
2078
|
+
if (await this.store.tryLockRun(runId, this.instanceId, now + this.leaseMs, now))
|
|
2079
|
+
return true;
|
|
2080
|
+
}
|
|
2081
|
+
}
|
|
2082
|
+
/** The waits between attempts to take a settled run's lifecycle lease (see {@link withLifecycleLease}). */
|
|
2083
|
+
static LIFECYCLE_LEASE_RETRY_MS = [50, 100, 200, 400, 800];
|
|
2084
|
+
/**
|
|
2085
|
+
* Run `fn` holding the run lease of the settled run `runId` under an owner unique to this call, so
|
|
2086
|
+
* every read-start-write of a run's lifecycle tags (live publish, sweeps, heal, republish, on any
|
|
2087
|
+
* instance or twice in this one) runs one at a time. A refused lease is asked for again after each
|
|
2088
|
+
* of {@link LIFECYCLE_LEASE_RETRY_MS}; still refused, resolves `{ acquired: false }` without
|
|
2089
|
+
* running `fn`. The lease is released when `fn` settles, and expires after `leaseMs` if this
|
|
2090
|
+
* process dies first.
|
|
2091
|
+
*/
|
|
2092
|
+
async withLifecycleLease(runId, fn) {
|
|
2093
|
+
const owner = `${this.instanceId}${LIFECYCLE_LEASE_MARK}${globalThis.crypto.randomUUID()}`;
|
|
2094
|
+
const delays = WorkflowEngine.LIFECYCLE_LEASE_RETRY_MS;
|
|
2095
|
+
for (let attempt = 0;; attempt += 1) {
|
|
2096
|
+
const nowMs = this.clock();
|
|
2097
|
+
if (await this.store.tryLockRun(runId, owner, nowMs + this.leaseMs, nowMs))
|
|
2098
|
+
break;
|
|
2099
|
+
const delay = delays[attempt];
|
|
2100
|
+
if (delay === undefined)
|
|
2101
|
+
return { acquired: false };
|
|
2102
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
2103
|
+
}
|
|
2104
|
+
try {
|
|
2105
|
+
return { acquired: true, value: await fn() };
|
|
2106
|
+
}
|
|
2107
|
+
finally {
|
|
2108
|
+
await this.store.releaseRunLock(runId, owner).catch(() => undefined);
|
|
2109
|
+
}
|
|
2110
|
+
}
|
|
2111
|
+
/**
|
|
2112
|
+
* Publish a settled run's lifecycle event (`run.completed:<workflow>` / `run.failed:<workflow>`)
|
|
2113
|
+
* to the subscribers it owes it to that this instance can deliver to (registered here, or announced
|
|
2114
|
+
* and started by name; see {@link deliverableSubscribers}), and settle their part of the marker:
|
|
2115
|
+
* drop each one's {@link LIFECYCLE_OWED_TAG_PREFIX} tag, record each started one as a
|
|
2116
|
+
* `durable:triggered:<workflow>` tag (a coalescing one as `durable:coalesced:<workflow>`), and drop
|
|
2117
|
+
* {@link LIFECYCLE_PENDING_TAG} once no owed tag is left.
|
|
2118
|
+
*
|
|
2119
|
+
* A run marked pending with no owed tag has an unresolved owed set (see
|
|
2120
|
+
* {@link LIFECYCLE_PENDING_TAG}). With a successful read of the fleet's subscriptions it is resolved
|
|
2121
|
+
* here: every subscriber of the ending (see {@link lifecycleSubscribers}) the run's tags do not
|
|
2122
|
+
* record as started or failed becomes owed, and the ones nobody here delivers to keep an owed tag.
|
|
2123
|
+
* Without one, the subscribers registered here are started and the run stays unresolved.
|
|
2124
|
+
*
|
|
2125
|
+
* `republish` publishes to every subscriber of the ending (or only `subscriber`, which must be one),
|
|
2126
|
+
* owed or not: the ones this instance can deliver to are started, and the others are marked owed
|
|
2127
|
+
* again (with the pending tag) for the sweep of an instance that can. `heal` names subscribers of
|
|
2128
|
+
* the ending the run was never marked for (see {@link healLifecycleEvents}): they are owed now, and
|
|
2129
|
+
* the ones this instance delivers to are started.
|
|
2130
|
+
* `nowMs` (the sweep's clock) also drops the owed tag of a subscriber that is neither registered
|
|
2131
|
+
* here nor announced by a live worker once the run settled {@link LIFECYCLE_ORPHAN_MS} ago,
|
|
2132
|
+
* reporting it as `run.trigger_failed`.
|
|
2133
|
+
*
|
|
2134
|
+
* Runs under the run's lifecycle lease (see {@link withLifecycleLease}) and reads the run again
|
|
2135
|
+
* under it, so concurrent publishers never overwrite each other's tags; `busy` when the lease
|
|
2136
|
+
* stayed taken. Idempotent, since every subscriber run id is deterministic and a debounce/batch
|
|
2137
|
+
* accumulator drops an event it already holds. A subscriber whose `where` throws, whose input is
|
|
2138
|
+
* rejected or whose start fails, and an orphan the sweep drops, gets a
|
|
2139
|
+
* {@link LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX} tag in the same tag write, and is not retried until a
|
|
2140
|
+
* republish; the write that records a subscriber reached removes its trigger-failed tag.
|
|
2141
|
+
* A debounce/batch subscriber routed into its accumulator gets a
|
|
2142
|
+
* {@link LIFECYCLE_COALESCED_TAG_PREFIX} tag in the same write instead of a triggered one; a
|
|
2143
|
+
* republish routes it again only when the run also carries its trigger-failed tag.
|
|
2144
|
+
* `run.triggered` and `run.trigger_failed` are emitted only for the subscribers whose triggered,
|
|
2145
|
+
* coalesced or trigger-failed tag this write added. A failure to write the marker leaves it, and
|
|
2146
|
+
* the sweep publishes the run again.
|
|
2147
|
+
*/
|
|
2148
|
+
async publishSettled(source, opts) {
|
|
2149
|
+
const leased = await this.withLifecycleLease(source.id, async () => {
|
|
2150
|
+
const run = await this.store.getRun(source.id);
|
|
2151
|
+
return run
|
|
2152
|
+
? this.publishSettledLeased(run, opts)
|
|
2153
|
+
: { started: 0, failed: [] };
|
|
2154
|
+
});
|
|
2155
|
+
return leased.acquired ? leased.value : { started: 0, failed: [], busy: true };
|
|
2156
|
+
}
|
|
2157
|
+
/** {@link publishSettled} once it holds the run's lifecycle lease and has read `run` under it. */
|
|
2158
|
+
async publishSettledLeased(run, opts) {
|
|
2159
|
+
const kind = lifecycleKindOf(run.status);
|
|
2160
|
+
if (!kind)
|
|
2161
|
+
return { started: 0, failed: [] };
|
|
2162
|
+
const name = lifecycleEventName(run.workflow, kind);
|
|
2163
|
+
const namespace = normalizeNamespace(run.namespace);
|
|
2164
|
+
const before = run.tags ?? [];
|
|
2165
|
+
const delivered = deliveredSubscribersOf(run);
|
|
2166
|
+
let owed = owedSubscribersOf(run);
|
|
2167
|
+
const pending = before.includes(LIFECYCLE_PENDING_TAG);
|
|
2168
|
+
// Subscribers this write marks owed: a resolved or healed owed set, or a republish's re-mark.
|
|
2169
|
+
let newlyOwed = [];
|
|
2170
|
+
let unresolved = false;
|
|
2171
|
+
if (pending && owed.length === 0 && !opts?.republish) {
|
|
2172
|
+
await this.freshAnnouncedSubscriptions();
|
|
2173
|
+
if (this.announcedSubscriptionsSnapshot?.ok) {
|
|
2174
|
+
owed = this.lifecycleSubscribers(run.workflow, kind, namespace).filter((s) => !delivered.has(s));
|
|
2175
|
+
newlyOwed = owed;
|
|
2176
|
+
}
|
|
2177
|
+
else {
|
|
2178
|
+
unresolved = true;
|
|
2179
|
+
}
|
|
2180
|
+
}
|
|
2181
|
+
if (opts?.heal?.length) {
|
|
2182
|
+
const healed = opts.heal.filter((s) => !owed.includes(s) && !delivered.has(s));
|
|
2183
|
+
owed = [...owed, ...healed];
|
|
2184
|
+
newlyOwed = [...newlyOwed, ...healed];
|
|
2185
|
+
}
|
|
2186
|
+
const { local, remote, all: here } = await this.deliverableSubscribers(name, namespace);
|
|
2187
|
+
let targets;
|
|
2188
|
+
const refused = [];
|
|
2189
|
+
if (opts?.republish) {
|
|
2190
|
+
await this.freshAnnouncedSubscriptions();
|
|
2191
|
+
// Without a read of the fleet, an unresolved run stays unresolved for the next publish.
|
|
2192
|
+
unresolved = pending && owed.length === 0 && !this.announcedSubscriptionsSnapshot?.ok;
|
|
2193
|
+
const subscribers = this.lifecycleSubscribers(run.workflow, kind, namespace);
|
|
2194
|
+
let all = subscribers;
|
|
2195
|
+
if (opts.subscriber !== undefined) {
|
|
2196
|
+
all = subscribers.includes(opts.subscriber) ? [opts.subscriber] : [];
|
|
2197
|
+
if (all.length === 0) {
|
|
2198
|
+
refused.push({
|
|
2199
|
+
workflow: opts.subscriber,
|
|
2200
|
+
error: new Error(`subscriber ${opts.subscriber} does not subscribe to ${name}`),
|
|
2201
|
+
});
|
|
2202
|
+
}
|
|
2203
|
+
}
|
|
2204
|
+
// An accumulator that already fired would start its target again, so a subscriber the run's
|
|
2205
|
+
// tags record as coalesced is routed again (or owed again) only after a failed delivery.
|
|
2206
|
+
const settledCoalesced = new Set(coalescedSubscribersOf(before).filter((s) => !before.includes(lifecycleTriggerFailedTag(s))));
|
|
2207
|
+
all = all.filter((s) => !settledCoalesced.has(s));
|
|
2208
|
+
targets = all.filter((s) => here.has(s));
|
|
2209
|
+
newlyOwed = all.filter((s) => !here.has(s));
|
|
2210
|
+
}
|
|
2211
|
+
else if (unresolved) {
|
|
2212
|
+
targets = [...local].filter((s) => !delivered.has(s));
|
|
2213
|
+
}
|
|
2214
|
+
else {
|
|
2215
|
+
targets = owed.filter((s) => here.has(s));
|
|
2216
|
+
}
|
|
2217
|
+
const { reached, failed } = targets.length > 0
|
|
2218
|
+
? await this.startSubscribers(name, lifecycleEventOf(run), (sub) => lifecycleRunId(run.id, kind, sub), new Set(targets), remote)
|
|
2219
|
+
: { reached: [], failed: [] };
|
|
2220
|
+
const orphaned = opts?.nowMs !== undefined ? await this.orphanedSubscribers(run, owed, here, opts.nowMs) : [];
|
|
2221
|
+
const done = new Set([...targets, ...orphaned]);
|
|
2222
|
+
// A started subscriber is recorded by its triggered tag, one routed into its accumulator by its
|
|
2223
|
+
// coalesced tag.
|
|
2224
|
+
const deliveryTag = (r) => r.runId !== undefined
|
|
2225
|
+
? `${LIFECYCLE_TRIGGERED_TAG_PREFIX}${r.workflow}`
|
|
2226
|
+
: lifecycleCoalescedTag(r.workflow);
|
|
2227
|
+
const deliveryTags = reached.map(deliveryTag);
|
|
2228
|
+
const reachedNow = new Set(reached.map((r) => r.workflow));
|
|
2229
|
+
const failedNow = [...failed.map((f) => f.workflow), ...orphaned];
|
|
2230
|
+
const kept = [
|
|
2231
|
+
...before.filter((t) => !((t.startsWith(LIFECYCLE_OWED_TAG_PREFIX) &&
|
|
2232
|
+
done.has(t.slice(LIFECYCLE_OWED_TAG_PREFIX.length))) ||
|
|
2233
|
+
(t.startsWith(LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX) &&
|
|
2234
|
+
reachedNow.has(t.slice(LIFECYCLE_TRIGGER_FAILED_TAG_PREFIX.length))))),
|
|
2235
|
+
...newlyOwed.filter((s) => !done.has(s)).map(lifecycleOwedTag),
|
|
2236
|
+
];
|
|
2237
|
+
const owedLeft = kept.some((t) => t.startsWith(LIFECYCLE_OWED_TAG_PREFIX));
|
|
2238
|
+
const tags = [
|
|
2239
|
+
...new Set([
|
|
2240
|
+
...kept.filter((t) => t !== LIFECYCLE_PENDING_TAG),
|
|
2241
|
+
...(owedLeft || unresolved ? [LIFECYCLE_PENDING_TAG] : []),
|
|
2242
|
+
...deliveryTags,
|
|
2243
|
+
...failedNow.map(lifecycleTriggerFailedTag),
|
|
2244
|
+
]),
|
|
2245
|
+
];
|
|
2246
|
+
if (tags.length !== before.length || tags.some((t) => !before.includes(t))) {
|
|
2247
|
+
await this.store.updateRun(run.id, { tags });
|
|
2248
|
+
}
|
|
2249
|
+
// Only the tag changes this write made are announced, so a publisher that repeats another's
|
|
2250
|
+
// delivery emits nothing.
|
|
2251
|
+
for (const r of reached) {
|
|
2252
|
+
if (before.includes(deliveryTag(r)))
|
|
2253
|
+
continue;
|
|
2254
|
+
const { workflow, runId } = r;
|
|
2255
|
+
this.emit({
|
|
2256
|
+
type: 'run.triggered',
|
|
2257
|
+
runId: run.id,
|
|
2258
|
+
workflow: run.workflow,
|
|
2259
|
+
namespace: run.namespace,
|
|
2260
|
+
name: workflow,
|
|
2261
|
+
triggeredRunId: runId,
|
|
2262
|
+
});
|
|
2263
|
+
}
|
|
2264
|
+
const unreached = [
|
|
2265
|
+
...failed,
|
|
2266
|
+
...orphaned.map((workflow) => ({
|
|
2267
|
+
workflow,
|
|
2268
|
+
error: new Error(`subscriber ${workflow} is not registered on any live worker`),
|
|
2269
|
+
})),
|
|
2270
|
+
];
|
|
2271
|
+
for (const { workflow, error } of unreached) {
|
|
2272
|
+
if (before.includes(lifecycleTriggerFailedTag(workflow)))
|
|
2273
|
+
continue;
|
|
2274
|
+
this.emit({
|
|
2275
|
+
type: 'run.trigger_failed',
|
|
2276
|
+
runId: run.id,
|
|
2277
|
+
workflow: run.workflow,
|
|
2278
|
+
namespace: run.namespace,
|
|
2279
|
+
name: workflow,
|
|
2280
|
+
error: { message: error instanceof Error ? error.message : String(error) },
|
|
2281
|
+
});
|
|
2282
|
+
}
|
|
2283
|
+
return {
|
|
2284
|
+
started: reached.filter((r) => r.created).length,
|
|
2285
|
+
failed: [...refused, ...unreached],
|
|
2286
|
+
};
|
|
2287
|
+
}
|
|
2288
|
+
/**
|
|
2289
|
+
* The subscribers `run` owes its event to that nothing can deliver it to: not in `here` (what
|
|
2290
|
+
* this instance delivers to), announced by no live worker, and owed since
|
|
2291
|
+
* {@link LIFECYCLE_ORPHAN_MS} or longer. Empty while the fleet's announcements cannot be read,
|
|
2292
|
+
* since then nobody is known to be gone.
|
|
2293
|
+
*/
|
|
2294
|
+
async orphanedSubscribers(run, owed, here, nowMs) {
|
|
2295
|
+
if (nowMs - run.updatedAt.getTime() < LIFECYCLE_ORPHAN_MS)
|
|
2296
|
+
return [];
|
|
2297
|
+
const candidates = owed.filter((s) => !here.has(s));
|
|
2298
|
+
if (candidates.length === 0)
|
|
2299
|
+
return [];
|
|
2300
|
+
await this.freshAnnouncedSubscriptions();
|
|
2301
|
+
const fresh = this.announcedSubscriptionsSnapshot;
|
|
2302
|
+
if (!fresh?.ok)
|
|
2303
|
+
return [];
|
|
2304
|
+
const kind = lifecycleKindOf(run.status);
|
|
2305
|
+
if (!kind)
|
|
2306
|
+
return [];
|
|
2307
|
+
const announced = this.announcedFor(lifecycleEventName(run.workflow, kind), normalizeNamespace(run.namespace));
|
|
2308
|
+
return candidates.filter((s) => !announced.has(s));
|
|
2309
|
+
}
|
|
2310
|
+
/** How long a settled run's marker is left to the live publish before the sweep takes it over. */
|
|
2311
|
+
static LIFECYCLE_SWEEP_GRACE_MS = 30_000;
|
|
2312
|
+
/** How many marked runs one lifecycle sweep reads at most, looking for ones it can act on. */
|
|
2313
|
+
static LIFECYCLE_SWEEP_SCAN_CAP = 1000;
|
|
2314
|
+
/**
|
|
2315
|
+
* Publish the lifecycle event of settled runs still marked {@link LIFECYCLE_PENDING_TAG} to the
|
|
2316
|
+
* subscribers this instance can deliver to that they still owe it to: a run whose settling process
|
|
2317
|
+
* died, or whose marker write failed, before publishing, or whose subscriber is registered only on
|
|
2318
|
+
* other instances. Also writes this instance's row of the lifecycle subscription registry (see
|
|
2319
|
+
* {@link heartbeatLifecycleRegistry}), refreshes the snapshot of the subscriptions the fleet
|
|
2320
|
+
* announces (registry rows and worker descriptors), drops
|
|
2321
|
+
* owed subscribers nothing can deliver to (see {@link orphanedSubscribers}) and wakes the
|
|
2322
|
+
* `ctx.waitForRun` waiters whose run already settled (see {@link sweepRunWaits}).
|
|
2323
|
+
*
|
|
2324
|
+
* Called by the worker poll loop. Reads the marked runs page by page and skips the ones it cannot
|
|
2325
|
+
* act on (every owed subscriber is delivered elsewhere and none is orphaned yet), so runs owed only
|
|
2326
|
+
* to other instances never hide the ones it can publish. Stops once it has published `limit` runs
|
|
2327
|
+
* or read {@link LIFECYCLE_SWEEP_SCAN_CAP} marked runs. A run settled in the last
|
|
2328
|
+
* {@link LIFECYCLE_SWEEP_GRACE_MS} is left to the live publish; a run another publisher holds
|
|
2329
|
+
* (see {@link withLifecycleLease}) is left to the next tick. Then heals the runs settled in the
|
|
2330
|
+
* last {@link LIFECYCLE_HEAL_WINDOW_MS} (see {@link healLifecycleEvents}). Returns how many marked
|
|
2331
|
+
* runs it published plus how many waiters it woke; healed runs are not counted.
|
|
2332
|
+
*/
|
|
2333
|
+
async sweepLifecycleEvents(nowMs = this.clock(), limit = 100) {
|
|
2334
|
+
await this.heartbeatLifecycleRegistry();
|
|
2335
|
+
// A failed read is tried again on every tick, so unresolved runs resolve once the fleet is back.
|
|
2336
|
+
if (this.announcedSubscriptionsSnapshot?.ok === false)
|
|
2337
|
+
await this.refreshAnnouncedSubscriptions();
|
|
2338
|
+
else
|
|
2339
|
+
await this.freshAnnouncedSubscriptions();
|
|
2340
|
+
const actionable = await this.lifecycleActionableRuns(nowMs, limit);
|
|
2341
|
+
let published = 0;
|
|
2342
|
+
for (const run of actionable) {
|
|
2343
|
+
try {
|
|
2344
|
+
const { busy } = await this.publishSettled(run, { nowMs });
|
|
2345
|
+
if (!busy)
|
|
2346
|
+
published += 1;
|
|
2347
|
+
}
|
|
2348
|
+
catch (error) {
|
|
2349
|
+
console.error(`[adonis-durable] lifecycle sweep: run ${run.id} failed; retrying next tick:`, error);
|
|
2350
|
+
}
|
|
2351
|
+
}
|
|
2352
|
+
try {
|
|
2353
|
+
await this.healLifecycleEvents(nowMs);
|
|
2354
|
+
}
|
|
2355
|
+
catch (error) {
|
|
2356
|
+
console.error('[adonis-durable] lifecycle sweep: heal failed; retrying next tick:', error);
|
|
2357
|
+
}
|
|
2358
|
+
return published + (await this.sweepRunWaits(limit));
|
|
2359
|
+
}
|
|
2360
|
+
/** How many settled runs of one source workflow a heal pass reads at most. */
|
|
2361
|
+
static LIFECYCLE_HEAL_SCAN_CAP = 100;
|
|
2362
|
+
/**
|
|
2363
|
+
* The (run, subscriber) pairs this engine already healed, with when each entry lapses (epoch ms):
|
|
2364
|
+
* a subscriber whose `where` declines the event leaves no tag, so without it every sweep would
|
|
2365
|
+
* offer it the event again for the whole heal window.
|
|
2366
|
+
*/
|
|
2367
|
+
lifecycleHealed = new Map();
|
|
2368
|
+
/**
|
|
2369
|
+
* Deliver lifecycle events the terminal write never marked: for every source workflow some
|
|
2370
|
+
* subscriber in the current snapshot of the fleet's subscriptions (or registered here) listens to,
|
|
2371
|
+
* read its runs settled in the last {@link LIFECYCLE_HEAL_WINDOW_MS} (at most
|
|
2372
|
+
* {@link LIFECYCLE_HEAL_SCAN_CAP}, one listing per workflow) and publish each one to the
|
|
2373
|
+
* subscribers of its ending its tags record as neither owed, started, coalesced nor failed (see
|
|
2374
|
+
* {@link publishSettled}'s `heal`). That covers a subscriber announced just after the run
|
|
2375
|
+
* settled, while the settling instance's snapshot was stale. Each pair is healed at most once per
|
|
2376
|
+
* engine. A run whose owed set is unresolved is left to {@link sweepLifecycleEvents}. Needs a
|
|
2377
|
+
* successful read of the fleet's subscriptions; returns how many runs it published.
|
|
2378
|
+
*/
|
|
2379
|
+
async healLifecycleEvents(nowMs) {
|
|
2380
|
+
const snapshot = this.announcedSubscriptionsSnapshot;
|
|
2381
|
+
if (!snapshot?.ok)
|
|
2382
|
+
return 0;
|
|
2383
|
+
for (const [key, lapsesAt] of this.lifecycleHealed) {
|
|
2384
|
+
if (lapsesAt <= nowMs)
|
|
2385
|
+
this.lifecycleHealed.delete(key);
|
|
2386
|
+
}
|
|
2387
|
+
const sources = new Map();
|
|
2388
|
+
for (const event of new Set([...snapshot.byEvent.keys(), ...this.eventTriggers.keys()])) {
|
|
2389
|
+
const source = parseLifecycleEvent(event);
|
|
2390
|
+
if (!source || source.workflow.startsWith('__'))
|
|
2391
|
+
continue;
|
|
2392
|
+
const kinds = sources.get(source.workflow) ?? new Set();
|
|
2393
|
+
kinds.add(source.kind);
|
|
2394
|
+
sources.set(source.workflow, kinds);
|
|
2395
|
+
}
|
|
2396
|
+
let healed = 0;
|
|
2397
|
+
for (const [workflow, kinds] of sources) {
|
|
2398
|
+
const statuses = [
|
|
2399
|
+
...(kinds.has('completed') ? ['completed'] : []),
|
|
2400
|
+
...(kinds.has('failed') ? ['failed', 'cancelled', 'dead'] : []),
|
|
2401
|
+
];
|
|
2402
|
+
const runs = await asHeartbeat(() => this.store.listRuns({
|
|
2403
|
+
workflow,
|
|
2404
|
+
statuses,
|
|
2405
|
+
...(this.namespace !== undefined ? { namespace: this.namespace } : {}),
|
|
2406
|
+
updatedAfter: nowMs - LIFECYCLE_HEAL_WINDOW_MS,
|
|
2407
|
+
size: WorkflowEngine.LIFECYCLE_HEAL_SCAN_CAP,
|
|
2408
|
+
}));
|
|
2409
|
+
for (const run of runs) {
|
|
2410
|
+
const kind = lifecycleKindOf(run.status);
|
|
2411
|
+
if (run.workflow !== workflow || !kind || !kinds.has(kind))
|
|
2412
|
+
continue;
|
|
2413
|
+
if (nowMs - run.updatedAt.getTime() > LIFECYCLE_HEAL_WINDOW_MS)
|
|
2414
|
+
continue;
|
|
2415
|
+
const owed = owedSubscribersOf(run);
|
|
2416
|
+
if (owed.length === 0 && run.tags?.includes(LIFECYCLE_PENDING_TAG))
|
|
2417
|
+
continue;
|
|
2418
|
+
const delivered = deliveredSubscribersOf(run);
|
|
2419
|
+
const missing = this.lifecycleSubscribers(workflow, kind, normalizeNamespace(run.namespace)).filter((s) => !owed.includes(s) &&
|
|
2420
|
+
!delivered.has(s) &&
|
|
2421
|
+
!this.lifecycleHealed.has(`${run.id}\u0000${s}`));
|
|
2422
|
+
if (missing.length === 0)
|
|
2423
|
+
continue;
|
|
2424
|
+
const { busy } = await this.publishSettled(run, { heal: missing, nowMs });
|
|
2425
|
+
if (busy)
|
|
2426
|
+
continue;
|
|
2427
|
+
for (const s of missing) {
|
|
2428
|
+
this.lifecycleHealed.set(`${run.id}\u0000${s}`, nowMs + LIFECYCLE_HEAL_WINDOW_MS);
|
|
2429
|
+
}
|
|
2430
|
+
healed += 1;
|
|
2431
|
+
}
|
|
2432
|
+
}
|
|
2433
|
+
return healed;
|
|
2434
|
+
}
|
|
2435
|
+
/**
|
|
2436
|
+
* Up to `limit` settled runs marked {@link LIFECYCLE_PENDING_TAG}, past the sweep grace, that this
|
|
2437
|
+
* instance can act on: a run owing its event to a subscriber it delivers to, a run marked without
|
|
2438
|
+
* owed tags, or a run owing it to a subscriber nothing delivers to past
|
|
2439
|
+
* {@link LIFECYCLE_ORPHAN_MS}. Reads at most {@link LIFECYCLE_SWEEP_SCAN_CAP} marked runs, in
|
|
2440
|
+
* pages, all read before any is acted on, since publishing a run takes it out of the listing.
|
|
2441
|
+
*/
|
|
2442
|
+
async lifecycleActionableRuns(nowMs, limit) {
|
|
2443
|
+
const size = Math.max(1, Math.min(limit, 100));
|
|
2444
|
+
const deliverable = new Map();
|
|
2445
|
+
const deliverableTo = (event, namespace) => {
|
|
2446
|
+
const key = `${namespace}\u0000${event}`;
|
|
2447
|
+
let entry = deliverable.get(key);
|
|
2448
|
+
if (!entry) {
|
|
2449
|
+
entry = this.deliverableSubscribers(event, namespace).then((d) => d.all);
|
|
2450
|
+
deliverable.set(key, entry);
|
|
2451
|
+
}
|
|
2452
|
+
return entry;
|
|
2453
|
+
};
|
|
2454
|
+
const out = [];
|
|
2455
|
+
const seen = new Set();
|
|
2456
|
+
let scanned = 0;
|
|
2457
|
+
for (let page = 1; out.length < limit && scanned < WorkflowEngine.LIFECYCLE_SWEEP_SCAN_CAP;) {
|
|
2458
|
+
const runs = await asHeartbeat(() => this.store.listRuns({
|
|
2459
|
+
tag: LIFECYCLE_PENDING_TAG,
|
|
2460
|
+
statuses: ['completed', 'failed', 'cancelled', 'dead'],
|
|
2461
|
+
...(this.namespace !== undefined ? { namespace: this.namespace } : {}),
|
|
2462
|
+
updatedBefore: nowMs - WorkflowEngine.LIFECYCLE_SWEEP_GRACE_MS,
|
|
2463
|
+
page,
|
|
2464
|
+
size,
|
|
2465
|
+
}));
|
|
2466
|
+
for (const run of runs) {
|
|
2467
|
+
if (out.length >= limit || scanned >= WorkflowEngine.LIFECYCLE_SWEEP_SCAN_CAP)
|
|
2468
|
+
break;
|
|
2469
|
+
scanned += 1;
|
|
2470
|
+
if (seen.has(run.id))
|
|
2471
|
+
continue;
|
|
2472
|
+
seen.add(run.id);
|
|
2473
|
+
if (nowMs - run.updatedAt.getTime() < WorkflowEngine.LIFECYCLE_SWEEP_GRACE_MS)
|
|
2474
|
+
continue;
|
|
2475
|
+
const kind = lifecycleKindOf(run.status);
|
|
2476
|
+
if (!kind)
|
|
2477
|
+
continue;
|
|
2478
|
+
const owed = owedSubscribersOf(run);
|
|
2479
|
+
if (owed.length === 0) {
|
|
2480
|
+
out.push(run);
|
|
2481
|
+
continue;
|
|
2482
|
+
}
|
|
2483
|
+
const here = await deliverableTo(lifecycleEventName(run.workflow, kind), normalizeNamespace(run.namespace));
|
|
2484
|
+
if (owed.some((s) => here.has(s)) ||
|
|
2485
|
+
(await this.orphanedSubscribers(run, owed, here, nowMs)).length > 0) {
|
|
2486
|
+
out.push(run);
|
|
2487
|
+
}
|
|
2488
|
+
}
|
|
2489
|
+
if (runs.length < size)
|
|
2490
|
+
break;
|
|
2491
|
+
page += 1;
|
|
2492
|
+
}
|
|
2493
|
+
return out;
|
|
2494
|
+
}
|
|
2495
|
+
/**
|
|
2496
|
+
* Publish settled runs' lifecycle events again, whether or not they are still marked: one run
|
|
2497
|
+
* (`{ runId }`), or a workflow's settled runs (`{ workflow, kind, since, until, limit }`, read in
|
|
2498
|
+
* pages of the store's listing, filtered by when the run settled). `subscriber` restricts it to
|
|
2499
|
+
* one subscribing workflow, which must subscribe to the run's ending (registered here or announced
|
|
2500
|
+
* by a live worker for the run's namespace): otherwise the run is reported in `failed` with
|
|
2501
|
+
* `subscriber <name> does not subscribe to <event>` and nothing is marked. The subscribers
|
|
2502
|
+
* registered on this instance are started; any other (announced by a live worker) is marked owed
|
|
2503
|
+
* again, so the sweep of an instance that registers it starts it, and is not counted in
|
|
2504
|
+
* `started`. The same path as the live publish, so a subscriber whose run already exists is not
|
|
2505
|
+
* started again; a subscriber that debounces or batches is not routed again once the run carries
|
|
2506
|
+
* its coalesced tag, unless it also carries its trigger-failed tag, since an accumulator that
|
|
2507
|
+
* already fired would start the target again. A run another
|
|
2508
|
+
* publisher holds past the lease retries is reported in `failed` with subscriber `*`. A
|
|
2509
|
+
* subscriber registered after the runs settled (more than {@link LIFECYCLE_HEAL_WINDOW_MS} after,
|
|
2510
|
+
* past what the sweep heals) is backfilled this way. A workflow target's `namespace` narrows it to
|
|
2511
|
+
* that namespace; on an engine bound to another namespace it publishes nothing.
|
|
2512
|
+
*/
|
|
2513
|
+
async republishLifecycle(target) {
|
|
2514
|
+
const result = { runs: 0, started: 0, failed: [] };
|
|
2515
|
+
// A backfill usually follows a deploy: read the fleet's subscriptions now, not from the cache.
|
|
2516
|
+
await this.refreshAnnouncedSubscriptions();
|
|
2517
|
+
const publish = async (run) => {
|
|
2518
|
+
if (!lifecycleKindOf(run.status))
|
|
2519
|
+
return;
|
|
2520
|
+
result.runs += 1;
|
|
2521
|
+
const { started, failed, busy } = await this.publishSettled(run, {
|
|
2522
|
+
republish: true,
|
|
2523
|
+
subscriber: target.subscriber,
|
|
2524
|
+
});
|
|
2525
|
+
if (busy) {
|
|
2526
|
+
result.failed.push({
|
|
2527
|
+
runId: run.id,
|
|
2528
|
+
subscriber: '*',
|
|
2529
|
+
error: `run ${run.id} is being published elsewhere; try again`,
|
|
2530
|
+
});
|
|
2531
|
+
return;
|
|
2532
|
+
}
|
|
2533
|
+
result.started += started;
|
|
2534
|
+
for (const { workflow, error } of failed) {
|
|
2535
|
+
result.failed.push({
|
|
2536
|
+
runId: run.id,
|
|
2537
|
+
subscriber: workflow,
|
|
2538
|
+
error: error instanceof Error ? error.message : String(error),
|
|
2539
|
+
});
|
|
2540
|
+
}
|
|
2541
|
+
};
|
|
2542
|
+
if ('runId' in target) {
|
|
2543
|
+
const run = await this.store.getRun(target.runId);
|
|
2544
|
+
if (run)
|
|
2545
|
+
await publish(run);
|
|
2546
|
+
return result;
|
|
2547
|
+
}
|
|
2548
|
+
if (target.namespace !== undefined &&
|
|
2549
|
+
this.namespace !== undefined &&
|
|
2550
|
+
target.namespace !== this.namespace) {
|
|
2551
|
+
return result;
|
|
2552
|
+
}
|
|
2553
|
+
const namespace = target.namespace ?? this.namespace;
|
|
2554
|
+
const limit = target.limit ?? 1000;
|
|
2555
|
+
const statuses = target.kind === 'completed'
|
|
2556
|
+
? ['completed']
|
|
2557
|
+
: target.kind === 'failed'
|
|
2558
|
+
? ['failed', 'cancelled', 'dead']
|
|
2559
|
+
: ['completed', 'failed', 'cancelled', 'dead'];
|
|
2560
|
+
const size = Math.min(limit, 100);
|
|
2561
|
+
let seen = 0;
|
|
2562
|
+
for (let page = 1; seen < limit; page += 1) {
|
|
2563
|
+
const runs = await this.store.listRuns({
|
|
2564
|
+
workflow: target.workflow,
|
|
2565
|
+
statuses,
|
|
2566
|
+
...(namespace !== undefined ? { namespace } : {}),
|
|
2567
|
+
page,
|
|
2568
|
+
size,
|
|
2569
|
+
});
|
|
2570
|
+
for (const run of runs) {
|
|
2571
|
+
if (seen >= limit)
|
|
2572
|
+
break;
|
|
2573
|
+
const settledAt = run.updatedAt.getTime();
|
|
2574
|
+
if (target.since && settledAt < target.since.getTime())
|
|
2575
|
+
continue;
|
|
2576
|
+
if (target.until && settledAt > target.until.getTime())
|
|
2577
|
+
continue;
|
|
2578
|
+
seen += 1;
|
|
2579
|
+
await publish(run);
|
|
2580
|
+
}
|
|
2581
|
+
if (runs.length < size)
|
|
2582
|
+
break;
|
|
2583
|
+
}
|
|
2584
|
+
return result;
|
|
2585
|
+
}
|
|
2586
|
+
/**
|
|
2587
|
+
* Wake the `ctx.waitForRun` waiters on `runId`, which just settled, and on the earlier runs of its
|
|
2588
|
+
* continue-as-new chain, with the event of the run that ends the chain (none yet when `runId`
|
|
2589
|
+
* handed off). One waiter listing per settle. Best effort: a waiter missed here (this process dies
|
|
2590
|
+
* first) is woken by {@link sweepRunWaits}. Internal `__` runs have no waiters, so they are
|
|
2591
|
+
* skipped without a read.
|
|
2592
|
+
*/
|
|
2593
|
+
notifyRunWaiters(runId, workflow) {
|
|
2594
|
+
if (workflow?.startsWith('__'))
|
|
2595
|
+
return;
|
|
2596
|
+
void (async () => {
|
|
2597
|
+
const predecessors = chainPredecessorsOf(runId);
|
|
2598
|
+
const targets = new Set([runId, ...predecessors]);
|
|
2599
|
+
// Every run of a chain shares the first one's id as a prefix.
|
|
2600
|
+
const prefix = predecessors[0] !== undefined
|
|
2601
|
+
? `${RUN_WAIT_TOKEN_PREFIX}${predecessors[0]}`
|
|
2602
|
+
: runWaitPrefix(runId);
|
|
2603
|
+
const waiters = (await this.store.listSignalWaiters(prefix)).filter((w) => targets.has(runWaitTargetOf(w) ?? ''));
|
|
2604
|
+
if (waiters.length === 0)
|
|
2605
|
+
return;
|
|
2606
|
+
const run = await lastRunOfChain((id) => this.store.getRun(id), runId);
|
|
2607
|
+
if (!run || !lifecycleKindOf(run.status))
|
|
2608
|
+
return;
|
|
2609
|
+
const event = lifecycleEventOf(run);
|
|
2610
|
+
for (const waiter of waiters)
|
|
2611
|
+
await this.wakeSignalWaiter(waiter.token, event);
|
|
2612
|
+
})().catch(() => undefined);
|
|
2613
|
+
}
|
|
2614
|
+
/** How many waited-on runs one {@link sweepRunWaits} reads at most. */
|
|
2615
|
+
static RUN_WAIT_SWEEP_SCAN_CAP = 1000;
|
|
2616
|
+
/** Where the next {@link sweepRunWaits} starts in the list of waited-on runs. */
|
|
2617
|
+
runWaitSweepCursor = 0;
|
|
2618
|
+
/**
|
|
2619
|
+
* Wake the `ctx.waitForRun` waiters (up to `limit`) whose run, or the run ending its
|
|
2620
|
+
* continue-as-new chain, has already settled: the recovery for a process that died between
|
|
2621
|
+
* settling a run and waking its waiters. Waiters on runs still in flight are skipped, reading at
|
|
2622
|
+
* most {@link RUN_WAIT_SWEEP_SCAN_CAP} waited-on runs per call; the next call starts where this one
|
|
2623
|
+
* stopped, so waiters on long-running runs never hide the ones that can be woken. Returns how many
|
|
2624
|
+
* it woke.
|
|
2625
|
+
*/
|
|
2626
|
+
async sweepRunWaits(limit = 100) {
|
|
2627
|
+
const waiters = await asHeartbeat(() => this.store.listSignalWaiters(RUN_WAIT_TOKEN_PREFIX));
|
|
2628
|
+
const byTarget = new Map();
|
|
2629
|
+
for (const waiter of waiters) {
|
|
2630
|
+
const target = runWaitTargetOf(waiter);
|
|
2631
|
+
if (target === undefined)
|
|
2632
|
+
continue;
|
|
2633
|
+
byTarget.set(target, [...(byTarget.get(target) ?? []), waiter]);
|
|
2634
|
+
}
|
|
2635
|
+
const targets = [...byTarget.keys()].sort();
|
|
2636
|
+
if (targets.length === 0)
|
|
2637
|
+
return 0;
|
|
2638
|
+
const from = this.runWaitSweepCursor % targets.length;
|
|
2639
|
+
const scan = Math.min(targets.length, WorkflowEngine.RUN_WAIT_SWEEP_SCAN_CAP);
|
|
2640
|
+
let woken = 0;
|
|
2641
|
+
let read = 0;
|
|
2642
|
+
for (; read < scan && woken < limit; read += 1) {
|
|
2643
|
+
const target = targets[(from + read) % targets.length];
|
|
2644
|
+
const run = await lastRunOfChain((id) => this.store.getRun(id), target);
|
|
2645
|
+
if (!run || !lifecycleKindOf(run.status))
|
|
2646
|
+
continue;
|
|
2647
|
+
const event = lifecycleEventOf(run);
|
|
2648
|
+
for (const waiter of byTarget.get(target) ?? []) {
|
|
2649
|
+
if (await this.wakeSignalWaiter(waiter.token, event))
|
|
2650
|
+
woken += 1;
|
|
2651
|
+
}
|
|
2652
|
+
}
|
|
2653
|
+
this.runWaitSweepCursor = read >= targets.length ? 0 : (from + read) % targets.length;
|
|
2654
|
+
return woken;
|
|
2655
|
+
}
|
|
1423
2656
|
/** Events waiting for the control-plane publish in flight, in emit order (see {@link broadcastEvents}). */
|
|
1424
2657
|
eventsToBroadcast = [];
|
|
1425
2658
|
broadcastingEvents = false;
|
|
@@ -1686,8 +2919,11 @@ export class WorkflowEngine {
|
|
|
1686
2919
|
* engine's read API for inspection surfaces (the dashboard, the `durable:runs` CLI) so they query
|
|
1687
2920
|
* through the engine rather than reaching for its private store.
|
|
1688
2921
|
*/
|
|
1689
|
-
listRuns(query) {
|
|
1690
|
-
|
|
2922
|
+
async listRuns(query) {
|
|
2923
|
+
const scoped = withoutBookkeepingRuns(query);
|
|
2924
|
+
const runs = await this.store.listRuns(scoped);
|
|
2925
|
+
const hidden = scoped.excludeWorkflows;
|
|
2926
|
+
return hidden?.length ? runs.filter((run) => !hidden.includes(run.workflow)) : runs;
|
|
1691
2927
|
}
|
|
1692
2928
|
/**
|
|
1693
2929
|
* The distinct values of ONE filter axis over the runs matching `query`, with counts — what a
|
|
@@ -1696,10 +2932,11 @@ export class WorkflowEngine {
|
|
|
1696
2932
|
* {@link listRuns}.
|
|
1697
2933
|
*/
|
|
1698
2934
|
runValueFacets(axis, query, opts) {
|
|
2935
|
+
const scoped = withoutBookkeepingRuns(query);
|
|
1699
2936
|
if (this.store.runValueFacets) {
|
|
1700
|
-
return this.store.runValueFacets(axis,
|
|
2937
|
+
return this.store.runValueFacets(axis, scoped, opts);
|
|
1701
2938
|
}
|
|
1702
|
-
return scanRunValueFacets(this.store, axis,
|
|
2939
|
+
return scanRunValueFacets(this.store, axis, scoped, opts);
|
|
1703
2940
|
}
|
|
1704
2941
|
/** List a run's step checkpoints (its timeline). Part of the engine's read API; see {@link listRuns}. */
|
|
1705
2942
|
listCheckpoints(runId) {
|
|
@@ -1710,15 +2947,149 @@ export class WorkflowEngine {
|
|
|
1710
2947
|
* has {@link StateStore.runFacets} (else counted over a listing). Part of the engine's read API.
|
|
1711
2948
|
*/
|
|
1712
2949
|
runFacets(query) {
|
|
1713
|
-
return runFacetsOf(this.store, query);
|
|
2950
|
+
return runFacetsOf(this.store, withoutBookkeepingRuns(query));
|
|
2951
|
+
}
|
|
2952
|
+
/**
|
|
2953
|
+
* A run's detail (see `loadRunDetail`), built from the store's checkpoint aggregates when it has
|
|
2954
|
+
* `checkpointStats` + `queryCheckpoints` — a summarized detail then reads a few rows per wide fan
|
|
2955
|
+
* instead of the whole history, and carries an `etag` that `options.ifNoneMatch` can match.
|
|
2956
|
+
*/
|
|
2957
|
+
async getRunDetail(runId, options) {
|
|
2958
|
+
const source = this.detailSource();
|
|
2959
|
+
const withExtras = async (detail) => {
|
|
2960
|
+
const extras = await this.lifecycleDetail(detail.run);
|
|
2961
|
+
return {
|
|
2962
|
+
...detail,
|
|
2963
|
+
...extras,
|
|
2964
|
+
...(detail.etag ? { etag: extendEtag(detail.etag, nonEmptyObject(extras)) } : {}),
|
|
2965
|
+
};
|
|
2966
|
+
};
|
|
2967
|
+
const ifNoneMatch = options?.ifNoneMatch;
|
|
2968
|
+
if (ifNoneMatch === undefined) {
|
|
2969
|
+
const detail = await loadRunDetail(source, runId, options);
|
|
2970
|
+
return detail ? withExtras(detail) : null;
|
|
2971
|
+
}
|
|
2972
|
+
// A revalidation: read the run and its validator only ('*' always matches), add what the
|
|
2973
|
+
// detail reads from outside the run, and answer 304 when both are unchanged. A source that
|
|
2974
|
+
// cannot version the detail answers it in full instead.
|
|
2975
|
+
const head = await loadRunDetail(source, runId, { ...options, ifNoneMatch: '*' });
|
|
2976
|
+
if (!head)
|
|
2977
|
+
return null;
|
|
2978
|
+
if (!head.notModified || !head.etag)
|
|
2979
|
+
return withExtras(head);
|
|
2980
|
+
const extras = await this.lifecycleDetail(head.run);
|
|
2981
|
+
const etag = extendEtag(head.etag, nonEmptyObject(extras));
|
|
2982
|
+
if (etagMatches(ifNoneMatch, etag)) {
|
|
2983
|
+
return { run: head.run, timeline: [], children: [], notModified: true, etag };
|
|
2984
|
+
}
|
|
2985
|
+
const { ifNoneMatch: _matched, ...fresh } = options;
|
|
2986
|
+
const detail = await loadRunDetail(source, runId, fresh);
|
|
2987
|
+
return detail ? withExtras(detail) : null;
|
|
2988
|
+
}
|
|
2989
|
+
/**
|
|
2990
|
+
* What a run detail reads beyond the run row and its checkpoints: the subscribers its lifecycle
|
|
2991
|
+
* event goes to (of its namespace) and where it stands for each, its workflow's lifecycle links,
|
|
2992
|
+
* and the `ctx.waitForRun` waits on either side of it with the other run's status. A few reads per
|
|
2993
|
+
* detail: the run's own waiters while it is suspended, the waiters on it (one prefix read), and
|
|
2994
|
+
* one batched read of every linked run.
|
|
2995
|
+
*/
|
|
2996
|
+
async lifecycleDetail(run) {
|
|
2997
|
+
const namespace = normalizeNamespace(run.namespace);
|
|
2998
|
+
const kind = lifecycleKindOf(run.status);
|
|
2999
|
+
const subscribers = kind ? this.lifecycleSubscribers(run.workflow, kind, namespace) : [];
|
|
3000
|
+
const links = await this.workflowLifecycle(run.workflow);
|
|
3001
|
+
const deliveries = kind
|
|
3002
|
+
? lifecycleDeliveriesOf(run, {
|
|
3003
|
+
subscribers,
|
|
3004
|
+
live: new Set(subscribers),
|
|
3005
|
+
coalesced: this.coalescedSubscribers(run, kind),
|
|
3006
|
+
})
|
|
3007
|
+
: [];
|
|
3008
|
+
const [own, onIt] = await Promise.all([
|
|
3009
|
+
run.status === 'suspended' ? listSignalWaitersOfRuns(this.store, [run.id]) : [],
|
|
3010
|
+
this.store.listSignalWaiters(runWaitPrefix(run.id)),
|
|
3011
|
+
]);
|
|
3012
|
+
const waitsFor = own.flatMap((w) => {
|
|
3013
|
+
const target = runWaitTargetOf(w);
|
|
3014
|
+
return target !== undefined ? [{ runId: target, seq: w.seq }] : [];
|
|
3015
|
+
});
|
|
3016
|
+
const awaitedBy = onIt
|
|
3017
|
+
.filter((w) => runWaitTargetOf(w) === run.id)
|
|
3018
|
+
.map((w) => ({ runId: w.runId, seq: w.seq }));
|
|
3019
|
+
const started = deliveries.flatMap((d) => d.state === 'started' && d.runId !== undefined ? [d.runId] : []);
|
|
3020
|
+
const source = lifecycleSourceOf(run);
|
|
3021
|
+
const linked = await getRunsByIds(this.store, [
|
|
3022
|
+
...(source ? [source.runId] : []),
|
|
3023
|
+
...started,
|
|
3024
|
+
...waitsFor.map((l) => l.runId),
|
|
3025
|
+
...awaitedBy.map((l) => l.runId),
|
|
3026
|
+
]);
|
|
3027
|
+
const withRun = (link) => {
|
|
3028
|
+
const other = link.runId !== undefined ? linked.get(link.runId) : undefined;
|
|
3029
|
+
return other ? { ...link, status: other.status, namespace: other.namespace } : link;
|
|
3030
|
+
};
|
|
3031
|
+
const sourceRun = source ? linked.get(source.runId) : undefined;
|
|
3032
|
+
const sourceTrace = lifecycleTraceparentOf(sourceRun?.tags)?.split('-')[1];
|
|
3033
|
+
return {
|
|
3034
|
+
...(source
|
|
3035
|
+
? {
|
|
3036
|
+
triggeredBy: {
|
|
3037
|
+
...source,
|
|
3038
|
+
...(sourceRun ? { status: sourceRun.status, namespace: sourceRun.namespace } : {}),
|
|
3039
|
+
...(sourceTrace ? { traceId: sourceTrace } : {}),
|
|
3040
|
+
},
|
|
3041
|
+
}
|
|
3042
|
+
: {}),
|
|
3043
|
+
...(subscribers.length > 0 ? { lifecycleSubscribers: subscribers } : {}),
|
|
3044
|
+
...(links.startsOnSettle.length > 0 || links.startedBy.length > 0
|
|
3045
|
+
? { workflowLifecycle: links }
|
|
3046
|
+
: {}),
|
|
3047
|
+
...(deliveries.length > 0
|
|
3048
|
+
? {
|
|
3049
|
+
lifecycleDeliveries: deliveries.map((d) => (d.state === 'started' ? withRun(d) : d)),
|
|
3050
|
+
}
|
|
3051
|
+
: {}),
|
|
3052
|
+
...(waitsFor.length > 0 ? { waitsFor: waitsFor.map(withRun) } : {}),
|
|
3053
|
+
...(awaitedBy.length > 0 ? { awaitedBy: awaitedBy.map(withRun) } : {}),
|
|
3054
|
+
};
|
|
1714
3055
|
}
|
|
1715
3056
|
/**
|
|
1716
|
-
*
|
|
1717
|
-
*
|
|
1718
|
-
*
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
3057
|
+
* The subscribers registered here that coalesce `run`'s lifecycle event (debounce/batch) and
|
|
3058
|
+
* accept it, each with the accumulator run the event went to: the subscription's `where`, `input`
|
|
3059
|
+
* and `key` applied to the event as `startSubscribers` does. Undefined when the key cannot be
|
|
3060
|
+
* computed. A subscriber whose `where` declines the event is left out.
|
|
3061
|
+
*/
|
|
3062
|
+
coalescedSubscribers(run, kind) {
|
|
3063
|
+
const out = new Map();
|
|
3064
|
+
const event = lifecycleEventName(run.workflow, kind);
|
|
3065
|
+
const triggers = this.eventTriggers.get(event);
|
|
3066
|
+
if (!triggers)
|
|
3067
|
+
return out;
|
|
3068
|
+
let payload;
|
|
3069
|
+
for (const [workflow, subscriptions] of triggers) {
|
|
3070
|
+
const batch = this.latest.get(workflow)?.eventBatch;
|
|
3071
|
+
if (!batch)
|
|
3072
|
+
continue;
|
|
3073
|
+
if (!batch.key && subscriptions.every((sub) => !sub.where)) {
|
|
3074
|
+
out.set(workflow, accumulatorRunId(workflow));
|
|
3075
|
+
continue;
|
|
3076
|
+
}
|
|
3077
|
+
try {
|
|
3078
|
+
payload ??= lifecycleEventOf(run);
|
|
3079
|
+
const { accepted } = acceptingSubscription(subscriptions, payload);
|
|
3080
|
+
if (!accepted)
|
|
3081
|
+
continue;
|
|
3082
|
+
const input = accepted.input ? accepted.input(payload) : payload;
|
|
3083
|
+
const key = batch.key ? batch.key(input) : undefined;
|
|
3084
|
+
out.set(workflow, key === undefined || typeof key === 'string'
|
|
3085
|
+
? accumulatorRunId(workflow, key)
|
|
3086
|
+
: undefined);
|
|
3087
|
+
}
|
|
3088
|
+
catch {
|
|
3089
|
+
out.set(workflow, undefined);
|
|
3090
|
+
}
|
|
3091
|
+
}
|
|
3092
|
+
return out;
|
|
1722
3093
|
}
|
|
1723
3094
|
/** One page of a summarized fan's members (see `loadFanMembers`), versioned like {@link getRunDetail}. */
|
|
1724
3095
|
getFanMembers(runId, group, query) {
|
|
@@ -1961,7 +3332,7 @@ export class WorkflowEngine {
|
|
|
1961
3332
|
passes.push(scoped);
|
|
1962
3333
|
}
|
|
1963
3334
|
else {
|
|
1964
|
-
const [lastKept] = await this.store.listRuns({ ...scoped, page: maxCount, size: 1 });
|
|
3335
|
+
const [lastKept] = await this.store.listRuns(withoutBookkeepingRuns({ ...scoped, page: maxCount, size: 1 }));
|
|
1965
3336
|
if (lastKept) {
|
|
1966
3337
|
passes.push({ ...scoped, createdBefore: lastKept.createdAt.getTime() - 1 });
|
|
1967
3338
|
}
|
|
@@ -1969,9 +3340,13 @@ export class WorkflowEngine {
|
|
|
1969
3340
|
}
|
|
1970
3341
|
}
|
|
1971
3342
|
for (const pass of passes) {
|
|
1972
|
-
const candidates = await this.store.listRuns({ ...pass, size: 100 });
|
|
3343
|
+
const candidates = await this.store.listRuns(withoutBookkeepingRuns({ ...pass, size: 100 }));
|
|
1973
3344
|
const archived = [];
|
|
1974
3345
|
for (const run of candidates) {
|
|
3346
|
+
// A run whose lifecycle event is still owed is kept until it is published, and a registry
|
|
3347
|
+
// run is replaced by its engine's heartbeat instead.
|
|
3348
|
+
if (retentionKeeps(run.tags) || run.workflow === LIFECYCLE_REGISTRY_WORKFLOW)
|
|
3349
|
+
continue;
|
|
1975
3350
|
try {
|
|
1976
3351
|
if (this.evictListeners.size > 0) {
|
|
1977
3352
|
const checkpoints = await this.store.listCheckpoints(run.id);
|
|
@@ -2109,7 +3484,7 @@ export class WorkflowEngine {
|
|
|
2109
3484
|
let deleted = 0;
|
|
2110
3485
|
let previous = '';
|
|
2111
3486
|
for (;;) {
|
|
2112
|
-
const batch = await this.store.listRuns({ ...scope, statuses, size });
|
|
3487
|
+
const batch = await this.store.listRuns(withoutBookkeepingRuns({ ...scope, statuses, size }));
|
|
2113
3488
|
if (batch.length === 0)
|
|
2114
3489
|
break;
|
|
2115
3490
|
const fingerprint = batch.map((r) => r.id).join('\u0000');
|
|
@@ -2259,7 +3634,38 @@ export class WorkflowEngine {
|
|
|
2259
3634
|
});
|
|
2260
3635
|
}
|
|
2261
3636
|
async resumeRunNow(runId, allowFailed) {
|
|
2262
|
-
|
|
3637
|
+
// A run just claimed for this turn (runPending) is held, and was read by the claim.
|
|
3638
|
+
const claimed = this.claimedRuns.get(runId);
|
|
3639
|
+
if (claimed) {
|
|
3640
|
+
this.claimedRuns.delete(runId);
|
|
3641
|
+
return this.resumeLoadedRun(claimed, runId, allowFailed);
|
|
3642
|
+
}
|
|
3643
|
+
// Take the lease and read the run in one round trip when the store can: `execute` then finds
|
|
3644
|
+
// the lease held for this turn. Not when a caller already holds it for this turn (a leased
|
|
3645
|
+
// sweep) or a turn of the run is in flight here; a failed take reads the run as before.
|
|
3646
|
+
const lockAndGet = this.lockedForTurn.has(runId) || this.executionLeases.has(runId)
|
|
3647
|
+
? undefined
|
|
3648
|
+
: this.store.tryLockRunAndGet;
|
|
3649
|
+
let run = null;
|
|
3650
|
+
if (lockAndGet) {
|
|
3651
|
+
const nowMs = this.clock();
|
|
3652
|
+
run = await lockAndGet.call(this.store, runId, this.instanceId, nowMs + this.leaseMs, nowMs);
|
|
3653
|
+
}
|
|
3654
|
+
if (!run)
|
|
3655
|
+
return this.resumeLoadedRun(await this.store.getRun(runId), runId, allowFailed);
|
|
3656
|
+
this.lockedForTurn.add(runId);
|
|
3657
|
+
try {
|
|
3658
|
+
return await this.resumeLoadedRun(run, runId, allowFailed);
|
|
3659
|
+
}
|
|
3660
|
+
finally {
|
|
3661
|
+
// Still marked: the turn never started (a finished run, another namespace, a registration
|
|
3662
|
+
// that failed), so the lease taken above goes back.
|
|
3663
|
+
if (this.lockedForTurn.delete(runId)) {
|
|
3664
|
+
await this.store.releaseRunLock(runId, this.instanceId).catch(() => undefined);
|
|
3665
|
+
}
|
|
3666
|
+
}
|
|
3667
|
+
}
|
|
3668
|
+
async resumeLoadedRun(run, runId, allowFailed) {
|
|
2263
3669
|
if (!run)
|
|
2264
3670
|
throw new Error(`run ${runId} not found`);
|
|
2265
3671
|
// A definitively-finished run must not be re-executed (e.g. a worker result landing after the
|
|
@@ -2400,6 +3806,8 @@ export class WorkflowEngine {
|
|
|
2400
3806
|
*/
|
|
2401
3807
|
beginDrain(spentMs = 0) {
|
|
2402
3808
|
this.draining = true;
|
|
3809
|
+
// Parked bodies are only an optimization: the runs replay wherever they resume next.
|
|
3810
|
+
this.sticky.clear();
|
|
2403
3811
|
// Tell the local steps still running: a long one can stop between units of work instead of
|
|
2404
3812
|
// being cut off when the drain budget runs out.
|
|
2405
3813
|
if (!this.drainAbort.signal.aborted)
|
|
@@ -2550,11 +3958,11 @@ export class WorkflowEngine {
|
|
|
2550
3958
|
// (and the listRuns calls above it), so this run could have settled on its own — via its own
|
|
2551
3959
|
// in-flight turn, an operator cancel, or another sweep — in the gap. Only clobber it into
|
|
2552
3960
|
// `cancelled` if it's still genuinely `running`/`suspended`.
|
|
2553
|
-
const applied = await this.store.updateRunIf(run.id, ['running', 'suspended'], {
|
|
3961
|
+
const applied = await this.store.updateRunIf(run.id, ['running', 'suspended'], this.withLifecycleMarker(run, {
|
|
2554
3962
|
status: 'cancelled',
|
|
2555
3963
|
error,
|
|
2556
3964
|
updatedAt: new Date(),
|
|
2557
|
-
});
|
|
3965
|
+
}));
|
|
2558
3966
|
if (!applied)
|
|
2559
3967
|
continue;
|
|
2560
3968
|
this.emit({
|
|
@@ -2878,11 +4286,11 @@ export class WorkflowEngine {
|
|
|
2878
4286
|
* (e.g. a lease-free `cancel()`), so it is not ours to touch.
|
|
2879
4287
|
*/
|
|
2880
4288
|
async deadLetterLeased(run, from, error) {
|
|
2881
|
-
const applied = await this.store.updateRunIf(run.id, from, {
|
|
4289
|
+
const applied = await this.store.updateRunIf(run.id, from, this.withLifecycleMarker(run, {
|
|
2882
4290
|
status: 'dead',
|
|
2883
4291
|
error,
|
|
2884
4292
|
updatedAt: new Date(),
|
|
2885
|
-
});
|
|
4293
|
+
}));
|
|
2886
4294
|
await this.store.releaseRunLock(run.id, this.instanceId);
|
|
2887
4295
|
if (!applied)
|
|
2888
4296
|
return null;
|
|
@@ -3311,9 +4719,58 @@ export class WorkflowEngine {
|
|
|
3311
4719
|
async runPending(nowMs = this.clock()) {
|
|
3312
4720
|
// Oldest-first (FIFO), capped per call so a backlog drains over several polls without one sweep
|
|
3313
4721
|
// fetching unboundedly. A run not picked up this tick is picked up the next.
|
|
3314
|
-
|
|
4722
|
+
// A store that can claims a batch in one statement (rows another instance is claiming are
|
|
4723
|
+
// skipped, not raced for), and the batch runs concurrently, within this engine's run budget.
|
|
4724
|
+
const claim = this.store.claimPendingRuns;
|
|
4725
|
+
if (claim) {
|
|
4726
|
+
if (this.draining)
|
|
4727
|
+
return [];
|
|
4728
|
+
const room = Math.min(PENDING_POLL_BATCH, LOCAL_RUN_CONCURRENCY - this.inflight.size);
|
|
4729
|
+
if (room <= 0)
|
|
4730
|
+
return [];
|
|
4731
|
+
const claimed = await asHeartbeat(() => claim.call(this.store, room, this.instanceId, nowMs + this.leaseMs, nowMs, this.namespace));
|
|
4732
|
+
return this.runClaimed(claimed, nowMs);
|
|
4733
|
+
}
|
|
4734
|
+
const pending = await asHeartbeat(() => this.store.listPendingRuns(PENDING_POLL_BATCH, this.namespace));
|
|
3315
4735
|
return this.resumeLeased(pending, nowMs);
|
|
3316
4736
|
}
|
|
4737
|
+
/**
|
|
4738
|
+
* Run `claimed` (pending runs this instance just leased, read as locked): every turn starts at
|
|
4739
|
+
* once, and each re-enters the lease taken for it without reading the run again. A run this
|
|
4740
|
+
* instance cannot resume is skipped and logged, as {@link resumeLeased} does; its lease lapses.
|
|
4741
|
+
*/
|
|
4742
|
+
async runClaimed(claimed, nowMs) {
|
|
4743
|
+
const turns = claimed.map(async (run) => {
|
|
4744
|
+
if (this.draining) {
|
|
4745
|
+
// Drain began after the claim: give the lease back; the run stays for another instance.
|
|
4746
|
+
await this.store.releaseRunLock(run.id, this.instanceId).catch(() => undefined);
|
|
4747
|
+
return undefined;
|
|
4748
|
+
}
|
|
4749
|
+
this.lockedForTurn.add(run.id);
|
|
4750
|
+
this.claimedRuns.set(run.id, run);
|
|
4751
|
+
try {
|
|
4752
|
+
return await this.resumeImplicit(run.id);
|
|
4753
|
+
}
|
|
4754
|
+
catch (error) {
|
|
4755
|
+
if (error instanceof NamespaceMismatch)
|
|
4756
|
+
return undefined;
|
|
4757
|
+
if (error instanceof WorkflowNotRegistered) {
|
|
4758
|
+
const settled = await this.settleUnregistered(run, nowMs).catch(() => undefined);
|
|
4759
|
+
if (settled)
|
|
4760
|
+
return settled;
|
|
4761
|
+
if (settled !== undefined)
|
|
4762
|
+
return undefined;
|
|
4763
|
+
}
|
|
4764
|
+
console.error(`[adonis-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance or a later tick:`, error);
|
|
4765
|
+
return undefined;
|
|
4766
|
+
}
|
|
4767
|
+
finally {
|
|
4768
|
+
this.lockedForTurn.delete(run.id);
|
|
4769
|
+
this.claimedRuns.delete(run.id);
|
|
4770
|
+
}
|
|
4771
|
+
});
|
|
4772
|
+
return (await Promise.all(turns)).filter((r) => r !== undefined);
|
|
4773
|
+
}
|
|
3317
4774
|
/**
|
|
3318
4775
|
* Resolve once `runId` reaches a settled state — terminal (completed/failed/cancelled/dead) or
|
|
3319
4776
|
* suspended (handed off to a timer/signal/event). The async counterpart to dispatch: pair it with
|
|
@@ -3462,7 +4919,9 @@ export class WorkflowEngine {
|
|
|
3462
4919
|
* match the payload satisfies (fan-out, vs `signal`'s point-to-point token).
|
|
3463
4920
|
* 2. **Starts** a fresh run of every workflow registered with `onEvent: [name]`, passing the
|
|
3464
4921
|
* payload as input. Idempotent by `evt:<id>:<workflow>` — pass `opts.id` to dedupe redeliveries
|
|
3465
|
-
* of the same logical event (default: a fresh uuid, so each publish triggers once).
|
|
4922
|
+
* of the same logical event (default: a fresh uuid, so each publish triggers once). A workflow
|
|
4923
|
+
* a non-Node worker announces as subscribing to `name` (a Python `on_event=[...]`) is started
|
|
4924
|
+
* the same way, by name, when this engine can reach it (see {@link announcedRemoteSubscribers}).
|
|
3466
4925
|
*
|
|
3467
4926
|
* Reliable (buffered) delivery — mirrors `signalWithStart`'s reliability contract, so document it
|
|
3468
4927
|
* exactly: a publish that resumes ≥1 live waiter, OR routes into an `eventBatch` accumulator /
|
|
@@ -3484,27 +4943,14 @@ export class WorkflowEngine {
|
|
|
3484
4943
|
touched += 1;
|
|
3485
4944
|
}
|
|
3486
4945
|
}
|
|
3487
|
-
|
|
3488
|
-
|
|
4946
|
+
await this.freshAnnouncedSubscriptions();
|
|
4947
|
+
const remote = await this.announcedRemoteSubscribers(name);
|
|
4948
|
+
if (this.eventTriggers.get(name)?.size || remote.length > 0) {
|
|
3489
4949
|
const eventId = opts?.id ?? globalThis.crypto.randomUUID();
|
|
3490
|
-
|
|
3491
|
-
|
|
3492
|
-
|
|
3493
|
-
|
|
3494
|
-
const batch = this.latest.get(workflow)?.eventBatch;
|
|
3495
|
-
if (batch) {
|
|
3496
|
-
// Coalesce: route the event into a per-workflow accumulator (one long-lived run that
|
|
3497
|
-
// debounces/batches and then starts the target with the collected payload(s)).
|
|
3498
|
-
await this.accumulators.route(workflow, batch, payload);
|
|
3499
|
-
}
|
|
3500
|
-
else {
|
|
3501
|
-
await this.start(workflow, payload, `evt:${eventId}:${workflow}`);
|
|
3502
|
-
}
|
|
3503
|
-
touched += 1;
|
|
3504
|
-
}
|
|
3505
|
-
catch {
|
|
3506
|
-
// skip this subscriber
|
|
3507
|
-
}
|
|
4950
|
+
const { reached, failed } = await this.startSubscribers(name, payload, (workflow) => `evt:${eventId}:${workflow}`, undefined, remote);
|
|
4951
|
+
touched += reached.length;
|
|
4952
|
+
for (const { workflow, error } of failed) {
|
|
4953
|
+
console.warn(`[adonis-durable] event ${name}: subscriber ${workflow} not started:`, error);
|
|
3508
4954
|
}
|
|
3509
4955
|
}
|
|
3510
4956
|
// Nobody received it live (no waiter matched, no subscriber exists/started) — buffer ONE copy
|
|
@@ -3852,12 +5298,12 @@ export class WorkflowEngine {
|
|
|
3852
5298
|
}
|
|
3853
5299
|
const error = { message: 'cancelled' };
|
|
3854
5300
|
// Clears a remote turn's decision marker too: a cancelled run awaits no decision.
|
|
3855
|
-
await this.store.updateRun(runId, {
|
|
5301
|
+
await this.store.updateRun(runId, this.withLifecycleMarker(run, {
|
|
3856
5302
|
status: 'cancelled',
|
|
3857
5303
|
error,
|
|
3858
5304
|
awaitingDecisionTaskId: undefined,
|
|
3859
5305
|
updatedAt: new Date(),
|
|
3860
|
-
});
|
|
5306
|
+
}));
|
|
3861
5307
|
this.emit({
|
|
3862
5308
|
type: 'run.failed',
|
|
3863
5309
|
runId,
|
|
@@ -3964,7 +5410,7 @@ export class WorkflowEngine {
|
|
|
3964
5410
|
* the full listing for a store without one.
|
|
3965
5411
|
*/
|
|
3966
5412
|
countRuns(query) {
|
|
3967
|
-
return countRuns(this.store, query);
|
|
5413
|
+
return countRuns(this.store, withoutBookkeepingRuns(query));
|
|
3968
5414
|
}
|
|
3969
5415
|
/**
|
|
3970
5416
|
* Every workflow the live fleet can execute in this engine's partition: what workers DECLARE in
|
|
@@ -4365,7 +5811,7 @@ export class WorkflowEngine {
|
|
|
4365
5811
|
(run.lockedUntil ?? 0) > lockNow &&
|
|
4366
5812
|
(lockedForTurn || (await this.proveOwnLease(run.id, lockNow)));
|
|
4367
5813
|
if (!reentered) {
|
|
4368
|
-
if (!(await this.
|
|
5814
|
+
if (!(await this.lockRunPastLifecyclePublish(run.id, lockNow))) {
|
|
4369
5815
|
const stamping = this.draining ? undefined : this.stampWakeRequest(run.id, reason);
|
|
4370
5816
|
this.trackEffect(this.requestWake(run.id, run.lockedUntil, reason, stamping));
|
|
4371
5817
|
const refused = { runId: run.id, status: run.status };
|
|
@@ -4639,12 +6085,12 @@ export class WorkflowEngine {
|
|
|
4639
6085
|
// resurrect/clobber it — the conditional write (status still one of SETTLE_ELIGIBLE_STATUSES) is
|
|
4640
6086
|
// the guard; the database, not a prior JS read, arbitrates. Also clears any error from an
|
|
4641
6087
|
// earlier failed-then-retried attempt — a completed run is a success.
|
|
4642
|
-
const applied = await this.store.updateRunIf(run.id, SETTLE_ELIGIBLE_STATUSES, {
|
|
6088
|
+
const applied = await this.store.updateRunIf(run.id, SETTLE_ELIGIBLE_STATUSES, this.withLifecycleMarker(run, {
|
|
4643
6089
|
status: 'completed',
|
|
4644
6090
|
output: outcome.output,
|
|
4645
6091
|
error: undefined,
|
|
4646
6092
|
updatedAt,
|
|
4647
|
-
});
|
|
6093
|
+
}));
|
|
4648
6094
|
if (!applied)
|
|
4649
6095
|
return this.echoCurrentStatus(run.id);
|
|
4650
6096
|
this.emit({
|
|
@@ -4670,11 +6116,11 @@ export class WorkflowEngine {
|
|
|
4670
6116
|
// (observed: a cascade-requeued child's fresh success lost to the stale buffered failure). A
|
|
4671
6117
|
// legitimate retry always transitions through `pending`/`running` first (requeue flips the
|
|
4672
6118
|
// status), so it is unaffected.
|
|
4673
|
-
const applied = await this.store.updateRunIf(run.id, SETTLE_ELIGIBLE_STATUSES.filter((s) => s !== 'failed'), {
|
|
6119
|
+
const applied = await this.store.updateRunIf(run.id, SETTLE_ELIGIBLE_STATUSES.filter((s) => s !== 'failed'), this.withLifecycleMarker(run, {
|
|
4674
6120
|
status: 'failed',
|
|
4675
6121
|
error,
|
|
4676
6122
|
updatedAt,
|
|
4677
|
-
});
|
|
6123
|
+
}));
|
|
4678
6124
|
if (!applied)
|
|
4679
6125
|
return this.echoCurrentStatus(run.id);
|
|
4680
6126
|
this.emit({
|
|
@@ -4694,13 +6140,20 @@ export class WorkflowEngine {
|
|
|
4694
6140
|
// conditional write IS the guard — it subsumes the read-then-write this branch used to do (that
|
|
4695
6141
|
// was itself a narrower, TOCTOU-prone version of the same check, and only looked for `cancelled`;
|
|
4696
6142
|
// this now also catches `completed`/`dead` reached via any other path mid-turn).
|
|
4697
|
-
|
|
6143
|
+
// Strictly later than the row this turn read: a parked body is recognized by the exact stamp its
|
|
6144
|
+
// suspend wrote (see takeContinuation), so no later settle, by any instance with any clock, may
|
|
6145
|
+
// write that same stamp again, even within the same millisecond.
|
|
6146
|
+
const stamp = new Date(Math.max(updatedAt.getTime(), run.updatedAt.getTime() + 1));
|
|
6147
|
+
const applied = await this.store.updateRunIf(run.id, SETTLE_ELIGIBLE_STATUSES, await this.withRunTrace(run, {
|
|
4698
6148
|
status: 'suspended',
|
|
4699
6149
|
wakeAt: this.reconcileWakeAt(outcome.wakeAt),
|
|
4700
|
-
updatedAt,
|
|
4701
|
-
});
|
|
6150
|
+
updatedAt: stamp,
|
|
6151
|
+
}));
|
|
4702
6152
|
if (!applied)
|
|
4703
6153
|
return this.echoCurrentStatus(run.id);
|
|
6154
|
+
run.updatedAt = stamp;
|
|
6155
|
+
if (this.turnContinuations.has(run.id))
|
|
6156
|
+
this.suspendStamps.set(run.id, stamp.getTime());
|
|
4704
6157
|
this.emit({
|
|
4705
6158
|
type: 'run.suspended',
|
|
4706
6159
|
runId: run.id,
|
|
@@ -4745,7 +6198,8 @@ export class WorkflowEngine {
|
|
|
4745
6198
|
* settled it), so report what is persisted rather than writing over it.
|
|
4746
6199
|
*/
|
|
4747
6200
|
/**
|
|
4748
|
-
* A run's first execution: `pending` -> `running
|
|
6201
|
+
* A run's first execution: `pending` -> `running`, resolving whether it moved it (a run cancelled
|
|
6202
|
+
* or deleted since it was read did not move). A singleton run not yet admitted is stamped
|
|
4749
6203
|
* {@link SINGLETON_TRACKED_TAG} in the same write, BEFORE it competes for a slot, so a run this
|
|
4750
6204
|
* replica is about to admit or deny is never mistaken for a holder admitted by a replica older than
|
|
4751
6205
|
* 0.51 (see `legacySingletonHolders`).
|
|
@@ -4753,14 +6207,19 @@ export class WorkflowEngine {
|
|
|
4753
6207
|
async markStarted(run, singleton) {
|
|
4754
6208
|
const tags = singleton ? withSingletonTracked(run.tags) : run.tags;
|
|
4755
6209
|
const retag = tags !== run.tags;
|
|
4756
|
-
|
|
6210
|
+
// A compare-and-set on `pending`: the run was read before this turn took it, and a cancel (or a
|
|
6211
|
+
// delete) landing in between must not be overwritten with `running`, which would resurrect it.
|
|
6212
|
+
const started = await this.store.updateRunIf(run.id, ['pending'], {
|
|
4757
6213
|
status: 'running',
|
|
4758
6214
|
...(retag ? { tags } : {}),
|
|
4759
6215
|
updatedAt: new Date(),
|
|
4760
6216
|
});
|
|
6217
|
+
if (!started)
|
|
6218
|
+
return false;
|
|
4761
6219
|
run.status = 'running';
|
|
4762
6220
|
if (retag)
|
|
4763
6221
|
run.tags = tags;
|
|
6222
|
+
return true;
|
|
4764
6223
|
}
|
|
4765
6224
|
async singletonWaitResult(run) {
|
|
4766
6225
|
const current = await this.store.getRun(run.id);
|
|
@@ -4804,12 +6263,17 @@ export class WorkflowEngine {
|
|
|
4804
6263
|
return { runId: run.id, status: 'suspended' };
|
|
4805
6264
|
}
|
|
4806
6265
|
if (run.status === 'pending') {
|
|
4807
|
-
await this.markStarted(run, registered?.singleton !== undefined)
|
|
6266
|
+
if (!(await this.markStarted(run, registered?.singleton !== undefined))) {
|
|
6267
|
+
// No longer `pending` (cancelled or deleted since it was read): nothing to run.
|
|
6268
|
+
await this.store.releaseRunLock(run.id, this.instanceId).catch(() => undefined);
|
|
6269
|
+
return await this.echoCurrentStatus(run.id);
|
|
6270
|
+
}
|
|
4808
6271
|
this.emit({
|
|
4809
6272
|
type: 'run.started',
|
|
4810
6273
|
runId: run.id,
|
|
4811
6274
|
workflow: run.workflow,
|
|
4812
6275
|
namespace: run.namespace,
|
|
6276
|
+
...(await this.sourceTraceOf(run)),
|
|
4813
6277
|
});
|
|
4814
6278
|
}
|
|
4815
6279
|
if (registered.singleton && !(await this.singletons.admit(run, registered.singleton))) {
|
|
@@ -4818,7 +6282,14 @@ export class WorkflowEngine {
|
|
|
4818
6282
|
// Capability/protocol guard (design §7.5): a remote workflow's turn dispatches a WorkflowTask to
|
|
4819
6283
|
// its group — if no live worker on that group can run it, park `blocked` before advancing rather
|
|
4820
6284
|
// than handing the task to the executor to enqueue into a void. Re-checked each turn (recovery).
|
|
4821
|
-
const
|
|
6285
|
+
const routeToken = registered.convention
|
|
6286
|
+
? conventionGroup(run.workflow, run.namespace)
|
|
6287
|
+
: remote.group;
|
|
6288
|
+
let routePlan = await this.planRoute(routeToken, registered.requires);
|
|
6289
|
+
// As in ensureRoutable: re-read before parking, never park on reused descriptors.
|
|
6290
|
+
if (routePlan?.status === 'blocked') {
|
|
6291
|
+
routePlan = await this.planRoute(routeToken, registered.requires, true);
|
|
6292
|
+
}
|
|
4822
6293
|
if (routePlan?.status === 'blocked')
|
|
4823
6294
|
return this.parkBlocked(run, routePlan);
|
|
4824
6295
|
const checkpoints = await this.replayCheckpoints(run.id);
|
|
@@ -4883,7 +6354,7 @@ export class WorkflowEngine {
|
|
|
4883
6354
|
message: err instanceof Error ? err.message : String(err),
|
|
4884
6355
|
stack: err instanceof Error ? err.stack : undefined,
|
|
4885
6356
|
};
|
|
4886
|
-
await this.store.updateRun(run.id, { status: 'failed', error, updatedAt: new Date() });
|
|
6357
|
+
await this.store.updateRun(run.id, this.withLifecycleMarker(run, { status: 'failed', error, updatedAt: new Date() }));
|
|
4887
6358
|
this.emit({
|
|
4888
6359
|
type: 'run.failed',
|
|
4889
6360
|
runId: run.id,
|
|
@@ -4965,12 +6436,12 @@ export class WorkflowEngine {
|
|
|
4965
6436
|
// id (the `db` transport dedupes a turn by it, and the marker gates the decision on it).
|
|
4966
6437
|
const taskId = `${run.id}:wf:${globalThis.crypto.randomUUID()}`;
|
|
4967
6438
|
const windowMs = this.decisionWindowMs(registered);
|
|
4968
|
-
const marked = await this.store.updateRunIf(run.id, SETTLE_ELIGIBLE_STATUSES, {
|
|
6439
|
+
const marked = await this.store.updateRunIf(run.id, SETTLE_ELIGIBLE_STATUSES, await this.withRunTrace(run, {
|
|
4969
6440
|
status: 'suspended',
|
|
4970
6441
|
awaitingDecisionTaskId: taskId,
|
|
4971
6442
|
wakeAt: windowMs != null ? this.clock() + Math.min(windowMs, MAX_BACKOFF_MS) : undefined,
|
|
4972
6443
|
updatedAt: new Date(),
|
|
4973
|
-
});
|
|
6444
|
+
}));
|
|
4974
6445
|
if (!marked) {
|
|
4975
6446
|
// Cancelled / completed / dead while this turn was being prepared: nothing to dispatch.
|
|
4976
6447
|
await this.releaseTurnLease(run.id);
|
|
@@ -5004,13 +6475,13 @@ export class WorkflowEngine {
|
|
|
5004
6475
|
const current = await this.store.getRun(run.id);
|
|
5005
6476
|
if (current?.awaitingDecisionTaskId !== taskId)
|
|
5006
6477
|
return this.echoCurrentStatus(run.id);
|
|
5007
|
-
const failed = await this.store.updateRunIf(run.id, ['suspended'], {
|
|
6478
|
+
const failed = await this.store.updateRunIf(run.id, ['suspended'], this.withLifecycleMarker(run, {
|
|
5008
6479
|
status: 'failed',
|
|
5009
6480
|
error,
|
|
5010
6481
|
awaitingDecisionTaskId: undefined,
|
|
5011
6482
|
wakeAt: undefined,
|
|
5012
6483
|
updatedAt: new Date(),
|
|
5013
|
-
});
|
|
6484
|
+
}));
|
|
5014
6485
|
if (!failed)
|
|
5015
6486
|
return this.echoCurrentStatus(run.id);
|
|
5016
6487
|
this.emit({
|
|
@@ -5822,12 +7293,12 @@ export class WorkflowEngine {
|
|
|
5822
7293
|
await this.runCompensation(run, comp, i);
|
|
5823
7294
|
}
|
|
5824
7295
|
const error = { message: 'cancelled' };
|
|
5825
|
-
await this.store.updateRun(run.id, {
|
|
7296
|
+
await this.store.updateRun(run.id, this.withLifecycleMarker(run, {
|
|
5826
7297
|
status: 'cancelled',
|
|
5827
7298
|
error,
|
|
5828
7299
|
awaitingDecisionTaskId: undefined,
|
|
5829
7300
|
updatedAt: new Date(),
|
|
5830
|
-
});
|
|
7301
|
+
}));
|
|
5831
7302
|
// The cancellation is durably recorded — consume the marker(s) so a later retry of this run id
|
|
5832
7303
|
// doesn't get spuriously re-cancelled.
|
|
5833
7304
|
for (;;) {
|
|
@@ -5864,12 +7335,17 @@ export class WorkflowEngine {
|
|
|
5864
7335
|
// gate, so an admitted run's body never starts while it still reads `pending`. A resumed run is
|
|
5865
7336
|
// already past `pending`, so this fires exactly once.
|
|
5866
7337
|
if (run.status === 'pending') {
|
|
5867
|
-
await this.markStarted(run, registered?.singleton !== undefined)
|
|
7338
|
+
if (!(await this.markStarted(run, registered?.singleton !== undefined))) {
|
|
7339
|
+
// No longer `pending` (cancelled or deleted since it was read): nothing to run.
|
|
7340
|
+
await this.store.releaseRunLock(run.id, this.instanceId).catch(() => undefined);
|
|
7341
|
+
return await this.echoCurrentStatus(run.id);
|
|
7342
|
+
}
|
|
5868
7343
|
this.emit({
|
|
5869
7344
|
type: 'run.started',
|
|
5870
7345
|
runId: run.id,
|
|
5871
7346
|
workflow: run.workflow,
|
|
5872
7347
|
namespace: run.namespace,
|
|
7348
|
+
...(await this.sourceTraceOf(run)),
|
|
5873
7349
|
});
|
|
5874
7350
|
}
|
|
5875
7351
|
// Singleton admission gate: if this run shares its key with `limit` older in-flight runs, wait
|
|
@@ -5877,20 +7353,37 @@ export class WorkflowEngine {
|
|
|
5877
7353
|
if (registered?.singleton && !(await this.singletons.admit(run, registered.singleton))) {
|
|
5878
7354
|
return this.singletonWaitResult(run);
|
|
5879
7355
|
}
|
|
7356
|
+
// A body this instance parked on dispatched steps in an earlier turn continues from there (see
|
|
7357
|
+
// `sticky-continuation.ts`), when nothing but this instance's own turn has settled the run since.
|
|
7358
|
+
const stickyOk = this.stickyEligible(run);
|
|
7359
|
+
let cont = stickyOk ? this.takeContinuation(run, fn) : undefined;
|
|
7360
|
+
const warm = cont !== undefined;
|
|
7361
|
+
// The parked steps' checkpoints, read before the body resumes: a failing read fails the turn
|
|
7362
|
+
// like a failing history read does, never the run.
|
|
7363
|
+
const parkedFresh = cont ? await this.readParked(run.id, cont) : undefined;
|
|
5880
7364
|
// Saga compensations registered by completed steps; run in reverse if the run later fails.
|
|
5881
|
-
const compensations = [];
|
|
7365
|
+
const compensations = cont ? cont.compensations : [];
|
|
5882
7366
|
// Load this run's checkpoints ONCE and key them by seq, so replaying the completed prefix reads
|
|
5883
7367
|
// from memory instead of one `getCheckpoint` SELECT per primitive (the O(N²) replay-reads fix).
|
|
5884
7368
|
// Read once at execution start: a checkpoint written AFTER this snapshot (the signal/timer/child
|
|
5885
7369
|
// this resume wakes on, or one written later in this same execution) is absent from the map, and
|
|
5886
7370
|
// the ctx falls back to the live store for any absent seq — so replay semantics are unchanged.
|
|
5887
|
-
|
|
5888
|
-
const
|
|
7371
|
+
// A warm continuation already replayed its prefix: it reads only what lies past its park point.
|
|
7372
|
+
const snapshot = cont ? [] : await this.replayCheckpoints(run.id);
|
|
7373
|
+
const replay = cont ? cont.replay : new Map();
|
|
5889
7374
|
for (const cp of snapshot)
|
|
5890
7375
|
replay.set(cp.seq, cp);
|
|
7376
|
+
if (!cont && stickyOk) {
|
|
7377
|
+
cont = new StickyContinuation(run.id, fn, run.workflowVersion, compensations, replay, (c) => this.sticky.evict(c.runId));
|
|
7378
|
+
}
|
|
7379
|
+
if (cont)
|
|
7380
|
+
this.turnContinuations.set(run.id, cont);
|
|
7381
|
+
let parkedTurn = false;
|
|
5891
7382
|
let suspendedOnFan;
|
|
5892
7383
|
let settledSuspended = false;
|
|
5893
|
-
const ctx =
|
|
7384
|
+
const ctx = warm
|
|
7385
|
+
? undefined
|
|
7386
|
+
: createWorkflowCtx(this.ctxHostFor(replay, run.namespace, cont), run.id, compensations, run.workflow);
|
|
5894
7387
|
// Every settle below is `return await`ed, never a bare `return this.settleRun(...)`: inside
|
|
5895
7388
|
// `try … finally` a bare return runs the `finally` — which RELEASES THE LEASE — as soon as the
|
|
5896
7389
|
// return expression is evaluated, while the settle's own write is still in flight. A first turn
|
|
@@ -5907,8 +7400,33 @@ export class WorkflowEngine {
|
|
|
5907
7400
|
// Establish the ambient ctx for the duration of this body turn so context-aware statics
|
|
5908
7401
|
// (`BaseWorkflow.start`/`dispatch`) reachable from `fn` route through this run's `ctx.child`/
|
|
5909
7402
|
// `ctx.startChild`. Re-set on every replay turn (each is its own async scope) — correct.
|
|
5910
|
-
|
|
5911
|
-
|
|
7403
|
+
let output;
|
|
7404
|
+
if (cont) {
|
|
7405
|
+
if (parkedFresh) {
|
|
7406
|
+
cont.attached = true;
|
|
7407
|
+
await cont.resumeParked(parkedFresh);
|
|
7408
|
+
}
|
|
7409
|
+
else {
|
|
7410
|
+
const input = await this.turnInput(run, registered);
|
|
7411
|
+
const body = ctx;
|
|
7412
|
+
cont.begin(() => workflowAls.run(body, () => fn(body, input)));
|
|
7413
|
+
}
|
|
7414
|
+
const outcome = await cont.next();
|
|
7415
|
+
if (outcome.kind === 'parked') {
|
|
7416
|
+
// Parked on dispatched steps with nothing else in flight: suspend exactly as a thrown
|
|
7417
|
+
// suspension would, and keep the body for the run's next turn here.
|
|
7418
|
+
parkedTurn = true;
|
|
7419
|
+
throw new WorkflowSuspended(outcome.wakeAt, outcome.waitSeqs);
|
|
7420
|
+
}
|
|
7421
|
+
if (outcome.kind === 'threw')
|
|
7422
|
+
throw outcome.error;
|
|
7423
|
+
output = outcome.value;
|
|
7424
|
+
}
|
|
7425
|
+
else {
|
|
7426
|
+
const input = await this.turnInput(run, registered);
|
|
7427
|
+
const body = ctx;
|
|
7428
|
+
output = await workflowAls.run(body, () => fn(body, input));
|
|
7429
|
+
}
|
|
5912
7430
|
// A compensating cancel requested while this turn ran (or one whose resume replays straight to
|
|
5913
7431
|
// `return`, with no suspension point left) still wins: undo + settle cancelled, never completed.
|
|
5914
7432
|
if (await this.takeCancelIntent(run.id)) {
|
|
@@ -5949,7 +7467,7 @@ export class WorkflowEngine {
|
|
|
5949
7467
|
runId: run.id,
|
|
5950
7468
|
workflow: run.workflow,
|
|
5951
7469
|
namespace: run.namespace,
|
|
5952
|
-
});
|
|
7470
|
+
}, { lifecycle: false });
|
|
5953
7471
|
this.notifyParent(run.id, { ok: true, value: undefined });
|
|
5954
7472
|
// Tracked handoff (see {@link handoffRun}): held in `postSettle` from this settle until the
|
|
5955
7473
|
// continuation has entered `inflight` and settled, so `drain()` can't return before the next
|
|
@@ -5971,6 +7489,11 @@ export class WorkflowEngine {
|
|
|
5971
7489
|
return await this.compensateAndCancel(run, compensations);
|
|
5972
7490
|
}
|
|
5973
7491
|
const settled = await this.settleRun(run, { kind: 'suspended', wakeAt: err.wakeAt });
|
|
7492
|
+
const stamp = this.suspendStamps.get(run.id);
|
|
7493
|
+
this.suspendStamps.delete(run.id);
|
|
7494
|
+
if (parkedTurn && cont && settled.status === 'suspended' && stamp !== undefined) {
|
|
7495
|
+
this.keepContinuation(cont, stamp);
|
|
7496
|
+
}
|
|
5974
7497
|
// Close the settle-vs-delivery race: a signal/result that landed WHILE this turn was still
|
|
5975
7498
|
// executing wrote its checkpoint durably, but its resume no-oped against our held lease —
|
|
5976
7499
|
// and this settle just parked the run on nothing but the reconcile fallback (or nothing at
|
|
@@ -6029,6 +7552,9 @@ export class WorkflowEngine {
|
|
|
6029
7552
|
//
|
|
6030
7553
|
// A suspended turn keeps the lease instead when a wake it refused asked for another turn, and
|
|
6031
7554
|
// runs that turn (see `requestWake`).
|
|
7555
|
+
if (cont && this.turnContinuations.get(run.id) === cont) {
|
|
7556
|
+
this.turnContinuations.delete(run.id);
|
|
7557
|
+
}
|
|
6032
7558
|
let released = true;
|
|
6033
7559
|
try {
|
|
6034
7560
|
if (settledSuspended)
|
|
@@ -6279,10 +7805,11 @@ export class WorkflowEngine {
|
|
|
6279
7805
|
/** The seam handed to {@link createWorkflowCtx}: the authoring API reaches durability + lifecycle
|
|
6280
7806
|
* (checkpointing, dispatch, child start) through this, so the ctx primitives live in their own
|
|
6281
7807
|
* module and the engine stays the orchestrator. */
|
|
6282
|
-
ctxHostFor(replay, runNamespace) {
|
|
7808
|
+
ctxHostFor(replay, runNamespace, sticky) {
|
|
6283
7809
|
return {
|
|
6284
7810
|
store: this.store,
|
|
6285
7811
|
replay,
|
|
7812
|
+
sticky,
|
|
6286
7813
|
clock: this.clock,
|
|
6287
7814
|
stepSignal: this.drainAbort.signal,
|
|
6288
7815
|
eventBufferTtlMs: this.eventBufferTtlMs,
|
|
@@ -6290,7 +7817,18 @@ export class WorkflowEngine {
|
|
|
6290
7817
|
startStep: (s) => this.startStep(s),
|
|
6291
7818
|
completeStep: (s) => this.completeStep(s),
|
|
6292
7819
|
failStep: (s) => this.failStep(s),
|
|
6293
|
-
callRemote: (runId, seq, step, input, queue, transport, admission) =>
|
|
7820
|
+
callRemote: (runId, seq, step, input, queue, transport, admission) => {
|
|
7821
|
+
const call = this.callRemote(runId, seq, step, input, queue, transport, replay, admission, runNamespace);
|
|
7822
|
+
if (!sticky)
|
|
7823
|
+
return call;
|
|
7824
|
+
// In a sticky continuation a call that would suspend parks instead: a later turn
|
|
7825
|
+
// re-evaluates it (the same logic, against its freshly read checkpoint).
|
|
7826
|
+
return call.catch((err) => {
|
|
7827
|
+
if (!(err instanceof WorkflowSuspended) || !sticky.attached)
|
|
7828
|
+
throw err;
|
|
7829
|
+
return sticky.park(seq, err.wakeAt, (fresh) => this.callRemote(runId, seq, step, input, queue, transport, fresh ? new Map([[seq, fresh]]) : undefined, admission, runNamespace, undefined, true));
|
|
7830
|
+
});
|
|
7831
|
+
},
|
|
6294
7832
|
callRemoteAll: (runId, positions, step, inputs, opts) => this.callRemoteAll(runId, positions, step, inputs, opts, runNamespace),
|
|
6295
7833
|
// Defer so a fast child can't reentrantly resume a still-running parent. The child inherits the
|
|
6296
7834
|
// PARENT RUN's namespace, not this engine's: an operator legitimately executes every tenant's
|
|
@@ -6421,12 +7959,12 @@ export class WorkflowEngine {
|
|
|
6421
7959
|
* descriptor keyspace. Empty when no transport can introspect it (in-process / pre-handshake broker)
|
|
6422
7960
|
* OR nobody is publishing — either way the guard is skipped and dispatch proceeds legacy
|
|
6423
7961
|
* assume-compatible (design §7.7). Never throws (the transport degrades to `[]`). */
|
|
6424
|
-
async liveDescriptorsFor(token) {
|
|
7962
|
+
async liveDescriptorsFor(token, fresh = false) {
|
|
6425
7963
|
const read = this.primaryTransport?.listWorkerDescriptors;
|
|
6426
7964
|
if (!read)
|
|
6427
7965
|
return [];
|
|
6428
7966
|
try {
|
|
6429
|
-
return await read.call(this.primaryTransport, token);
|
|
7967
|
+
return await read.call(this.primaryTransport, token, fresh ? { fresh } : undefined);
|
|
6430
7968
|
}
|
|
6431
7969
|
catch {
|
|
6432
7970
|
return [];
|
|
@@ -6439,8 +7977,8 @@ export class WorkflowEngine {
|
|
|
6439
7977
|
* - a `routable` plan — ≥1 live worker is capability-capable AND protocol-compatible (dispatch);
|
|
6440
7978
|
* - a `blocked` plan — descriptors exist but none can run it (park the run, never dispatch into it).
|
|
6441
7979
|
*/
|
|
6442
|
-
async planRoute(token, requires) {
|
|
6443
|
-
const descriptors = await this.liveDescriptorsFor(token);
|
|
7980
|
+
async planRoute(token, requires, fresh = false) {
|
|
7981
|
+
const descriptors = await this.liveDescriptorsFor(token, fresh);
|
|
6444
7982
|
if (descriptors.length === 0)
|
|
6445
7983
|
return undefined; // legacy/no-runtime path — skip the guard
|
|
6446
7984
|
return planDispatch(requires ?? [], descriptors, this.cpDescriptor, token);
|
|
@@ -6450,7 +7988,10 @@ export class WorkflowEngine {
|
|
|
6450
7988
|
* checkpoint + dispatching into a queue nobody consumes (design §7.5). A no-op when the guard does
|
|
6451
7989
|
* not engage (no descriptors) or a capable+compatible worker exists. */
|
|
6452
7990
|
async ensureRoutable(token, requires) {
|
|
6453
|
-
|
|
7991
|
+
let plan = await this.planRoute(token, requires);
|
|
7992
|
+
// Never park a run on reused descriptors: a capable worker may have joined since they were read.
|
|
7993
|
+
if (plan?.status === 'blocked')
|
|
7994
|
+
plan = await this.planRoute(token, requires, true);
|
|
6454
7995
|
if (plan?.status === 'blocked')
|
|
6455
7996
|
throw new WorkflowBlocked(plan);
|
|
6456
7997
|
}
|
|
@@ -6506,7 +8047,21 @@ export class WorkflowEngine {
|
|
|
6506
8047
|
}));
|
|
6507
8048
|
return blocked;
|
|
6508
8049
|
}
|
|
6509
|
-
async callRemote(runId, seq, step, input, queue, transport, replay, admission, runNamespace, parallelGroup
|
|
8050
|
+
async callRemote(runId, seq, step, input, queue, transport, replay, admission, runNamespace, parallelGroup,
|
|
8051
|
+
/** `replay` holds this step's checkpoint as just read from the store: no re-read needed. */
|
|
8052
|
+
fresh) {
|
|
8053
|
+
// A position the turn's history does not hold is normally a first dispatch: write its `pending`
|
|
8054
|
+
// row only if none is there, instead of reading the position first. A row that is there (a turn
|
|
8055
|
+
// this one did not see wrote it) is then read and handled below like any other.
|
|
8056
|
+
if (replay !== undefined &&
|
|
8057
|
+
!replay.has(seq) &&
|
|
8058
|
+
this.store.insertCheckpointIfAbsent &&
|
|
8059
|
+
!step.timeoutMs &&
|
|
8060
|
+
this.pool.size > 0 &&
|
|
8061
|
+
!(queue && this.admission.handles(queue)) &&
|
|
8062
|
+
(await this.dispatchIfAbsent(runId, seq, step, input, transport, admission, runNamespace, parallelGroup))) {
|
|
8063
|
+
throw new WorkflowSuspended(undefined, [seq]);
|
|
8064
|
+
}
|
|
6510
8065
|
// Read the prefix from the per-execution snapshot (avoids the O(N²) replay SELECTs); a seq absent
|
|
6511
8066
|
// from the snapshot — not yet dispatched, or written after the snapshot — falls back to the store.
|
|
6512
8067
|
let existing = replay?.get(seq) ?? (await this.store.getCheckpoint(runId, seq));
|
|
@@ -6516,7 +8071,7 @@ export class WorkflowEngine {
|
|
|
6516
8071
|
// the lost-dispatch lease, or a re-dispatch) — written from a stale row, that overwrites the landed
|
|
6517
8072
|
// `completed` result with `pending` and parks the run on a lease an hour out. So a pending step is
|
|
6518
8073
|
// re-read from the store before anything acts on it (one read, only at a suspension point).
|
|
6519
|
-
if (existing?.status === 'pending' && replay?.has(seq)) {
|
|
8074
|
+
if (existing?.status === 'pending' && replay?.has(seq) && !fresh) {
|
|
6520
8075
|
existing = (await this.store.getCheckpoint(runId, seq)) ?? existing;
|
|
6521
8076
|
}
|
|
6522
8077
|
if (existing && existing.name !== step.name) {
|
|
@@ -6597,6 +8152,8 @@ export class WorkflowEngine {
|
|
|
6597
8152
|
priority: admission?.priority,
|
|
6598
8153
|
attempt: reAttempt,
|
|
6599
8154
|
transport,
|
|
8155
|
+
// A fan's steps are joined by its barrier, not a parked body: their results stay shared.
|
|
8156
|
+
replyHere: parallelGroup === undefined,
|
|
6600
8157
|
});
|
|
6601
8158
|
this.emit({
|
|
6602
8159
|
type: 'step.started',
|
|
@@ -6701,10 +8258,66 @@ export class WorkflowEngine {
|
|
|
6701
8258
|
attempt,
|
|
6702
8259
|
transport,
|
|
6703
8260
|
enqueuedAt,
|
|
8261
|
+
replyHere: parallelGroup === undefined,
|
|
6704
8262
|
});
|
|
6705
8263
|
this.emit({ type: 'step.started', runId, seq, name: step.name, kind: 'remote' });
|
|
6706
8264
|
throw new WorkflowSuspended(undefined, [seq]);
|
|
6707
8265
|
}
|
|
8266
|
+
/**
|
|
8267
|
+
* {@link callRemote}'s first dispatch of a step, written with
|
|
8268
|
+
* {@link StateStore.insertCheckpointIfAbsent}: resolves whether it wrote the `pending` row and
|
|
8269
|
+
* dispatched. `false` when the position already has a row, or when no live worker can take the
|
|
8270
|
+
* step (the read-first path then decides, so a step that is in fact done is never blocked).
|
|
8271
|
+
*/
|
|
8272
|
+
async dispatchIfAbsent(runId, seq, step, input, transport, admission, runNamespace, parallelGroup) {
|
|
8273
|
+
const validInput = step.input
|
|
8274
|
+
? await validateStandard(step.input, input, `step "${step.name}" input`, 'invalid_input')
|
|
8275
|
+
: input;
|
|
8276
|
+
const token = stepGroup(step, runNamespace);
|
|
8277
|
+
try {
|
|
8278
|
+
await this.ensureRoutable(token, step.requires);
|
|
8279
|
+
}
|
|
8280
|
+
catch (err) {
|
|
8281
|
+
if (err instanceof WorkflowBlocked)
|
|
8282
|
+
return false;
|
|
8283
|
+
throw err;
|
|
8284
|
+
}
|
|
8285
|
+
const id = stepId(runId, seq);
|
|
8286
|
+
const enqueuedAt = new Date();
|
|
8287
|
+
const insert = this.store.insertCheckpointIfAbsent;
|
|
8288
|
+
const wrote = await insert.call(this.store, {
|
|
8289
|
+
runId,
|
|
8290
|
+
seq,
|
|
8291
|
+
name: step.name,
|
|
8292
|
+
kind: 'remote',
|
|
8293
|
+
stepId: id,
|
|
8294
|
+
status: 'pending',
|
|
8295
|
+
input: validInput,
|
|
8296
|
+
attempts: 1,
|
|
8297
|
+
workerGroup: token,
|
|
8298
|
+
...(parallelGroup !== undefined ? { parallelGroup } : {}),
|
|
8299
|
+
enqueuedAt,
|
|
8300
|
+
startedAt: enqueuedAt, // placeholders until the worker result lands
|
|
8301
|
+
finishedAt: enqueuedAt,
|
|
8302
|
+
});
|
|
8303
|
+
if (!wrote)
|
|
8304
|
+
return false;
|
|
8305
|
+
await this.dispatchRemoteTask({
|
|
8306
|
+
runId,
|
|
8307
|
+
seq,
|
|
8308
|
+
name: step.name,
|
|
8309
|
+
stepId: id,
|
|
8310
|
+
group: token,
|
|
8311
|
+
input: validInput,
|
|
8312
|
+
priority: admission?.priority,
|
|
8313
|
+
attempt: 1,
|
|
8314
|
+
transport,
|
|
8315
|
+
enqueuedAt,
|
|
8316
|
+
replyHere: parallelGroup === undefined,
|
|
8317
|
+
});
|
|
8318
|
+
this.emit({ type: 'step.started', runId, seq, name: step.name, kind: 'remote' });
|
|
8319
|
+
return true;
|
|
8320
|
+
}
|
|
6708
8321
|
/**
|
|
6709
8322
|
* `ctx.all` over a `@Step`: one durable step per input, dispatched together, joined once. The ctx
|
|
6710
8323
|
* reserves the whole position block up front, so replay re-derives the same seqs.
|
|
@@ -6942,11 +8555,85 @@ export class WorkflowEngine {
|
|
|
6942
8555
|
context: this.context?.(),
|
|
6943
8556
|
priority: task.priority,
|
|
6944
8557
|
attempt: task.attempt,
|
|
8558
|
+
...(task.replyHere && this.repliesRouted && !this.draining
|
|
8559
|
+
? { replyTo: this.instanceId }
|
|
8560
|
+
: {}),
|
|
6945
8561
|
};
|
|
6946
8562
|
if (task.enqueuedAt)
|
|
6947
8563
|
remote[DISPATCH_KEY] = dispatchKey(task.stepId, task.attempt, task.enqueuedAt);
|
|
6948
8564
|
return this.pool.dispatch(remote, task.transport);
|
|
6949
8565
|
}
|
|
8566
|
+
/**
|
|
8567
|
+
* Whether this turn of `run` may run in a sticky continuation (see `sticky-continuation.ts`). Not
|
|
8568
|
+
* while the engine drains, nor with a compensating cancel requested for the run.
|
|
8569
|
+
*/
|
|
8570
|
+
stickyEligible(run) {
|
|
8571
|
+
return this.sticky.enabled && !this.draining && !this.cancelRequested.has(run.id);
|
|
8572
|
+
}
|
|
8573
|
+
/**
|
|
8574
|
+
* The continuation this instance parked for `run`, if it is still the run's latest turn: the run
|
|
8575
|
+
* is `suspended` with the very stamp this instance's suspend wrote, on the same body and version.
|
|
8576
|
+
* Any other settle in between (another instance's turn, a retry, a cancel) changed the stamp, and
|
|
8577
|
+
* the run replays instead. Taking it removes it from the cache.
|
|
8578
|
+
*/
|
|
8579
|
+
takeContinuation(run, fn) {
|
|
8580
|
+
const cont = this.sticky.take(run.id);
|
|
8581
|
+
if (!cont)
|
|
8582
|
+
return undefined;
|
|
8583
|
+
const valid = cont.fn === fn &&
|
|
8584
|
+
cont.workflowVersion === run.workflowVersion &&
|
|
8585
|
+
run.status === 'suspended' &&
|
|
8586
|
+
cont.stamp !== undefined &&
|
|
8587
|
+
run.updatedAt.getTime() === cont.stamp;
|
|
8588
|
+
if (!valid) {
|
|
8589
|
+
this.sticky.misses += 1;
|
|
8590
|
+
return undefined;
|
|
8591
|
+
}
|
|
8592
|
+
this.sticky.hits += 1;
|
|
8593
|
+
return cont;
|
|
8594
|
+
}
|
|
8595
|
+
/** Keep `cont` (parked, detached) for the run's next turn here, stamped with its suspend write. */
|
|
8596
|
+
keepContinuation(cont, stamp) {
|
|
8597
|
+
if (cont.unusable)
|
|
8598
|
+
return;
|
|
8599
|
+
cont.stamp = stamp;
|
|
8600
|
+
// Everything up to the park point is consumed; later positions read the store.
|
|
8601
|
+
cont.replay.clear();
|
|
8602
|
+
this.sticky.put(cont);
|
|
8603
|
+
}
|
|
8604
|
+
/**
|
|
8605
|
+
* The checkpoints of the steps `cont` is parked on, as a replay would see them: in one read when
|
|
8606
|
+
* the store batches, and without the input of a step that completed (what
|
|
8607
|
+
* {@link StateStore.getCheckpointsForSettle} leaves out). A step still open or failed is read
|
|
8608
|
+
* whole, since a re-dispatch or a retry sends its input again.
|
|
8609
|
+
*/
|
|
8610
|
+
async readParked(runId, cont) {
|
|
8611
|
+
const fresh = new Map();
|
|
8612
|
+
for (const seq of cont.parked.keys())
|
|
8613
|
+
fresh.set(seq, cont.hints.get(seq) ?? null);
|
|
8614
|
+
cont.hints.clear();
|
|
8615
|
+
const seqs = [...fresh].flatMap(([seq, cp]) => (cp ? [] : [seq]));
|
|
8616
|
+
if (seqs.length === 0)
|
|
8617
|
+
return fresh;
|
|
8618
|
+
const lean = this.store.getCheckpointsForSettle ?? this.store.getCheckpoints;
|
|
8619
|
+
if (lean) {
|
|
8620
|
+
for (const cp of await lean.call(this.store, runId, seqs))
|
|
8621
|
+
fresh.set(cp.seq, cp);
|
|
8622
|
+
}
|
|
8623
|
+
else {
|
|
8624
|
+
for (const seq of seqs)
|
|
8625
|
+
fresh.set(seq, await this.store.getCheckpoint(runId, seq));
|
|
8626
|
+
}
|
|
8627
|
+
if (lean === this.store.getCheckpointsForSettle) {
|
|
8628
|
+
for (const seq of seqs) {
|
|
8629
|
+
const cp = fresh.get(seq);
|
|
8630
|
+
if (cp && cp.status !== 'completed') {
|
|
8631
|
+
fresh.set(seq, await this.store.getCheckpoint(runId, seq));
|
|
8632
|
+
}
|
|
8633
|
+
}
|
|
8634
|
+
}
|
|
8635
|
+
return fresh;
|
|
8636
|
+
}
|
|
6950
8637
|
/** A turn's read of its run's checkpoints: {@link StateStore.listCheckpointsForReplay}, or every
|
|
6951
8638
|
* column through {@link StateStore.listCheckpoints} on a store without it. */
|
|
6952
8639
|
replayCheckpoints(runId) {
|
|
@@ -7217,6 +8904,9 @@ export class WorkflowEngine {
|
|
|
7217
8904
|
* replays every write of the batch, so a second would only replay it again.
|
|
7218
8905
|
*/
|
|
7219
8906
|
async completeRemoteResults(runId, results, queuedIn = () => false) {
|
|
8907
|
+
if (this.store.settleResultsIfPending) {
|
|
8908
|
+
return this.settleResultsReturning(runId, results, queuedIn);
|
|
8909
|
+
}
|
|
7220
8910
|
const outcomes = results.map(() => true);
|
|
7221
8911
|
const settleOneByOne = async (indexes) => {
|
|
7222
8912
|
for (const i of indexes) {
|
|
@@ -7348,6 +9038,144 @@ export class WorkflowEngine {
|
|
|
7348
9038
|
}
|
|
7349
9039
|
return outcomes;
|
|
7350
9040
|
}
|
|
9041
|
+
/**
|
|
9042
|
+
* {@link completeRemoteResults} on a store with {@link StateStore.settleResultsIfPending}: the
|
|
9043
|
+
* results settle their `pending` rows and read them back, with their run's status, in one call,
|
|
9044
|
+
* instead of reading the rows and the run and then writing. An entry that settled nothing (a
|
|
9045
|
+
* redelivery, a step gone, a late result for a finished run) takes the single-result path, which
|
|
9046
|
+
* reads first and re-drives, drops or salvages. A settled `completed` row is also handed to the
|
|
9047
|
+
* run's parked body, if this instance holds one, so its next turn does not read it again.
|
|
9048
|
+
*/
|
|
9049
|
+
async settleResultsReturning(runId, results, queuedIn) {
|
|
9050
|
+
const outcomes = results.map(() => true);
|
|
9051
|
+
const settle = this.store.settleResultsIfPending.bind(this.store);
|
|
9052
|
+
// A second copy of a result in the same batch is a duplicate delivery: the first copy settles
|
|
9053
|
+
// the step (or re-drives the resume), so the copy is dropped like a late one.
|
|
9054
|
+
const seen = new Set();
|
|
9055
|
+
const live = results.flatMap((r, i) => {
|
|
9056
|
+
if (seen.has(r.seq))
|
|
9057
|
+
return [];
|
|
9058
|
+
seen.add(r.seq);
|
|
9059
|
+
return [i];
|
|
9060
|
+
});
|
|
9061
|
+
const finishedAt = new Date();
|
|
9062
|
+
const entries = live.map((i) => {
|
|
9063
|
+
const result = results[i];
|
|
9064
|
+
return {
|
|
9065
|
+
runId,
|
|
9066
|
+
seq: result.seq,
|
|
9067
|
+
status: result.status,
|
|
9068
|
+
output: result.status === 'completed' ? result.output : undefined,
|
|
9069
|
+
error: result.error,
|
|
9070
|
+
events: result.events,
|
|
9071
|
+
startedAt: result.startedAt ? new Date(result.startedAt) : undefined,
|
|
9072
|
+
finishedAt,
|
|
9073
|
+
};
|
|
9074
|
+
});
|
|
9075
|
+
// Never rejects as a whole: the batch call rejecting wrote nothing, so the entries are then
|
|
9076
|
+
// settled one by one and a failing one fails only its own result.
|
|
9077
|
+
let rows;
|
|
9078
|
+
try {
|
|
9079
|
+
rows = await settle(entries);
|
|
9080
|
+
}
|
|
9081
|
+
catch (err) {
|
|
9082
|
+
if (entries.length === 1) {
|
|
9083
|
+
rows = [{ error: err }];
|
|
9084
|
+
}
|
|
9085
|
+
else {
|
|
9086
|
+
const one = await Promise.allSettled(entries.map((e) => settle([e])));
|
|
9087
|
+
rows = one.map((o) => o.status === 'fulfilled' ? (o.value[0] ?? null) : { error: o.reason });
|
|
9088
|
+
}
|
|
9089
|
+
}
|
|
9090
|
+
const resumable = [];
|
|
9091
|
+
const unsettled = [];
|
|
9092
|
+
let run;
|
|
9093
|
+
// The run's parked body: in the cache, or in a turn that is parking it right now.
|
|
9094
|
+
const parked = this.sticky.peek(runId) ?? this.turnContinuations.get(runId);
|
|
9095
|
+
for (const [k, row] of rows.entries()) {
|
|
9096
|
+
const index = live[k];
|
|
9097
|
+
const result = results[index];
|
|
9098
|
+
if (row === null) {
|
|
9099
|
+
unsettled.push(index);
|
|
9100
|
+
continue;
|
|
9101
|
+
}
|
|
9102
|
+
if ('error' in row) {
|
|
9103
|
+
outcomes[index] = { error: row.error };
|
|
9104
|
+
continue;
|
|
9105
|
+
}
|
|
9106
|
+
const cp = row.checkpoint;
|
|
9107
|
+
// The run as it read when this result settled: what the resume owes it (see `resumeOwed`).
|
|
9108
|
+
run ??= { id: runId, status: row.run.status, wakeAt: row.run.wakeAt };
|
|
9109
|
+
try {
|
|
9110
|
+
await this.releaseQueueSlot(cp);
|
|
9111
|
+
}
|
|
9112
|
+
catch (error) {
|
|
9113
|
+
// Written but not acknowledged: the redelivery finds the step settled and re-drives.
|
|
9114
|
+
outcomes[index] = { error };
|
|
9115
|
+
}
|
|
9116
|
+
if (cp.status === 'completed')
|
|
9117
|
+
parked?.hint(cp);
|
|
9118
|
+
this.emit({
|
|
9119
|
+
type: result.status === 'completed' ? 'step.completed' : 'step.failed',
|
|
9120
|
+
runId,
|
|
9121
|
+
seq: result.seq,
|
|
9122
|
+
name: cp.name,
|
|
9123
|
+
kind: cp.kind,
|
|
9124
|
+
output: result.output,
|
|
9125
|
+
error: result.error,
|
|
9126
|
+
queueMs: cp.startedAt.getTime() - cp.enqueuedAt.getTime(),
|
|
9127
|
+
durationMs: finishedAt.getTime() - cp.startedAt.getTime(),
|
|
9128
|
+
});
|
|
9129
|
+
resumable.push({ cp, result });
|
|
9130
|
+
}
|
|
9131
|
+
// Nothing written for these: read first, then re-drive, drop or salvage, one at a time.
|
|
9132
|
+
for (const index of unsettled) {
|
|
9133
|
+
try {
|
|
9134
|
+
await this.completeOneRemoteResult(results[index], queuedIn);
|
|
9135
|
+
}
|
|
9136
|
+
catch (error) {
|
|
9137
|
+
outcomes[index] = { error };
|
|
9138
|
+
}
|
|
9139
|
+
}
|
|
9140
|
+
if (resumable.length === 0)
|
|
9141
|
+
return outcomes;
|
|
9142
|
+
// The barrier logic of the read-first batch path (see completeRemoteResults): one resume.
|
|
9143
|
+
const barriers = new Map();
|
|
9144
|
+
const highest = new Map();
|
|
9145
|
+
for (const { cp } of resumable) {
|
|
9146
|
+
const fan = parseStepFanGroup(cp.parallelGroup);
|
|
9147
|
+
if (fan)
|
|
9148
|
+
highest.set(fan.firstSeq, Math.max(highest.get(fan.firstSeq) ?? cp.seq, cp.seq));
|
|
9149
|
+
}
|
|
9150
|
+
const singles = resumable
|
|
9151
|
+
.filter(({ cp }) => !parseStepFanGroup(cp.parallelGroup))
|
|
9152
|
+
.map(({ cp }) => cp.seq);
|
|
9153
|
+
for (const { cp, result } of resumable) {
|
|
9154
|
+
const fan = parseStepFanGroup(cp.parallelGroup);
|
|
9155
|
+
if (!fan || result.status === 'failed') {
|
|
9156
|
+
this.noteSettledHere(runId, singles);
|
|
9157
|
+
await this.resumeOwed(run, runId);
|
|
9158
|
+
return outcomes;
|
|
9159
|
+
}
|
|
9160
|
+
let closed = barriers.get(fan.firstSeq);
|
|
9161
|
+
if (!closed) {
|
|
9162
|
+
const lastSeq = fan.firstSeq + fan.size - 1;
|
|
9163
|
+
closed = queuedIn(fan.firstSeq, lastSeq)
|
|
9164
|
+
? Promise.resolve(true)
|
|
9165
|
+
: this.stepFanHasPending(runId, fan.firstSeq, lastSeq).catch(() => false);
|
|
9166
|
+
barriers.set(fan.firstSeq, closed);
|
|
9167
|
+
}
|
|
9168
|
+
if (!(await closed)) {
|
|
9169
|
+
this.noteSettledHere(runId, singles);
|
|
9170
|
+
await this.resumeOwed(run, runId, {
|
|
9171
|
+
first: fan.firstSeq,
|
|
9172
|
+
seq: highest.get(fan.firstSeq) ?? cp.seq,
|
|
9173
|
+
});
|
|
9174
|
+
return outcomes;
|
|
9175
|
+
}
|
|
9176
|
+
}
|
|
9177
|
+
return outcomes;
|
|
9178
|
+
}
|
|
7351
9179
|
/**
|
|
7352
9180
|
* Settle pending steps, each only while its row is still `pending`; per entry, whether it wrote
|
|
7353
9181
|
* or the error that kept it from writing. Through {@link StateStore.settleCheckpointsIfPending}
|
|
@@ -7798,8 +9626,30 @@ export class WorkflowEngine {
|
|
|
7798
9626
|
});
|
|
7799
9627
|
}
|
|
7800
9628
|
}
|
|
7801
|
-
/**
|
|
7802
|
-
*
|
|
9629
|
+
/**
|
|
9630
|
+
* The namespace an announcement serves: the descriptor's own, else the `@<namespace>` suffix of its
|
|
9631
|
+
* routing group, else `default`.
|
|
9632
|
+
*/
|
|
9633
|
+
function servedNamespace(namespace, group) {
|
|
9634
|
+
if (namespace)
|
|
9635
|
+
return namespace;
|
|
9636
|
+
const at = group?.lastIndexOf('@') ?? -1;
|
|
9637
|
+
return group && at >= 0 && at < group.length - 1 ? group.slice(at + 1) : 'default';
|
|
9638
|
+
}
|
|
9639
|
+
function withoutBookkeepingRuns(query) {
|
|
9640
|
+
if (query.workflow === LIFECYCLE_REGISTRY_WORKFLOW ||
|
|
9641
|
+
query.workflows?.includes(LIFECYCLE_REGISTRY_WORKFLOW)) {
|
|
9642
|
+
return query;
|
|
9643
|
+
}
|
|
9644
|
+
const excluded = query.excludeWorkflows ?? [];
|
|
9645
|
+
if (excluded.includes(LIFECYCLE_REGISTRY_WORKFLOW))
|
|
9646
|
+
return query;
|
|
9647
|
+
return { ...query, excludeWorkflows: [...excluded, LIFECYCLE_REGISTRY_WORKFLOW] };
|
|
9648
|
+
}
|
|
9649
|
+
/** `value` when it has at least one own key, else undefined. */
|
|
9650
|
+
function nonEmptyObject(value) {
|
|
9651
|
+
return Object.keys(value).length > 0 ? value : undefined;
|
|
9652
|
+
}
|
|
7803
9653
|
function conventionGroup(name, namespace) {
|
|
7804
9654
|
return tenantGroup(sanitizeQueueToken(name), namespace);
|
|
7805
9655
|
}
|