@adonis-agora/durable 0.46.1 → 0.48.0

package/dist/src/engine.js

@@ -1,7 +1,7 @@
 import { InMemoryAdmissionBackend } from './admission.js';
 import { backoffDelay, MAX_BACKOFF_MS } from './backoff.js';
 import { instantCheckpoint, lostStepEvent, stepCheckpoint } from './checkpoints.js';
-import { isFailedCompletion } from './completion.js';
+import { completionError, failedCompletion, isFailedCompletion, } from './completion.js';
 import { assertConcurrency, concurrencyTag, countRuns, resolveConcurrency, } from './concurrency.js';
 import { controlPlaneDescriptor, planDispatch, WorkflowBlocked, } from './dispatch-routing.js';
 import { parseDuration } from './duration.js';
@@ -10,8 +10,10 @@ import { ContinueAsNew, NonDeterminismError, RemoteStepTimeout, WorkflowSuspende
 import { EventAccumulators } from './event-accumulators.js';
 import { eventMatches, eventMatchOf, eventPrefix } from './events.js';
 import { FOLDED_PREFIX, fanOfWaiter, foldedChildIds } from './fan-join.js';
+import { aggregateAnnouncements } from './handshake/announced.js';
 import { isNonEmptyRunScope, TERMINAL_RUN_STATUSES } from './interfaces.js';
 import { asHeartbeat } from './observability-scope.js';
+import { assertRoutes, codeAndDetails, HANDOFF_SEQ_BASE, handoffHandlerId, handoffName, liveHandoff, onErrorRunId, retiredHandoffName, routeFor, } from './on-error.js';
 import { breakpointToken, stepId } from './protocol.js';
 import { RemoteWorkflowExecutor } from './remote-workflow-executor.js';
 import { scanRunValueFacets } from './run-value-facets.js';
@@ -58,6 +60,8 @@ const SETTLE_ELIGIBLE_STATUSES = [
     'blocked',
     'failed',
 ];
+/** How long a read of the onError routes a fleet announces is reused. */
+const ANNOUNCED_ROUTES_TTL_MS = 5_000;
 const versionKey = (name, version) => `${name}@${version}`;
 /**
  * First checkpoint seq reserved for saga compensations: compensation `idx` (registration order)
@@ -198,6 +202,8 @@ export class WorkflowEngine {
     updateValidators = new Map();
     /** Runs being cancelled WITH saga compensation — see `cancel({ compensate: true })`. */
     cancelRequested = new Set();
+    /** {@link announcedRoutes} reads, by routing token + workflow + version. */
+    announcedRoutesCache = new Map();
     /** The merged schedule set the worker loop ticks with (see {@link adoptSchedules}). */
     #allSchedules = [];
     /** Runtime pause overrides by schedule key (console-issued; win over the config's `paused`). */
@@ -545,6 +551,7 @@ export class WorkflowEngine {
             name,
             version,
             fn,
+            onError: this.checkedRoutes(name, opts?.onError),
             tags: opts?.tags,
             origin: opts?.origin,
             singleton: opts?.singleton,
@@ -575,6 +582,7 @@ export class WorkflowEngine {
         const registered = {
             name,
             version,
+            onError: this.checkedRoutes(name, opts.onError),
             // A remote workflow has no in-process body; execute() branches on `remote` before this is read.
             fn: () => {
                 throw new Error(`workflow ${name} is remote — it has no in-process body`);
@@ -613,6 +621,9 @@ export class WorkflowEngine {
             group: name,
             executor: new RemoteWorkflowExecutor(this.primaryTransport, name),
         });
+        const registered = this.workflows.get(versionKey(name, '1'));
+        if (registered)
+            registered.convention = true;
         return true;
     }
     /**
@@ -1689,6 +1700,24 @@ export class WorkflowEngine {
         if (run.status === 'failed' || run.status === 'dead') {
             const isFailureCompletion = (v) => typeof v === 'object' && v !== null && v.ok === false;
             for (const cp of await this.store.listCheckpoints(runId)) {
+                // A retry runs the body again, so a failure it handed to an onError handler is retired: kept in
+                // the timeline under another name, never followed again, and the handler is not retried. Its
+                // `onError:<code>` tag goes too: the tag says how the run ended, and that is being redone.
+                const handlerId = handoffHandlerId(runId, cp);
+                if (handlerId !== undefined) {
+                    await this.store.takeSignalWaiter(`child:${handlerId}`);
+                    await this.store.saveCheckpoint({
+                        ...cp,
+                        name: retiredHandoffName(handlerId),
+                        status: 'completed',
+                    });
+                    const tags = run.tags?.filter((t) => !t.startsWith('onError:'));
+                    if (tags && tags.length !== run.tags?.length) {
+                        await this.store.updateRun(runId, { tags, updatedAt: new Date() });
+                        run.tags = tags;
+                    }
+                    continue;
+                }
                 if (cp.status === 'failed') {
                     await this.store.saveCheckpoint({
                         ...cp,
@@ -2208,7 +2237,7 @@ export class WorkflowEngine {
             ? buffered.payload
             : settled.status === 'completed'
                 ? { ok: true, value: settled.output }
-                : { ok: false, error: settled.error?.message ?? settled.status };
+                : failedCompletion(settled.error ?? { message: settled.status });
         await this.signal(token, completion);
     }
     /**
@@ -2597,6 +2626,169 @@ export class WorkflowEngine {
      * {@link runExecution}'s settle/suspend; the lease is held by {@link execute}. The result that lands
      * us back here (a remote step finished, a timer fired) goes through `resume` like any TS workflow.
      */
+    /**
+     * The registered onError routes whose handler workflow nothing can run: not registered on this
+     * engine, and no live worker group serves it. Meant for a boot-time warning: a typo in a handler's
+     * name would otherwise surface only when the first run fails.
+     */
+    async unservedOnErrorRoutes() {
+        const unserved = [];
+        let groups;
+        for (const registered of this.workflows.values()) {
+            for (const route of registered.onError ?? []) {
+                if (this.latest.has(route.workflow))
+                    continue;
+                groups ??= await this.pool.listWorkerGroups().catch(() => []);
+                if (groups.some((g) => g === route.workflow || g.startsWith(`${route.workflow}@`)))
+                    continue;
+                unserved.push({ workflow: registered.name, route });
+            }
+        }
+        return unserved;
+    }
+    /**
+     * The onError routes the live workers on `token` announce for a workflow the engine serves by
+     * convention: the one set they agree on, or none when they state none or cannot be read. Workers
+     * that announce different sets (a rollout in progress) route nothing, and the run's failure is
+     * reported as a `run.unrouted` event. Announcers of another version of the workflow are ignored. The
+     * read is cached for {@link ANNOUNCED_ROUTES_TTL_MS}, so a burst of failures costs one read.
+     */
+    async announcedRoutes(run, token, error) {
+        const key = `${token}\n${run.workflow}\n${run.workflowVersion}`;
+        const now = this.clock();
+        let cached = this.announcedRoutesCache.get(key);
+        if (!cached || now - cached.at > ANNOUNCED_ROUTES_TTL_MS) {
+            const sets = new Map();
+            for (const announced of aggregateAnnouncements(await this.liveDescriptorsFor(token))) {
+                if (announced.name !== run.workflow)
+                    continue;
+                if (announced.version !== undefined && announced.version !== run.workflowVersion)
+                    continue;
+                for (const routes of announced.onError)
+                    sets.set(JSON.stringify(routes), routes);
+            }
+            cached = { at: now, sets: [...sets.values()] };
+            this.announcedRoutesCache.set(key, cached);
+        }
+        if (cached.sets.length > 1) {
+            this.emit({
+                type: 'run.unrouted',
+                runId: run.id,
+                workflow: run.workflow,
+                namespace: run.namespace,
+                error,
+                reason: `live workers announce ${cached.sets.length} different onError route sets`,
+            });
+            return undefined;
+        }
+        return cached.sets[0];
+    }
+    /** Validate a workflow's {@link OnErrorRoute}s at registration. */
+    checkedRoutes(name, routes) {
+        if (!routes?.length)
+            return undefined;
+        assertRoutes(name, routes);
+        return routes;
+    }
+    /**
+     * Route a failed run to its {@link OnErrorRoute} handler. Starts the handler run and waits for it the
+     * way a parent waits for a `ctx.child`: a `child:<handlerId>` waiter plus a `running` placeholder
+     * checkpoint carrying the handler's name and input, so the handler's terminal state wakes the run
+     * (buffered and reconciled like any child). From then on every execution of the run only follows the
+     * handler ({@link followHandoff}); the body never runs again. Undefined when no route matches: the
+     * run fails as usual.
+     *
+     * The placeholder sits past every seq the body used and is left out of {@link remoteHistory}.
+     */
+    async routeFailure(run, error) {
+        const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion));
+        const routes = registered?.convention
+            ? await this.announcedRoutes(run, registered.remote?.group ?? run.workflow, error)
+            : registered?.onError;
+        const route = routeFor(routes, error);
+        if (!route)
+            return undefined;
+        const checkpoints = await this.handoffCheckpoints(run.id);
+        const live = liveHandoff(run.id, checkpoints);
+        if (live)
+            return this.followHandoff(run, live);
+        // One handoff per attempt: a retry retires the previous one, so the next sits past it.
+        const attempt = checkpoints.length;
+        const seq = HANDOFF_SEQ_BASE + attempt;
+        const handlerId = onErrorRunId(run.id, attempt);
+        const handler = {
+            workflow: route.workflow,
+            input: { runId: run.id, workflow: run.workflow, input: run.input, error },
+        };
+        await this.store.putSignalWaiter({ token: `child:${handlerId}`, runId: run.id, seq });
+        await this.store.saveCheckpoint({
+            ...instantCheckpoint({
+                runId: run.id,
+                seq,
+                name: handoffName(handlerId),
+                kind: 'signal',
+                status: 'running',
+            }),
+            input: handler,
+        });
+        // Tagged, so the runs whose failures were handed off are a filter away in the console.
+        const tag = `onError:${route.code ?? '*'}`;
+        if (!run.tags?.includes(tag)) {
+            const tags = [...(run.tags ?? []), tag];
+            await this.store.updateRun(run.id, { tags, updatedAt: new Date() });
+            run.tags = tags;
+        }
+        this.emit({
+            type: 'run.routed',
+            runId: run.id,
+            workflow: run.workflow,
+            namespace: run.namespace,
+            name: route.workflow,
+            error,
+        });
+        this.startChildDeferred(handler.workflow, handler.input, handlerId, {
+            namespace: run.namespace,
+        });
+        return { kind: 'suspended' };
+    }
+    /** A run's handoff checkpoints, live and retired: the only ones at {@link HANDOFF_SEQ_BASE} and up. */
+    async handoffCheckpoints(runId) {
+        const handoffs = await this.store.listCheckpointsByNamePrefix?.(runId, [
+            `signal:child:${runId}.onError.`,
+            `onError:retried:${runId}.onError.`,
+        ]);
+        return (handoffs ?? (await this.store.listCheckpoints(runId))).filter((cp) => cp.seq >= HANDOFF_SEQ_BASE);
+    }
+    /**
+     * Where a run that handed its failure off stands: the handler's outcome once it settled (from the
+     * wake's checkpoint, or off the handler run when the wake never arrived), else still waiting. Starts
+     * the handler if the turn that handed off crashed before its deferred start ran.
+     */
+    async followHandoff(run, handoff) {
+        if (handoff.status === 'completed') {
+            const outcome = handoff.output;
+            return outcome.ok
+                ? { kind: 'completed', output: outcome.value }
+                : { kind: 'failed', error: completionError(outcome) };
+        }
+        const handlerId = handoffHandlerId(run.id, handoff);
+        const handlerRun = await this.store.getRun(handlerId);
+        if (handlerRun?.status === 'completed')
+            return { kind: 'completed', output: handlerRun.output };
+        if (handlerRun && TERMINAL_RUN_STATUSES.includes(handlerRun.status)) {
+            return {
+                kind: 'failed',
+                error: handlerRun.error ?? { message: `onError handler ${handlerId} ${handlerRun.status}` },
+            };
+        }
+        if (!handlerRun) {
+            const handler = handoff.input;
+            this.startChildDeferred(handler.workflow, handler.input, handlerId, {
+                namespace: run.namespace,
+            });
+        }
+        return { kind: 'suspended' };
+    }
     /**
      * Apply a terminal/suspended transition once: persist the new run status, emit the matching
      * lifecycle event, and (on completion/failure) wake a waiting parent. Both the TS and remote
@@ -2639,6 +2831,10 @@ export class WorkflowEngine {
             return { runId: run.id, status: 'completed', output: outcome.output };
         }
         if (outcome.kind === 'failed') {
+            const handed = await this.routeFailure(run, outcome.error);
+            if (handed && handed.kind !== 'failed')
+                return this.settleRun(run, handed);
+            const error = handed?.error ?? outcome.error;
             // Same race as the `completed` branch above — guard identically. ADDITIONALLY exclude
             // `failed`: a stray resume of an already-failed run (a redelivered signal, the settle-recheck
             // re-drive) replays and deterministically re-fails — re-applying the settle would re-emit
@@ -2649,7 +2845,7 @@ export class WorkflowEngine {
             // status), so it is unaffected.
             const applied = await this.store.updateRunIf(run.id, SETTLE_ELIGIBLE_STATUSES.filter((s) => s !== 'failed'), {
                 status: 'failed',
-                error: outcome.error,
+                error,
                 updatedAt,
             });
             if (!applied)
@@ -2659,10 +2855,10 @@ export class WorkflowEngine {
                 runId: run.id,
                 workflow: run.workflow,
                 namespace: run.namespace,
-                error: outcome.error,
+                error,
             });
-            this.notifyParent(run.id, { ok: false, error: outcome.error.message });
-            return { runId: run.id, status: 'failed', error: outcome.error };
+            this.notifyParent(run.id, failedCompletion(error));
+            return { runId: run.id, status: 'failed', error };
         }
         // This outcome was computed by a turn that started from a possibly-stale run snapshot. If the run
         // was cancelled WHILE that turn was still executing (e.g. `ctx.all`'s failFast cancelling a
@@ -2746,7 +2942,13 @@ export class WorkflowEngine {
         const routePlan = await this.planRoute(remote.group, registered.requires);
         if (routePlan?.status === 'blocked')
             return this.parkBlocked(run, routePlan);
-        const history = await this.remoteHistory(run.id);
+        const checkpoints = await this.store.listCheckpoints(run.id);
+        const handoff = liveHandoff(run.id, checkpoints);
+        if (handoff) {
+            // A run that handed its failure to an onError handler only follows the handler from here on.
+            return this.settleRun(run, await this.followHandoff(run, handoff));
+        }
+        const history = await this.remoteHistory(run.id, checkpoints);
         let decision;
         try {
             decision = await remote.executor.advance(run, history);
@@ -3007,8 +3209,8 @@ export class WorkflowEngine {
         }
     }
     /** The run's resolved durable ops as replay inputs: completed/failed steps + elapsed timers. */
-    async remoteHistory(runId) {
-        const checkpoints = await this.store.listCheckpoints(runId);
+    async remoteHistory(runId, listed) {
+        const checkpoints = listed ?? (await this.store.listCheckpoints(runId));
         const kindOf = {
             remote: 'call',
             local: 'step',
@@ -3021,13 +3223,16 @@ export class WorkflowEngine {
                 // A child run resolves THROUGH the signal machinery (a `child:<id>` waiter notified on the
                 // child's terminal state), so its checkpoint is kind `signal` with a `signal:child:` name and
                 // a Completion payload. Surface it as a `child` event with the value/error unwrapped.
+                // onError handoffs sit past every op a body can reach; no replay is told about them.
+                if (cp.seq >= HANDOFF_SEQ_BASE)
+                    continue;
                 if (cp.kind === 'signal' && cp.name.startsWith('signal:child:')) {
                     const completion = cp.output;
                     events.push({
                         seq: cp.seq,
                         kind: 'child',
                         output: completion?.ok ? completion.value : undefined,
-                        error: completion && completion.ok === false ? { message: completion.error } : undefined,
+                        error: completion && completion.ok === false ? completionError(completion) : undefined,
                     });
                     continue;
                 }
@@ -3270,7 +3475,7 @@ export class WorkflowEngine {
                         child.status === 'dead') {
                         await deliverChild(child.status === 'completed'
                             ? { ok: true, value: child.output }
-                            : { ok: false, error: child.error?.message ?? child.status });
+                            : failedCompletion(child.error ?? { message: child.status }));
                     }
                 }
             }
@@ -3330,6 +3535,11 @@ export class WorkflowEngine {
         // counted an attempt (or dead-lettered it), flipped it to `pending` and re-enqueued a second
         // execution. The lease must outlive the state it protects.
         try {
+            // A run that handed its failure to an onError handler only follows the handler from here on.
+            const handoff = liveHandoff(run.id, snapshot);
+            if (handoff && !this.cancelRequested.has(run.id)) {
+                return await this.settleRun(run, await this.followHandoff(run, handoff));
+            }
             // Establish the ambient ctx for the duration of this body turn so context-aware statics
             // (`BaseWorkflow.start`/`dispatch`) reachable from `fn` route through this run's `ctx.child`/
             // `ctx.startChild`. Re-set on every replay turn (each is its own async scope) — correct.
@@ -3439,6 +3649,7 @@ export class WorkflowEngine {
             const error = {
                 message: err instanceof Error ? err.message : String(err),
                 stack: err instanceof Error ? err.stack : undefined,
+                ...codeAndDetails(err),
             };
             // Saga: undo completed steps in reverse, each retried up to `compensationRetries`. Outcomes
             // are emitted as `compensate:<step>` step events so a stranded undo is VISIBLE (not silently
@@ -3950,7 +4161,11 @@ export class WorkflowEngine {
             // becomes this deadline, so a reconcile re-drive lands exactly when it's due to re-dispatch.
             if (existing.wakeAt == null) {
                 const wakeAt = this.clock() + this.remoteRedispatchMs;
-                await this.store.saveCheckpoint({ ...existing, wakeAt });
+                // The result may have settled the row since it was read: then nothing was written, and the
+                // step is re-read from the store.
+                if (!(await this.savePendingCheckpoint({ ...existing, wakeAt }))) {
+                    return this.callRemote(runId, seq, step, input, queue, transport, undefined, admission, runNamespace);
+                }
                 throw new WorkflowSuspended(wakeAt, [seq]);
             }
             if (this.clock() < existing.wakeAt)
@@ -3966,7 +4181,7 @@ export class WorkflowEngine {
             const reAttempt = existing.attempts + 1;
             const nextDeadline = this.clock() + this.remoteRedispatchMs;
             const reEnqueuedAt = new Date();
-            await this.store.saveCheckpoint({
+            const claimed = await this.savePendingCheckpoint({
                 ...existing,
                 attempts: reAttempt,
                 wakeAt: nextDeadline,
@@ -3980,6 +4195,9 @@ export class WorkflowEngine {
                     lostStepEvent(this.clock(), reAttempt, 'redispatched'),
                 ],
             });
+            if (!claimed) {
+                return this.callRemote(runId, seq, step, input, queue, transport, undefined, admission, runNamespace);
+            }
             await this.dispatchRemoteTask({
                 runId,
                 seq,
@@ -4132,6 +4350,13 @@ export class WorkflowEngine {
      * (never an unhandled rejection) — a failure warns and surfaces as a still-`suspended` run that
      * recovery re-drives, never as a silent strand.
      */
+    /** {@link StateStore.saveCheckpointIfPending}, or a plain save on a store without it. */
+    async savePendingCheckpoint(checkpoint) {
+        if (this.store.saveCheckpointIfPending)
+            return this.store.saveCheckpointIfPending(checkpoint);
+        await this.store.saveCheckpoint(checkpoint);
+        return true;
+    }
     async completeRemoteResult(result) {
         const runId = await this.settleRemoteCheckpoint(result);
         if (runId === undefined)
@@ -4175,17 +4400,20 @@ export class WorkflowEngine {
                 return result.runId;
             return undefined;
         }
-        // A result settling this step frees its flow-control slot (no-op if it wasn't queued). Done
-        // before the terminal-run early-returns below, so a cancellation can't leak the slot.
-        await this.releaseQueueSlot(cp);
+        // A result settling this step frees its flow-control slot (no-op if it wasn't queued). Released
+        // on every early return below too, so a cancellation can't leak the slot; on the settling path
+        // only by the delivery that claims the row, so a duplicated result can't free it twice.
         // Drop a late result for a run that was cancelled/finished meanwhile — don't complete the step
         // or resume (the run is already terminal). This is the engine side of cooperative cancellation.
-        if (run && (run.status === 'cancelled' || run.status === 'completed'))
+        if (run && (run.status === 'cancelled' || run.status === 'completed')) {
+            await this.releaseQueueSlot(cp);
             return undefined;
+        }
         // A late SUCCESS for a failed/dead run: salvage the finished work onto the checkpoint — an
         // explicit retry's replay then short-circuits this step instead of re-running it (the work may
         // be minutes of real compute) — but never resume. A late FAILURE has nothing to salvage: drop.
         if (run && (run.status === 'failed' || run.status === 'dead')) {
+            await this.releaseQueueSlot(cp);
             if (result.status === 'completed') {
                 await this.store.saveCheckpoint({
                     ...cp,
@@ -4200,7 +4428,9 @@ export class WorkflowEngine {
         }
         const finishedAt = new Date();
         const startedAt = result.startedAt ? new Date(result.startedAt) : cp.startedAt;
-        await this.store.saveCheckpoint({
+        // Claimed, not blindly written: another delivery settled it first, which is the redelivery case
+        // above — re-drive the resume rather than settle twice.
+        const claimed = await this.savePendingCheckpoint({
             ...cp,
             status: result.status,
             output: result.status === 'completed' ? result.output : cp.output,
@@ -4209,6 +4439,9 @@ export class WorkflowEngine {
             startedAt,
             finishedAt,
         });
+        if (!claimed)
+            return terminal ? undefined : result.runId;
+        await this.releaseQueueSlot(cp);
         this.emit({
             type: result.status === 'completed' ? 'step.completed' : 'step.failed',
             runId: result.runId,