@adonis-agora/durable 0.56.7 → 0.57.0

package/dist/src/engine.js

@@ -230,6 +230,8 @@ function isNewerVersion(a, b) {
         return na > nb;
     return a.localeCompare(b, undefined, { numeric: true }) > 0;
 }
+/** The statuses a run can be skipped in for an unregistered workflow — every non-terminal one. */
+const UNREGISTERED_DEAD_LETTER_FROM = ['pending', 'running', 'suspended', 'blocked'];
 /** Thrown by {@link WorkflowEngine.resume} when the run belongs to a different namespace. */
 export class NamespaceMismatch extends Error {
     constructor() {
@@ -237,6 +239,16 @@ export class NamespaceMismatch extends Error {
         this.name = 'NamespaceMismatch';
     }
 }
+/**
+ * Thrown by {@link WorkflowEngine.resume} when no workflow registered here (nor a live worker group)
+ * can run the run's pinned `workflow@version`.
+ */
+export class WorkflowNotRegistered extends Error {
+    constructor(workflow, version) {
+        super(`workflow ${workflow}@${version} is not registered — keep the prior version deployed so in-flight runs can drain (skew protection)`);
+        this.name = 'WorkflowNotRegistered';
+    }
+}
 /**
  * The orchestrator. Owns workflow state and replays runs deterministically: each step's
  * result is checkpointed, so on resume a completed step returns its saved output instead of
@@ -273,6 +285,7 @@ export class WorkflowEngine {
      */
     bootedAt;
     maxRecoveryAttempts;
+    unregisteredWorkflowGraceMs;
     reconcileMs;
     heartbeatPersistThrottleMs;
     timerBatchSize;
@@ -514,6 +527,9 @@ export class WorkflowEngine {
         this.leaseMs = deps.leaseMs ?? 30_000;
         this.bootedAt = this.clock();
         this.maxRecoveryAttempts = deps.maxRecoveryAttempts;
+        const grace = deps.unregisteredWorkflowGraceMs;
+        this.unregisteredWorkflowGraceMs =
+            grace !== undefined && Number.isFinite(grace) && grace >= 0 ? grace : 24 * 60 * 60 * 1000;
         // Default 5min; an explicit 0 disables the fallback (opt back into wake-forever-on-lost-event). A
         // non-finite or negative value falls back to the default rather than silently disabling the net
         // (NaN) or stamping a wake in the past that re-drives the run every tick (negative).
@@ -1336,7 +1352,8 @@ export class WorkflowEngine {
     }
     /**
      * Be notified when a run is **dead-lettered** — moved to `dead` after exceeding
-     * `maxRecoveryAttempts`. The listener receives the dead run (status `dead`, with its error), so a
+     * `maxRecoveryAttempts`, or because its workflow version is registered nowhere (see
+     * `unregisteredWorkflowGraceMs`). The listener receives the dead run (status `dead`, with its error), so a
      * DLQ handler can do something other than just leaving it parked: alert, push to a real queue, or
      * start a dead-letter workflow (e.g. `engine.onDead((run) => engine.start('pipeline-dlq', run, ...))`).
      * Returns an unsubscribe function.
@@ -2267,9 +2284,8 @@ export class WorkflowEngine {
                 this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ??
                     this.latest.get(run.workflow);
         }
-        if (!registered) {
-            throw new Error(`workflow ${run.workflow}@${run.workflowVersion} is not registered — keep the prior version deployed so in-flight runs can drain (skew protection)`);
-        }
+        if (!registered)
+            throw new WorkflowNotRegistered(run.workflow, run.workflowVersion);
         return this.track(runId, this.execute(run, registered.fn));
     }
     /**
@@ -2657,6 +2673,16 @@ export class WorkflowEngine {
                     results.push({ runId: run.id, status: 'suspended' });
                 continue;
             }
+            // An orphan of a workflow registered nowhere must not be counted or re-pended: both write
+            // `updatedAt`, so it would never age past the grace (see settleUnregistered).
+            if (!(await this.canResolveWorkflow(run))) {
+                const settled = await this.settleUnregistered(run, nowMs);
+                if (settled)
+                    results.push(settled);
+                else if (settled === undefined)
+                    this.logUnregisteredSkip(run);
+                continue;
+            }
             // Count the attempt / dead-letter a poison pill past maxRecoveryAttempts.
             const settled = await this.countRecovery(run);
             // `null`: the run left `running` between the re-read above and the dead-letter write (a
@@ -2821,38 +2847,93 @@ export class WorkflowEngine {
     async countRecovery(run) {
         const attempts = (run.recoveryAttempts ?? 0) + 1;
         if (this.maxRecoveryAttempts != null && attempts > this.maxRecoveryAttempts) {
-            const error = {
-                message: `run exceeded maxRecoveryAttempts (${this.maxRecoveryAttempts}) — moved to dead-letter`,
-                code: 'max_recovery_attempts',
-            };
             // Conditional on still-`running`, like the `pending` flip in recoverIncomplete: the lease does
             // not fence `cancel()`, which writes `cancelled` without it, so a cancel landing after the
             // caller's re-read must not be overwritten with `dead`.
-            const applied = await this.store.updateRunIf(run.id, ['running'], {
-                status: 'dead',
-                error,
-                updatedAt: new Date(),
-            });
-            await this.store.releaseRunLock(run.id, this.instanceId);
-            if (!applied)
-                return null;
-            this.emit({
-                type: 'run.failed',
-                runId: run.id,
-                workflow: run.workflow,
-                namespace: run.namespace,
-                error,
+            return this.deadLetterLeased({ ...run, recoveryAttempts: attempts }, ['running'], {
+                message: `run exceeded maxRecoveryAttempts (${this.maxRecoveryAttempts}) — moved to dead-letter`,
+                code: 'max_recovery_attempts',
             });
-            this.notifyDead({ ...run, status: 'dead', error, recoveryAttempts: attempts });
-            // `dead` is terminal like `failed`: a parent awaiting this run as a child (ctx.child, a ctx.all /
-            // allSettled fan) must learn it now, as a failed child, not on its next reconcile wake.
-            this.notifyParent(run.id, failedCompletion(error));
-            return { runId: run.id, status: 'dead', error };
         }
         // Count BEFORE resuming, so a crash mid-resume still advances the counter.
         await this.store.updateRun(run.id, { recoveryAttempts: attempts, updatedAt: new Date() });
         return undefined;
     }
+    /**
+     * Move a run whose lease this instance holds to `dead`, if it is still in one of `from`, and
+     * release the lease. Returns the terminal result, or `null` when the run left `from` meanwhile
+     * (e.g. a lease-free `cancel()`), so it is not ours to touch.
+     */
+    async deadLetterLeased(run, from, error) {
+        const applied = await this.store.updateRunIf(run.id, from, {
+            status: 'dead',
+            error,
+            updatedAt: new Date(),
+        });
+        await this.store.releaseRunLock(run.id, this.instanceId);
+        if (!applied)
+            return null;
+        this.emit({
+            type: 'run.failed',
+            runId: run.id,
+            workflow: run.workflow,
+            namespace: run.namespace,
+            error,
+        });
+        this.notifyDead({ ...run, status: 'dead', error });
+        // `dead` is terminal like `failed`: a parent awaiting this run as a child (ctx.child, a ctx.all /
+        // allSettled fan) must learn it now, as a failed child, not on its next reconcile wake.
+        this.notifyParent(run.id, failedCompletion(error));
+        return { runId: run.id, status: 'dead', error };
+    }
+    /**
+     * Whether this instance can run `run`'s pinned `workflow@version`: registered here, or routed by
+     * convention to a live worker group of the same name (the lookup {@link resumeRunNow} does).
+     */
+    async canResolveWorkflow(run) {
+        if (this.workflows.has(versionKey(run.workflow, run.workflowVersion)))
+            return true;
+        return this.ensureConventionWorkflow(run.workflow, run.namespace);
+    }
+    /**
+     * A run whose lease this instance holds but whose `workflow@version` nothing here can run (see
+     * {@link WorkflowNotRegistered}). Within {@link unregisteredWorkflowGraceMs} of becoming due it is a
+     * rolling deploy's skew: the lease is released at once so the pod that still has the version takes
+     * the run now, not after the lease expires. Past it, the workflow is gone from the whole fleet (a
+     * capable pod would have taken the run long ago): dead-letter it.
+     *
+     * The age is read off `updatedAt` / `wakeAt`, so every path that would touch either (a recovery
+     * count, a re-pend, a hand-off's lease-long `wakeAt`) must call this BEFORE writing — otherwise an
+     * orphan never ages.
+     *
+     * Returns the `dead` result, `null` when the run left its status meanwhile (a lease-free
+     * `cancel()`: settled, not ours to touch), or `undefined` when it was left for later.
+     */
+    async settleUnregistered(listed, nowMs) {
+        // Re-read under the lease: the listing is a snapshot (see recoverIncomplete).
+        const run = await this.store.getRun(listed.id);
+        if (!run || !UNREGISTERED_DEAD_LETTER_FROM.includes(run.status)) {
+            await this.store.releaseRunLock(listed.id, this.instanceId);
+            return null;
+        }
+        const dueSince = Math.max(run.updatedAt.getTime(), run.wakeAt ?? 0);
+        if (this.unregisteredWorkflowGraceMs === 0 ||
+            nowMs - dueSince < this.unregisteredWorkflowGraceMs) {
+            await this.store.releaseRunLock(run.id, this.instanceId);
+            return undefined;
+        }
+        const dead = await this.deadLetterLeased(run, [run.status], {
+            message: `workflow ${run.workflow}@${run.workflowVersion} is not registered on any live instance (unresumable for over ${this.unregisteredWorkflowGraceMs}ms) — moved to dead-letter`,
+            code: 'workflow_not_registered',
+        });
+        if (dead)
+            console.warn(`[adonis-durable] run ${run.id}: ${dead.error?.message}`);
+        return dead;
+    }
+    /** Log a run skipped for an unregistered workflow — still there, still not progressing. */
+    logUnregisteredSkip(run) {
+        console.error(`[adonis-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance or a later tick:`, new WorkflowNotRegistered(run.workflow, run.workflowVersion));
+    }
     /**
      * Resume every suspended run whose durable timer is due — AND every run parked `blocked` (no
      * capable/compatible worker, design §7.5) whose recovery poll is due. Call periodically (a poller)
@@ -2891,6 +2972,15 @@ export class WorkflowEngine {
                     if (!(await this.store.tryLockRun(run.id, this.instanceId, nowMs + this.leaseMs, nowMs))) {
                         continue;
                     }
+                    // Checked before the lease-long `wakeAt` below, which would keep the orphan from aging.
+                    if (!(await this.canResolveWorkflow(run))) {
+                        const settled = await this.settleUnregistered(run, nowMs);
+                        if (settled)
+                            results.push(settled);
+                        else if (settled === undefined)
+                            this.logUnregisteredSkip(run);
+                        continue;
+                    }
                     try {
                         // The lease-long `wakeAt` keeps the run due if the wake is lost on the way, so a refused
                         // wake's request it answers is not needed any more (and would keep it due every poll).
@@ -2949,9 +3039,15 @@ export class WorkflowEngine {
         }
         // resume() checks the namespace and throws NamespaceMismatch when it doesn't match (releasing the
         // lease) — a foreign run leased here is simply skipped rather than run.
-        return this.resumeImplicit(runId).catch((err) => {
+        return this.resumeImplicit(runId).catch(async (err) => {
             if (err instanceof NamespaceMismatch)
                 return null;
+            if (err instanceof WorkflowNotRegistered) {
+                const run = await this.store.getRun(runId);
+                const settled = run ? await this.settleUnregistered(run, nowMs) : null;
+                if (settled !== undefined)
+                    return settled;
+            }
             throw err;
         });
     }
@@ -3283,6 +3379,13 @@ export class WorkflowEngine {
                     // A foreign-namespace run is simply not ours (resume released its lease) — skip quietly.
                     if (error instanceof NamespaceMismatch)
                         continue;
+                    if (error instanceof WorkflowNotRegistered) {
+                        const settled = await this.settleUnregistered(run, nowMs).catch(() => undefined);
+                        if (settled)
+                            results.push(settled);
+                        if (settled !== undefined)
+                            continue;
+                    }
                     // Loud: a skip is not nothing — the run is still there and not progressing, and the reason
                     // (a version skew, a store outage) is what tells an operator which.
                     console.error(`[adonis-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance or a later tick:`, error);