@pikku/core 0.12.130 → 0.12.134

package/dist/wirings/workflow/pikku-workflow-service.js

@@ -1,23 +1,28 @@
 import { runPikkuFunc } from '../../function/function-runner.js';
 import { getSingletonServices, getCreateWireServices, pikkuState, } from '../../pikku-state.js';
 import { getDurationInMilliseconds } from '../../time-utils.js';
-import { ChildWorkflowStartedException, continueGraph, executeGraphStep, runWorkflowGraph, runFromMeta, stripInstanceOrdinal, } from './graph/graph-runner.js';
+import { ChildWorkflowStartedException, continueGraph, executeGraphStep, runWorkflowGraph, stripInstanceOrdinal, } from './graph/graph-runner.js';
 import { isExpectedError } from '../../errors/error-handler.js';
 import { PikkuMissingMetaError } from '../../errors/errors.js';
 import { RPCNotFoundError } from '../rpc/rpc-runner.js';
 import { deriveInvocationId } from './workflow-invocation-id.js';
 import { approvalDeciderFrom } from './workflow-approval-policy.js';
 import { buildRunTimeline, reconstructStateAt, } from './run-timeline.js';
-import { DEFAULT_STEP_RETRIES, WORKFLOW_CHILD_POLL_MAX_MS, WORKFLOW_END_STATES, WORKFLOW_POLL_FACTOR, WORKFLOW_POLL_MIN_MS, WORKFLOW_TERMINAL_STATES, isRunSettled, } from './workflow-constants.js';
-import { WorkflowAsyncException, WorkflowCancelledException, WorkflowDispatchException, WorkflowNotFoundError, WorkflowRunCancelledError, WorkflowRunFailedError, WorkflowRunNotFoundError, WorkflowStepNameNotString, WorkflowSuspendedException, } from './workflow-errors.js';
+import { DEFAULT_STEP_RETRIES, RUN_LEASE_RETRY_MS, WORKFLOW_CHILD_POLL_MAX_MS, WORKFLOW_END_STATES, WORKFLOW_POLL_FACTOR, WORKFLOW_POLL_MIN_MS, WORKFLOW_TERMINAL_STATES, isRunSettled, } from './workflow-constants.js';
+import { WorkflowAsyncException, WorkflowCancelledException, WorkflowDispatchException, WorkflowNotFoundError, WorkflowRunCancelledError, WorkflowRunFailedError, WorkflowRunNotFoundError, WorkflowStepNameNotString, WorkflowStepSupersededError, WorkflowSuspendedException, } from './workflow-errors.js';
 import { resolveWorkflowMeta } from './workflow-meta-resolver.js';
 import { storeWorkflowVersion } from './workflow-version-store.js';
-import { jobGroupFor, orchestratorQueueName, resolveWorkflowConfig, stepDispatchTarget, stepJobOptions, stepWorkerQueueName, } from './workflow-queue-routing.js';
+import { jobGroupFor, orchestratorQueueName, requireQueueService, resolveWorkflowConfig, stepDispatchTarget, stepJobOptions, stepWorkerQueueName, } from './workflow-queue-routing.js';
 import { wireWorkflowQueueWorkers } from './workflow-queue-wiring.js';
 import { approvalStepNameFor, evaluateApprovalStep, recordApprovalDecision, } from './workflow-approval.js';
 import { auditApprovalDecision } from './workflow-approval-audit.js';
 import { recordSuspension, suspendStepNameFor } from './workflow-suspend.js';
 import { claimStepByReadThenWrite } from './workflow-step-claim.js';
+import { holdWorkflowLease, isRunLeaseError, nullWhenStepHeld, } from './workflow-run-lease.js';
+import { summarizeRunStatus } from './workflow-run-status.js';
+import { runningStepLease, startStepLeaseRefresh, stepLeaseMsForQueue, } from './workflow-step-lease.js';
+import { runInlineRetryLoop } from './workflow-step-retry.js';
+import { runVersionMismatchFallback } from './workflow-version-fallback.js';
 import { RedispatchBackoff, sweepStalledRuns, sweepUndispatchedSteps, } from './workflow-recovery.js';
 export class PikkuWorkflowService {
     runExtension;
@@ -61,12 +66,19 @@ export class PikkuWorkflowService {
         return getSingletonServices()?.logger;
     }
     mirror;
+    /**
+     * Serialises orchestration passes over a run and claims on a step. A service
+     * whose state no other process can reach — in-memory, a Durable Object —
+     * has no competitor to exclude, overrides both locks and passes none.
+     */
+    leaseService;
     queueStrategy;
     queueConcurrency;
     queueGroupConcurrency;
     constructor(options = {}) {
         const wireQueues = options.wireQueues ?? true;
         this.mirror = options.mirror;
+        this.leaseService = options.leaseService;
         this.queueStrategy = options.queueStrategy ?? 'per-workflow';
         this.queueConcurrency = options.queueConcurrency ?? 20;
         this.queueGroupConcurrency = options.queueGroupConcurrency ?? 2;
@@ -136,41 +148,7 @@ export class PikkuWorkflowService {
         const run = await this.getRun(id);
         if (!run)
             return null;
-        const history = await this.getRunHistory(id);
-        const terminalStatuses = new Set(['completed', 'failed', 'cancelled']);
-        const stepMap = new Map();
-        for (const step of history) {
-            const existing = stepMap.get(step.stepName);
-            if (!existing || step.updatedAt > existing.completedAt) {
-                stepMap.set(step.stepName, {
-                    status: step.status,
-                    startedAt: step.runningAt ?? step.createdAt,
-                    completedAt: step.succeededAt ?? step.failedAt,
-                    attempts: step.attemptCount,
-                });
-            }
-        }
-        const steps = [...stepMap.entries()].map(([name, s]) => ({
-            name,
-            status: s.status,
-            duration: s.startedAt && s.completedAt
-                ? s.completedAt.getTime() - s.startedAt.getTime()
-                : undefined,
-            attempts: s.attempts,
-        }));
-        return {
-            id: run.id,
-            status: run.status,
-            startedAt: run.createdAt,
-            completedAt: terminalStatuses.has(run.status) ? run.updatedAt : undefined,
-            deterministic: run.deterministic,
-            plannedSteps: run.plannedSteps,
-            steps,
-            output: run.status === 'completed' ? run.output : undefined,
-            error: run.error
-                ? { message: run.error.message ?? 'Unknown error' }
-                : undefined,
-        };
+        return summarizeRunStatus(run, await this.getRunHistory(id));
     }
     async getRunTimeline(id) {
         const run = await this.getRun(id);
@@ -197,17 +175,40 @@ export class PikkuWorkflowService {
     async setStepRunning(stepId) {
         await this.mirrored(() => this.setStepRunningImpl(stepId), (mirror) => mirror.setStepRunning(stepId));
     }
+    /**
+     * Push the claim on a `running` step forward, so it keeps reading as owned
+     * for another lease.
+     *
+     * Does nothing by default, which leaves a store that records no lease exactly
+     * as it was: its `running` steps are owned until they move on their own, and
+     * a dispatch that dies takes the step with it. A store that overrides this
+     * gains recovery from a lost worker, through both `claimStepForExecution` and
+     * `recoverStalledRuns`.
+     *
+     * `null` releases the lease without ending the step, for a step that is
+     * legitimately `running` with no worker on it — one parked on a child run,
+     * which the child's completion drives rather than a redispatch.
+     * `attempt` fences it to the claim that holds it.
+     *
+     * The lease runs `leaseMs` from now on the store's own clock, so workers
+     * whose clocks disagree still agree on when it lapses. Resolves `false` when
+     * the step is no longer this attempt's to renew.
+     */
+    async refreshStepLease(_stepId, _leaseMs, _attempt) {
+        return true;
+    }
     async setStepScheduled(stepId) {
         await this.mirrored(() => this.setStepScheduledImpl(stepId), (mirror) => mirror.setStepScheduled(stepId));
     }
-    async setStepResult(stepId, result) {
-        await this.mirrored(() => this.setStepResultImpl(stepId, result), (mirror) => mirror.setStepResult(stepId, result));
+    /** `attempt` fences it: a newer claim's step throws `WorkflowStepSupersededError`. */
+    async setStepResult(stepId, result, attempt) {
+        await this.mirrored(() => this.setStepResultImpl(stepId, result, attempt), (mirror) => mirror.setStepResult(stepId, result));
     }
     async setStepChildRunId(stepId, childRunId) {
         await this.mirrored(() => this.setStepChildRunIdImpl(stepId, childRunId), (mirror) => mirror.setStepChildRunId(stepId, childRunId));
     }
-    async setStepError(stepId, error) {
-        await this.mirrored(() => this.setStepErrorImpl(stepId, error), (mirror) => {
+    async setStepError(stepId, error, attempt) {
+        await this.mirrored(() => this.setStepErrorImpl(stepId, error, attempt), (mirror) => {
             const serialized = {
                 message: error.message,
                 stack: error.stack,
@@ -223,6 +224,15 @@ export class PikkuWorkflowService {
             stepName: newStep.stepName ?? '',
         }));
     }
+    /** One orchestration pass under the run's lease; never waits (see `holdWorkflowLease`). */
+    async withRunLease(id, fn) {
+        return holdWorkflowLease(this, this.leaseService, `workflow-run:${id}`, fn);
+    }
+    /** `fn` under one step's lease; never waits (see `holdWorkflowLease`). */
+    async withStepLock(runId, stepName, fn) {
+        const key = `workflow-step:${runId}:${stepName}`;
+        return holdWorkflowLease(this, this.leaseService, key, fn);
+    }
     async setBranchTaken(stepId, branchKey) {
         await this.mirrored(() => this.setBranchTakenImpl(stepId, branchKey), (mirror) => mirror.setBranchTaken(stepId, branchKey));
     }
@@ -236,20 +246,25 @@ export class PikkuWorkflowService {
         await this.mirrored(() => this.updateWorkflowVersionStatusImpl(name, graphHash, status), (mirror) => mirror.updateWorkflowVersionStatus(name, graphHash, status));
     }
     async resumeWorkflow(runId, workflowName) {
-        const queueService = this.verifyQueueService();
-        if (!workflowName) {
-            const run = await this.getRun(runId);
-            workflowName = run?.workflow;
-        }
+        await this.enqueueOrchestration(runId, workflowName);
+    }
+    async enqueueOrchestration(runId, workflowName, delay) {
+        const queueService = requireQueueService();
+        workflowName ??= (await this.getRun(runId))?.workflow;
         await queueService.add(this.getOrchestratorQueueName(workflowName), { runId }, {
             ...this.resolveStepJobOptions(),
+            ...(delay !== undefined ? { delay } : {}),
             group: this.getJobGroup(workflowName),
         });
     }
     /**
      * Ids of runs that are stalled: still `running`, with no step in a state that
-     * something is expected to complete (`running`, `scheduled`, `suspended`),
-     * and no step activity since `before`.
+     * something is expected to complete (`scheduled`, `suspended`, or `running`
+     * under a live lease), and no step activity since `before`.
+     *
+     * A `running` step whose lease has lapsed is the opposite of in flight — the
+     * dispatch that claimed it is gone — so it does not hold its run out of the
+     * sweep.
      *
      * Returns nothing by default so a store that cannot express the query keeps
      * working unchanged; a store that overrides it gains crash recovery through
@@ -301,7 +316,7 @@ export class PikkuWorkflowService {
         return stepJobOptions(stepOptions);
     }
     async queueStepWorker(runId, stepName, rpcName, data, stepOptions, fromStepName) {
-        const queueService = this.verifyQueueService();
+        const queueService = requireQueueService();
         await queueService.add(this.getStepWorkerQueueName(rpcName), { runId, stepName, rpcName, data, fromStepName }, {
             ...this.resolveStepJobOptions(stepOptions),
             group: this.getJobGroup(rpcName),
@@ -312,7 +327,7 @@ export class PikkuWorkflowService {
         await this.resumeWorkflow(runId);
     }
     async scheduleOrchestratorRetry(runId, retryDelay, workflowName) {
-        const queueService = this.verifyQueueService();
+        const queueService = requireQueueService();
         if (!workflowName) {
             const run = await this.getRun(runId);
             workflowName = run?.workflow;
@@ -409,7 +424,8 @@ export class PikkuWorkflowService {
                 if (error.name !== 'WorkflowAsyncException' &&
                     error.name !== 'WorkflowCancelledException' &&
                     error.name !== 'WorkflowSuspendedException' &&
-                    error.name !== 'WorkflowDispatchException') {
+                    error.name !== 'WorkflowDispatchException' &&
+                    !isRunLeaseError(error)) {
                     await this.updateRunStatus(runId, 'failed', undefined, {
                         name: error.name,
                         message: error.message,
@@ -418,7 +434,11 @@ export class PikkuWorkflowService {
                     getSingletonServices().logger.error(`Workflow ${name} (run ${runId}) failed:`, isExpectedError(error) ? error.message : error);
                     throw error;
                 }
-                if (error.name === 'WorkflowDispatchException') {
+                // A lease error says nothing about how the run went — a lost lease is
+                // only seen once the body has already written its outcome — so the
+                // run keeps whatever status it reached.
+                if (error.name === 'WorkflowDispatchException' ||
+                    isRunLeaseError(error)) {
                     throw error;
                 }
             }
@@ -551,19 +571,24 @@ export class PikkuWorkflowService {
         if (run.graphHash &&
             workflowMeta?.graphHash &&
             run.graphHash !== workflowMeta.graphHash) {
-            await this.runVersionMismatchFallback(run, workflowMeta, rpcService);
+            await this.withRunLease(runId, () => runVersionMismatchFallback(this, run, workflowMeta, rpcService));
             return;
         }
         if (workflowMeta?.source === 'graph') {
-            await continueGraph(this, runId, run.workflow);
-            const updatedRun = await this.getRun(runId);
-            if (updatedRun?.status === 'completed') {
-                await this.onChildWorkflowCompleted(updatedRun, updatedRun.output);
-            }
-            else if (updatedRun?.status === 'failed' ||
-                updatedRun?.status === 'cancelled') {
-                await this.onChildWorkflowFailed(updatedRun, new Error(updatedRun.error?.message || 'Child workflow failed'));
-            }
+            // Two passes over one graph each plan from the state they read, so both
+            // can fire the same next node; the run lease keeps them one at a time,
+            // exactly as it does for a DSL run's replay.
+            await this.withRunLease(runId, async () => {
+                await continueGraph(this, runId, run.workflow);
+                const updatedRun = await this.getRun(runId);
+                if (updatedRun?.status === 'completed') {
+                    await this.onChildWorkflowCompleted(updatedRun, updatedRun.output);
+                }
+                else if (updatedRun?.status === 'failed' ||
+                    updatedRun?.status === 'cancelled') {
+                    await this.onChildWorkflowFailed(updatedRun, new Error(updatedRun.error?.message || 'Child workflow failed'));
+                }
+            });
             return;
         }
         const registrations = pikkuState(pkgName, 'workflows', 'registrations');
@@ -574,7 +599,7 @@ export class PikkuWorkflowService {
         if (!workflow) {
             throw new WorkflowNotFoundError(run.workflow);
         }
-        await this.withRunLock(runId, async () => {
+        await this.withRunLease(runId, async () => {
             const addonNs = run.workflow.includes(':')
                 ? run.workflow.substring(0, run.workflow.indexOf(':'))
                 : null;
@@ -667,32 +692,17 @@ export class PikkuWorkflowService {
         await this.setStepError(parentStepId, error);
         await this.resumeWorkflow(parentRunId);
     }
-    async runVersionMismatchFallback(run, currentMeta, rpcService) {
-        const source = currentMeta.source;
-        if (source === 'complex') {
-            await this.updateRunStatus(run.id, 'failed', undefined, {
-                message: `Workflow '${run.workflow}' definition changed. Complex workflows with inline steps cannot be migrated.`,
-                stack: '',
-                code: 'VERSION_CONFLICT',
-            });
-            return;
-        }
-        const version = await this.getWorkflowVersion(run.workflow, run.graphHash);
-        if (!version) {
-            await this.updateRunStatus(run.id, 'failed', undefined, {
-                message: `Workflow '${run.workflow}' version '${run.graphHash}' not found. Cannot resume with changed definition.`,
-                stack: '',
-                code: 'VERSION_NOT_FOUND',
-            });
-            return;
-        }
-        await runFromMeta(this, run.id, version.graph, rpcService);
-    }
     async executeWorkflowStep(runId, stepName, rpcName, data, rpcService) {
         this.enterExecution(runId);
         try {
             await this.executeWorkflowStepInner(runId, stepName, rpcName, data, rpcService);
         }
+        catch (error) {
+            // A newer claim owns the step and its outcome; this dispatch's is dropped.
+            if (!(error instanceof WorkflowStepSupersededError))
+                throw error;
+            this.logger?.warn(error.message);
+        }
         finally {
             this.exitExecution(runId);
         }
@@ -705,20 +715,28 @@ export class PikkuWorkflowService {
      * believes were dropped, and a queue can redeliver a job it already handed
      * out. This is the one place that keeps a duplicate dispatch from becoming a
      * second execution of a side-effecting step, so it is only as strong as the
-     * exclusion it is built on — and `withStepLock` excludes nothing unless the
-     * store backs it with a real primitive. A store able to express the decision
-     * as one conditional write should override this rather than reach for a lock,
-     * which is what `@pikku/kysely` does with a status-guarded `UPDATE`.
+     * exclusion it is built on: here, the step's lease on the app's
+     * `leaseService`. A store able to express the decision as one conditional
+     * write should override this rather than reach for a lock, which is what
+     * `@pikku/kysely` does with a status-guarded `UPDATE`.
+     *
+     * Ownership is held for `leaseMs` rather than forever. A dispatch that
+     * dies mid-step leaves the step `running` with nothing to complete it, and an
+     * unconditional rejection of `running` would then wedge it beyond the reach
+     * of both this claim and the stalled-run sweep.
      */
-    async claimStepForExecution(runId, stepName, rpcName) {
-        return this.withStepLock(runId, stepName, () => claimStepByReadThenWrite(this, runId, stepName, rpcName));
+    async claimStepForExecution(runId, stepName, rpcName, leaseMs) {
+        return nullWhenStepHeld(() => this.withStepLock(runId, stepName, () => claimStepByReadThenWrite(this, runId, stepName, rpcName, leaseMs)));
     }
     async executeWorkflowStepInner(runId, stepName, rpcName, data, rpcService) {
-        const claimed = await this.claimStepForExecution(runId, stepName, rpcName);
+        const leaseMs = stepLeaseMsForQueue(this.getStepWorkerQueueName(rpcName));
+        const claimed = await this.claimStepForExecution(runId, stepName, rpcName, leaseMs);
         if (!claimed) {
             return;
         }
         const stepState = claimed;
+        const attempt = stepState.attemptCount;
+        const stopLeaseRefresh = startStepLeaseRefresh(stepState.stepId, leaseMs, () => this.refreshStepLease(stepState.stepId, leaseMs, attempt));
         try {
             let result;
             const run = await this.getRun(runId);
@@ -773,23 +791,28 @@ export class PikkuWorkflowService {
                     result = await this.invokeStepRpc(runId, stepName, stepState, rpcName, data, rpcService, run);
                 }
             }
-            await this.setStepResult(stepState.stepId, result);
+            await this.setStepResult(stepState.stepId, result, attempt);
             await this.resumeWorkflow(runId);
         }
         catch (error) {
             if (error instanceof ChildWorkflowStartedException) {
+                // The step stays `running` with no worker on it, which is not the same
+                // as abandoned: the child run is what completes it. Releasing the lease
+                // says so, once no renewal in flight can undo it.
+                await stopLeaseRefresh();
+                await this.refreshStepLease(stepState.stepId, null, attempt);
                 this.logger?.debug(`Workflow step '${stepName}': child workflow ${error.childRunId} started, waiting for completion`);
                 return;
             }
             if (error instanceof RPCNotFoundError) {
-                await this.setStepError(stepState.stepId, error);
+                await this.setStepError(stepState.stepId, error, attempt);
                 await this.updateRunStatus(runId, 'suspended', undefined, {
                     message: `RPC '${rpcName}' not found. Deploy the missing function and resume.`,
                     code: 'RPC_NOT_FOUND',
                 });
                 return;
             }
-            await this.setStepError(stepState.stepId, error);
+            await this.setStepError(stepState.stepId, error, attempt);
             const maxAttempts = (stepState.retries ?? DEFAULT_STEP_RETRIES) + 1;
             const retriesExhausted = stepState.attemptCount >= maxAttempts;
             if (retriesExhausted) {
@@ -797,6 +820,9 @@ export class PikkuWorkflowService {
             }
             throw error;
         }
+        finally {
+            await stopLeaseRefresh();
+        }
     }
     async orchestrateWorkflow(runId, rpcService) {
         try {
@@ -808,6 +834,16 @@ export class PikkuWorkflowService {
                 error.name === 'WorkflowSuspendedException') {
                 return;
             }
+            // Another orchestrator holds the run. Its pass may already have read the
+            // state this message was sent to report, so the message cannot simply be
+            // dropped — and handing it back to the queue spends the retry budget the
+            // queue keeps for real failures, which a long pass exhausts. Wake the run
+            // again once the holder has had time to finish.
+            if (isRunLeaseError(error)) {
+                getSingletonServices().logger.info(`Workflow run ${runId} is being orchestrated elsewhere; waking it again in ${RUN_LEASE_RETRY_MS}ms`);
+                await this.enqueueOrchestration(runId, undefined, RUN_LEASE_RETRY_MS);
+                return;
+            }
             if (error.name === 'WorkflowDispatchException') {
                 getSingletonServices().logger.warn(`Workflow run ${runId} could not dispatch a step (queue unavailable); leaving run for orchestrator retry`, error);
                 throw error;
@@ -820,12 +856,6 @@ export class PikkuWorkflowService {
             throw error;
         }
     }
-    verifyQueueService() {
-        if (!getSingletonServices()?.queueService) {
-            throw new Error('QueueService not configured. Remote workflows require a queue service.');
-        }
-        return getSingletonServices().queueService;
-    }
     async invokeStepRpc(runId, stepName, stepState, rpcName, data, rpcService, knownRun) {
         const run = knownRun ?? (await this.getRunIdentity(runId));
         return rpcService.rpcWithWire(rpcName, data, {
@@ -841,34 +871,6 @@ export class PikkuWorkflowService {
             },
         });
     }
-    async runInlineRetryLoop(stepState, retries, retryDelay, doWork, onError) {
-        let currentStepState = stepState;
-        while (true) {
-            try {
-                await this.setStepRunning(currentStepState.stepId);
-                const result = await doWork(currentStepState);
-                await this.setStepResult(currentStepState.stepId, result);
-                return result;
-            }
-            catch (error) {
-                if (onError)
-                    await onError(error);
-                await this.setStepError(currentStepState.stepId, error);
-                if (currentStepState.attemptCount < retries) {
-                    currentStepState = await this.createRetryAttempt(currentStepState.stepId, 'pending');
-                    if (retryDelay) {
-                        await new Promise((resolve) => setTimeout(resolve, getDurationInMilliseconds(retryDelay)));
-                    }
-                }
-                else {
-                    throw error;
-                }
-            }
-        }
-    }
-    async runStepCompensation(runId, stepName, onErrorRpcName, rpcService, error) {
-        await this.rpcStep(runId, `${stepName}:onError`, onErrorRpcName, { error: { message: error.message } }, rpcService, { retries: 0 });
-    }
     async rpcStep(runId, logicalStepName, rpcName, data, rpcService, stepOptions) {
         const fromStepName = this.lastStepName(runId);
         const stepName = this.nextStepKey(runId, logicalStepName);
@@ -886,7 +888,7 @@ export class PikkuWorkflowService {
             const error = new Error(stepState.error?.message ||
                 `Step '${stepName}' failed after exhausting all retries`);
             if (resolvedStepOptions.onError) {
-                await this.runStepCompensation(runId, stepName, resolvedStepOptions.onError, rpcService, error);
+                await this.rpcStep(runId, `${stepName}:onError`, resolvedStepOptions.onError, { error: { message: error.message } }, rpcService, { retries: 0 });
             }
             if (stepState.error) {
                 Object.assign(error, stepState.error);
@@ -896,16 +898,22 @@ export class PikkuWorkflowService {
         if (stepState.status === 'scheduled') {
             throw new WorkflowAsyncException(runId, stepName);
         }
+        const lease = runningStepLease(stepState);
+        if (lease === 'held') {
+            throw new WorkflowAsyncException(runId, stepName);
+        }
         const dispatched = resolvedStepOptions.actor
             ? false
             : await this.dispatchStep(runId, stepName, rpcName, data, resolvedStepOptions, fromStepName);
         if (dispatched) {
-            await this.setStepScheduled(stepState.stepId);
+            if (lease !== 'lapsed') {
+                await this.setStepScheduled(stepState.stepId);
+            }
             throw new WorkflowAsyncException(runId, stepName);
         }
         const retries = resolvedStepOptions.retries ?? this.getConfig().retries;
         const retryDelay = resolvedStepOptions.retryDelay;
-        return this.runInlineRetryLoop(stepState, retries, retryDelay, async (currentStepState) => {
+        return runInlineRetryLoop(this, stepState, retries, retryDelay, async (currentStepState) => {
             if (resolvedStepOptions.actor) {
                 return resolvedStepOptions.actor.invoke(rpcName, data);
             }
@@ -949,7 +957,7 @@ export class PikkuWorkflowService {
         const retries = stepOptions?.retries ?? this.getConfig().retries;
         const retryDelay = stepOptions?.retryDelay ?? this.getConfig().retryDelay;
         if (await this.isInline(runId)) {
-            return this.runInlineRetryLoop(stepState, retries, retryDelay, () => fn());
+            return runInlineRetryLoop(this, stepState, retries, retryDelay, () => fn());
         }
         else {
             let currentStepState = stepState;
@@ -1001,15 +1009,10 @@ export class PikkuWorkflowService {
     }
     async scheduleRunWake(runId, delay) {
         try {
-            const queueService = this.verifyQueueService();
             const run = await this.getRun(runId);
             if (!run?.workflow)
                 return;
-            await queueService.add(this.getOrchestratorQueueName(run.workflow), { runId }, {
-                ...this.resolveStepJobOptions(),
-                delay,
-                group: this.getJobGroup(run.workflow),
-            });
+            await this.enqueueOrchestration(runId, run.workflow, delay);
         }
         catch (error) {
             this.logger?.warn(`Failed to schedule approval expiry wake for run ${runId}; expiry will still resolve on the next replay`, error);

package/dist/wirings/workflow/workflow-constants.d.ts

@@ -10,7 +10,7 @@ export declare const WORKFLOW_TERMINAL_STATES: ReadonlySet<string>;
  * such a message is routine rather than exceptional: the queue is
  * at-least-once, the relay re-dispatches on purpose, and a run can settle
  * while a message for it is still in flight. Replaying one is not free —
- * `runWorkflowJob` takes the run lock and re-enters the workflow body, and a
+ * `runWorkflowJob` takes the run lease and re-enters the workflow body, and a
  * body re-entered after its run failed can park on a wait that nothing will
  * ever satisfy, holding the lock and the connection under it until something
  * external gives up. Every leaked advisory lock seen in production traced back
@@ -39,5 +39,32 @@ export declare const DEFAULT_UNDISPATCHED_STEP_LIMIT = 100;
 export declare const REDISPATCH_BACKOFF_MS = 30000;
 /** Ceiling on the doubling backoff, so a permanently stuck step still gets swept. */
 export declare const REDISPATCH_BACKOFF_MAX_MS: number;
+/**
+ * How long a claim on a step is good for when the queue it was dispatched
+ * through declares no duration of its own. Sits above the queue's own lock so
+ * the two never disagree about who owns the job.
+ */
+export declare const DEFAULT_STEP_LEASE_MS = 60000;
+/**
+ * The refresh interval below which a lease is short enough to be worth saying
+ * so. It is a warning threshold, not a floor: applied as one it would push the
+ * first refresh past the expiry of any lease under three times this, which is how a
+ * step ends up claimed twice.
+ */
+export declare const STEP_LEASE_REFRESH_MIN_MS = 1000;
+/**
+ * Whether a `running` step is still owned by the dispatch that claimed it.
+ *
+ * No lease at all counts as live: a store that records none has no way to tell
+ * an abandoned step from a working one, so it keeps the older, safer rule that
+ * a `running` step is never taken from underneath its holder.
+ */
+export declare const isStepLeaseLive: (leaseExpiresAt: Date | null | undefined, now?: number) => boolean;
+/**
+ * How long an orchestrator message that found its run held waits before it
+ * wakes the run again. Long enough for the holder's pass to finish, short
+ * enough that the run does not visibly stall behind it.
+ */
+export declare const RUN_LEASE_RETRY_MS = 1000;
 /** Bound on the in-process backoff map, so a long-lived process cannot grow it without bound. */
 export declare const REDISPATCH_BACKOFF_MAX_ENTRIES = 10000;

package/dist/wirings/workflow/workflow-constants.js

@@ -19,7 +19,7 @@ export const WORKFLOW_TERMINAL_STATES = new Set([
  * such a message is routine rather than exceptional: the queue is
  * at-least-once, the relay re-dispatches on purpose, and a run can settle
  * while a message for it is still in flight. Replaying one is not free —
- * `runWorkflowJob` takes the run lock and re-enters the workflow body, and a
+ * `runWorkflowJob` takes the run lease and re-enters the workflow body, and a
  * body re-entered after its run failed can park on a wait that nothing will
  * ever satisfy, holding the lock and the connection under it until something
  * external gives up. Every leaked advisory lock seen in production traced back
@@ -48,5 +48,32 @@ export const DEFAULT_UNDISPATCHED_STEP_LIMIT = 100;
 export const REDISPATCH_BACKOFF_MS = 30_000;
 /** Ceiling on the doubling backoff, so a permanently stuck step still gets swept. */
 export const REDISPATCH_BACKOFF_MAX_MS = 10 * 60_000;
+/**
+ * How long a claim on a step is good for when the queue it was dispatched
+ * through declares no duration of its own. Sits above the queue's own lock so
+ * the two never disagree about who owns the job.
+ */
+export const DEFAULT_STEP_LEASE_MS = 60_000;
+/**
+ * The refresh interval below which a lease is short enough to be worth saying
+ * so. It is a warning threshold, not a floor: applied as one it would push the
+ * first refresh past the expiry of any lease under three times this, which is how a
+ * step ends up claimed twice.
+ */
+export const STEP_LEASE_REFRESH_MIN_MS = 1_000;
+/**
+ * Whether a `running` step is still owned by the dispatch that claimed it.
+ *
+ * No lease at all counts as live: a store that records none has no way to tell
+ * an abandoned step from a working one, so it keeps the older, safer rule that
+ * a `running` step is never taken from underneath its holder.
+ */
+export const isStepLeaseLive = (leaseExpiresAt, now = Date.now()) => leaseExpiresAt == null || leaseExpiresAt.getTime() > now;
+/**
+ * How long an orchestrator message that found its run held waits before it
+ * wakes the run again. Long enough for the holder's pass to finish, short
+ * enough that the run does not visibly stall behind it.
+ */
+export const RUN_LEASE_RETRY_MS = 1_000;
 /** Bound on the in-process backoff map, so a long-lived process cannot grow it without bound. */
 export const REDISPATCH_BACKOFF_MAX_ENTRIES = 10_000;

package/dist/wirings/workflow/workflow-errors.d.ts

@@ -57,6 +57,27 @@ export declare class WorkflowStepFunctionMismatchError extends PikkuError {
     readonly stepName: string;
     constructor(runId: string, stepName: string);
 }
+/**
+ * Every dispatch that took this step lost its worker before finishing it, and
+ * the step has no attempts left to hand out. Failing here is the loud end of
+ * the loop: without it the step would be re-claimed and abandoned forever.
+ */
+export declare class WorkflowStepLeaseExpiredError extends PikkuError {
+    readonly runId: string;
+    readonly stepName: string;
+    readonly attemptCount: number;
+    constructor(runId: string, stepName: string, attemptCount: number);
+}
+/**
+ * A write from a dispatch that no longer owns its step: its lease lapsed and
+ * another dispatch claimed the step as a newer attempt. The newer attempt owns
+ * the outcome, so the stale one is dropped rather than recorded over it.
+ */
+export declare class WorkflowStepSupersededError extends PikkuError {
+    readonly stepId: string;
+    readonly attempt: number;
+    constructor(stepId: string, attempt: number);
+}
 export declare class WorkflowStepNameNotString extends Error {
     constructor(stepName: unknown);
 }