@pikku/core 0.12.37 → 0.12.39

package/dist/wirings/workflow/pikku-workflow-service.js

@@ -29,9 +29,19 @@ const resolveWorkflowMeta = (name) => {
 };
 const toKebab = (s) => s.replace(/([a-z0-9])([A-Z])/g, '$1-$2').toLowerCase();
 import { continueGraph, executeGraphStep, runWorkflowGraph, runFromMeta, } from './graph/graph-runner.js';
-import { PikkuError, addError } from '../../errors/error-handler.js';
+import { PikkuError, addError, isExpectedError, } from '../../errors/error-handler.js';
 import { RPCNotFoundError } from '../rpc/rpc-runner.js';
 import { ChildWorkflowStartedException } from './graph/graph-runner.js';
+import { deriveInvocationId } from './workflow-invocation-id.js';
+import { buildRunTimeline, reconstructStateAt, } from './run-timeline.js';
+/**
+ * Default number of retries for a workflow step when none is specified. The
+ * workflow — not the queue — owns retry policy; a step inherits this unless it
+ * sets its own `retries` (including `retries: 0` to opt out entirely). Picked >0
+ * so a transient failure (a DB blip, a downstream restart, a deploy) is ridden
+ * out by default; safe because every step gets a stable `invocationId` to dedupe on.
+ */
+export const DEFAULT_STEP_RETRIES = 5;
 /**
  * Exception thrown when workflow needs to pause for async step
  */
@@ -71,6 +81,24 @@ export class WorkflowSuspendedException extends Error {
         this.name = 'WorkflowSuspendedException';
     }
 }
+/**
+ * Thrown when a step (or the orchestrator) could not be enqueued — the queue
+ * itself failed (e.g. pg-boss is momentarily down), NOT the step's own logic.
+ * This is transient infrastructure failure: the run is left untouched (the step
+ * stays `pending`, the run stays running) and the orchestrator job is rethrown
+ * so the queue redelivers it and the workflow replays from its snapshot. Treat
+ * it as non-terminal — never mark the run `failed` for it.
+ */
+export class WorkflowDispatchException extends Error {
+    runId;
+    stepName;
+    constructor(runId, stepName, options) {
+        super(`Failed to dispatch workflow step '${stepName}' (run ${runId})`, options);
+        this.runId = runId;
+        this.stepName = stepName;
+        this.name = 'WorkflowDispatchException';
+    }
+}
 /**
  * Error class for workflow not found
  */
@@ -266,6 +294,7 @@ export class PikkuWorkflowService {
                     status: step.status,
                     startedAt: step.runningAt ?? step.createdAt,
                     completedAt: step.succeededAt ?? step.failedAt,
+                    attempts: step.attemptCount,
                 });
             }
         }
@@ -275,6 +304,7 @@ export class PikkuWorkflowService {
             duration: s.startedAt && s.completedAt
                 ? s.completedAt.getTime() - s.startedAt.getTime()
                 : undefined,
+            attempts: s.attempts,
         }));
         return {
             id: run.id,
@@ -290,6 +320,30 @@ export class PikkuWorkflowService {
                 : undefined,
         };
     }
+    /**
+     * Build the run's time-travel event stream from durable history.
+     * @param id - Run ID
+     * @returns Ordered timeline, or null if the run doesn't exist
+     */
+    async getRunTimeline(id) {
+        const run = await this.getRun(id);
+        if (!run)
+            return null;
+        return buildRunTimeline(await this.getRunHistory(id));
+    }
+    /**
+     * Reconstruct the run's state at a point in its timeline.
+     * @param id - Run ID
+     * @param at - A seq index (inclusive) or a Date (inclusive); omit for the
+     *             final state.
+     * @returns Reconstructed state, or null if the run doesn't exist
+     */
+    async reconstructRunStateAt(id, at) {
+        const timeline = await this.getRunTimeline(id);
+        if (!timeline)
+            return null;
+        return reconstructStateAt(timeline, at ?? timeline.length - 1);
+    }
     /**
      * Update workflow run status
      * @param id - Run ID
@@ -309,8 +363,8 @@ export class PikkuWorkflowService {
      * @param stepOptions - Step options (retries, retryDelay)
      * @returns Step state with generated stepId
      */
-    async insertStepState(runId, stepName, rpcName, data, stepOptions) {
-        const step = await this.insertStepStateImpl(runId, stepName, rpcName, data, stepOptions);
+    async insertStepState(runId, stepName, rpcName, data, stepOptions, fromStepName) {
+        const step = await this.insertStepStateImpl(runId, stepName, rpcName, data, stepOptions, fromStepName);
         await this.safeMirror(() => this.mirror.insertStepState(runId, { ...step, stepName, rpcName, data }));
         return step;
     }
@@ -363,6 +417,7 @@ export class PikkuWorkflowService {
             message: error.message,
             stack: error.stack,
             code: error.code,
+            expected: isExpectedError(error),
         };
         await this.safeMirror(() => this.mirror.setStepError(stepId, serialized));
     }
@@ -423,29 +478,38 @@ export class PikkuWorkflowService {
             const run = await this.getRun(runId);
             workflowName = run?.workflow;
         }
-        await queueService.add(this.getOrchestratorQueueName(workflowName), {
-            runId,
-        });
+        // Carry an explicit retry policy on the orchestrator job too. Orchestrator
+        // runs are idempotent (they replay from the snapshot, returning cached step
+        // results), so redelivery is always safe — and it's what lets a transient
+        // dispatch/infra failure recover: the job is rethrown and retried instead of
+        // the run hanging. Passing `attempts` per-job overrides the queue default, so
+        // this holds even when the orchestrator queue is configured `retry_limit 0`.
+        await queueService.add(this.getOrchestratorQueueName(workflowName), { runId }, this.resolveStepJobOptions());
     }
-    async queueStepWorker(runId, stepName, rpcName, data, stepOptions) {
-        const queueService = this.verifyQueueService();
-        const retries = stepOptions?.retries ?? 0;
+    /**
+     * Resolve a step's retry policy into queue job options. The workflow is the
+     * sole source of truth for retries: an explicitly-set `retries` (including 0)
+     * is always honored, an unset one defaults to {@link DEFAULT_STEP_RETRIES},
+     * and we ALWAYS pass `attempts` so the queue can never fall back to its own
+     * default — which would re-run a step the workflow said not to retry. Backoff
+     * defaults to exponential whenever there's at least one retry, so retries ride
+     * out a transient outage instead of firing instantly.
+     */
+    resolveStepJobOptions(stepOptions) {
+        const retries = stepOptions?.retries ?? DEFAULT_STEP_RETRIES;
         const retryDelay = stepOptions?.retryDelay;
-        await queueService.add(this.getStepWorkerQueueName(rpcName), JSON.parse(JSON.stringify({
-            runId,
-            stepName,
-            rpcName,
-            data,
-        })), retries > 0 || retryDelay
-            ? {
-                attempts: retries + 1,
-                backoff: typeof retryDelay === 'number'
-                    ? { type: 'fixed', delay: retryDelay }
-                    : retryDelay === 'exponential'
-                        ? 'exponential'
-                        : undefined,
-            }
-            : undefined);
+        const backoff = typeof retryDelay === 'number'
+            ? { type: 'fixed', delay: retryDelay }
+            : retryDelay === 'exponential'
+                ? 'exponential'
+                : retries > 0
+                    ? 'exponential'
+                    : undefined;
+        return { attempts: retries + 1, ...(backoff ? { backoff } : {}) };
+    }
+    async queueStepWorker(runId, stepName, rpcName, data, stepOptions, fromStepName) {
+        const queueService = this.verifyQueueService();
+        await queueService.add(this.getStepWorkerQueueName(rpcName), JSON.parse(JSON.stringify({ runId, stepName, rpcName, data, fromStepName })), this.resolveStepJobOptions(stepOptions));
     }
     /**
      * Execute a workflow sleep step completion
@@ -483,7 +547,7 @@ export class PikkuWorkflowService {
      * @returns true if dispatch was async (caller should pause), false to fall
      *   through to the inline execution path.
      */
-    async dispatchStep(runId, stepName, rpcName, data, stepOptions) {
+    async dispatchStep(runId, stepName, rpcName, data, stepOptions, fromStepName) {
         // Step execution is decided purely by the function's `inline` flag (default
         // true). Only a function explicitly marked `inline: false` dispatches via
         // the queue. The run-level inline state is intentionally NOT consulted
@@ -505,21 +569,16 @@ export class PikkuWorkflowService {
             getSingletonServices()?.logger.warn(`Workflow step '${stepName}' (function '${rpcName}') is marked 'inline: false' but no queue service is configured — running it inline instead of dispatching to a queue.`);
             return false;
         }
-        const retries = stepOptions?.retries ?? 0;
-        const retryDelay = stepOptions?.retryDelay;
-        await getSingletonServices().queueService.add(this.getStepWorkerQueueName(rpcName), JSON.parse(JSON.stringify({
-            runId,
-            stepName,
-            rpcName,
-            data,
-        })), {
-            attempts: retries + 1,
-            backoff: typeof retryDelay === 'number'
-                ? { type: 'fixed', delay: retryDelay }
-                : retryDelay === 'exponential'
-                    ? 'exponential'
-                    : undefined,
-        });
+        try {
+            await getSingletonServices().queueService.add(this.getStepWorkerQueueName(rpcName), JSON.parse(JSON.stringify({ runId, stepName, rpcName, data, fromStepName })), this.resolveStepJobOptions(stepOptions));
+        }
+        catch (cause) {
+            // The queue is down/unreachable — NOT a step failure. Surface it as a
+            // transient dispatch error so the caller leaves the step `pending` and the
+            // orchestrator job is retried (replayed from snapshot) rather than the run
+            // being marked failed. `add` already throws on failure in every adapter.
+            throw new WorkflowDispatchException(runId, stepName, { cause });
+        }
         return true;
     }
     /**
@@ -587,13 +646,25 @@ export class PikkuWorkflowService {
             catch (error) {
                 if (error.name !== 'WorkflowAsyncException' &&
                     error.name !== 'WorkflowCancelledException' &&
-                    error.name !== 'WorkflowSuspendedException') {
+                    error.name !== 'WorkflowSuspendedException' &&
+                    // Transient queue failure — leave the run resumable, don't fail it.
+                    error.name !== 'WorkflowDispatchException') {
                     await this.updateRunStatus(runId, 'failed', undefined, {
                         name: error.name,
                         message: error.message,
                         stack: error.stack,
                     });
-                    getSingletonServices().logger.error(`Workflow ${name} (run ${runId}) failed:`, error);
+                    // An expected failure (a PikkuError, e.g. a build gate tripping) —
+                    // its message is the whole story, so don't dump the stack. The
+                    // `expected` flag survives the step-boundary rehydration that strips
+                    // the class. Anything else is an uncaught/unexpected error: log it in
+                    // full so the trace is there to debug.
+                    getSingletonServices().logger.error(`Workflow ${name} (run ${runId}) failed:`, isExpectedError(error) ? error.message : error);
+                    throw error;
+                }
+                if (error.name === 'WorkflowDispatchException') {
+                    // Rethrow so the caller (poll loop / starter) sees the transient
+                    // failure; the run stays running and can be resumed.
                     throw error;
                 }
             }
@@ -626,7 +697,48 @@ export class PikkuWorkflowService {
             await new Promise((resolve) => setTimeout(resolve, pollInterval));
         }
     }
+    // Per-run, per-replay ordinal counters (runId → stepName → count).
+    stepOrdinals = new Map();
+    // Previous step key reached in the current DSL walk (runId → stepName), so a
+    // new step records where it came from. Rebuilt each replay alongside ordinals.
+    stepLineage = new Map();
+    resetStepOrdinals(runId) {
+        this.stepOrdinals.set(runId, new Map());
+        this.stepLineage.delete(runId);
+    }
+    /** The step the DSL walk last reached (the predecessor for the next step). */
+    lastStepName(runId) {
+        return this.stepLineage.get(runId);
+    }
+    /**
+     * Physical, replay-stable key for the Nth reach of `logicalStepName` in a run:
+     * bare name for the first reach (ordinal 0, unchanged behavior), `name#N` for
+     * repeats — so the same literal step name can be invoked multiple times without
+     * the rows clobbering. Deterministic given a deterministic DSL body.
+     */
+    nextStepKey(runId, logicalStepName) {
+        let perRun = this.stepOrdinals.get(runId);
+        if (!perRun) {
+            perRun = new Map();
+            this.stepOrdinals.set(runId, perRun);
+        }
+        const ordinal = perRun.get(logicalStepName) ?? 0;
+        perRun.set(logicalStepName, ordinal + 1);
+        const stepName = ordinal === 0 ? logicalStepName : `${logicalStepName}#${ordinal}`;
+        this.stepLineage.set(runId, stepName);
+        return stepName;
+    }
     async runWorkflowJob(runId, rpcService) {
+        // Fresh ordinal counters per replay so step keys are deterministic.
+        this.resetStepOrdinals(runId);
+        try {
+            await this.runWorkflowJobInner(runId, rpcService);
+        }
+        finally {
+            this.stepOrdinals.delete(runId);
+        }
+    }
+    async runWorkflowJobInner(runId, rpcService) {
         const run = await this.getRun(runId);
         if (!run) {
             throw new WorkflowRunNotFoundError(runId);
@@ -799,10 +911,22 @@ export class PikkuWorkflowService {
                 const workflowMeta = meta[run.workflow];
                 const isGraphWorkflow = workflowMeta?.source === 'graph' ||
                     workflowMeta?.source === 'dynamic-workflow';
-                if (isGraphWorkflow &&
-                    workflowMeta?.nodes &&
-                    stepName in workflowMeta.nodes) {
-                    result = await executeGraphStep(this, rpcService, runId, stepState.stepId, stepName, rpcName, data, run.workflow);
+                // Map the physical step key back to its logical node: a revisit instance
+                // is `node#N` (ordinal), which isn't a literal key in `nodes`.
+                let graphNodeId;
+                if (isGraphWorkflow && workflowMeta?.nodes) {
+                    if (stepName in workflowMeta.nodes) {
+                        graphNodeId = stepName;
+                    }
+                    else {
+                        const hash = stepName.lastIndexOf('#');
+                        const base = hash > 0 ? stepName.slice(0, hash) : undefined;
+                        if (base && base in workflowMeta.nodes)
+                            graphNodeId = base;
+                    }
+                }
+                if (graphNodeId) {
+                    result = await executeGraphStep(this, rpcService, runId, stepState.stepId, graphNodeId, rpcName, data, run.workflow);
                 }
                 else {
                     // Check if rpcName refers to a sub-workflow
@@ -833,13 +957,7 @@ export class PikkuWorkflowService {
                         }
                     }
                     else {
-                        result = await rpcService.rpcWithWire(rpcName, data, {
-                            workflowStep: {
-                                runId,
-                                stepId: stepState.stepId,
-                                attemptCount: stepState.attemptCount,
-                            },
-                        });
+                        result = await this.invokeStepRpc(runId, stepName, stepState, rpcName, data, rpcService);
                     }
                 }
                 // Store result and mark succeeded
@@ -862,7 +980,7 @@ export class PikkuWorkflowService {
                 }
                 // Store error and mark failed
                 await this.setStepError(stepState.stepId, error);
-                const maxAttempts = (stepState.retries ?? 0) + 1;
+                const maxAttempts = (stepState.retries ?? DEFAULT_STEP_RETRIES) + 1;
                 const retriesExhausted = stepState.attemptCount >= maxAttempts;
                 if (retriesExhausted) {
                     // No more retries - resume orchestrator to mark workflow as failed
@@ -888,6 +1006,13 @@ export class PikkuWorkflowService {
                 error.name === 'WorkflowSuspendedException') {
                 return;
             }
+            if (error.name === 'WorkflowDispatchException') {
+                // Transient: the queue was unreachable, not a workflow failure. Leave the
+                // run running and rethrow so the orchestrator job is redelivered and the
+                // workflow replays from its snapshot. Do NOT mark the run failed.
+                getSingletonServices().logger.warn(`Workflow run ${runId} could not dispatch a step (queue unavailable); leaving run for orchestrator retry`, error);
+                throw error;
+            }
             await this.updateRunStatus(runId, 'failed', undefined, {
                 message: error.message,
                 stack: error.stack,
@@ -902,7 +1027,75 @@ export class PikkuWorkflowService {
         }
         return getSingletonServices().queueService;
     }
-    async rpcStep(runId, stepName, rpcName, data, rpcService, stepOptions) {
+    /**
+     * Invoke a step's RPC with the workflow-step wire (step identity + provenance).
+     * Identical for the queue executor and the inline executor — the only thing
+     * that differs between transports is who calls it, not the call itself.
+     */
+    async invokeStepRpc(runId, stepName, stepState, rpcName, data, rpcService) {
+        return rpcService.rpcWithWire(rpcName, data, {
+            workflowStep: {
+                runId,
+                stepId: stepState.stepId,
+                invocationId: deriveInvocationId(runId, stepName),
+                attemptCount: stepState.attemptCount,
+                fromInvocationId: stepState.fromStepName
+                    ? deriveInvocationId(runId, stepState.fromStepName)
+                    : undefined,
+            },
+        });
+    }
+    /**
+     * Inline (straight-through) step execution with an in-process retry loop —
+     * shared by inline RPC steps and inline function steps. Same scaffolding
+     * (running → result, or fail → retry-attempt → backoff → retry) wrapped
+     * around a step-specific `doWork` body. Stays O(K): no suspend/replay.
+     *
+     * `onError` is an optional hook for terminal errors that must NOT retry
+     * (e.g. RPC-not-found → suspend the run for redeploy). If it throws, the
+     * loop exits immediately without recording a step error or retrying.
+     */
+    async runInlineRetryLoop(stepState, retries, retryDelay, doWork, onError) {
+        let currentStepState = stepState;
+        while (true) {
+            try {
+                await this.setStepRunning(currentStepState.stepId);
+                const result = await doWork(currentStepState);
+                await this.setStepResult(currentStepState.stepId, result);
+                return result;
+            }
+            catch (error) {
+                if (onError)
+                    await onError(error);
+                // Record the error (marks step as failed)
+                await this.setStepError(currentStepState.stepId, error);
+                if (currentStepState.attemptCount < retries) {
+                    // Create a new pending retry attempt, then back off if configured.
+                    currentStepState = await this.createRetryAttempt(currentStepState.stepId, 'pending');
+                    if (retryDelay) {
+                        await new Promise((resolve) => setTimeout(resolve, getDurationInMilliseconds(retryDelay)));
+                    }
+                    // Continue loop to retry
+                }
+                else {
+                    // No more retries, fail the workflow
+                    throw error;
+                }
+            }
+        }
+    }
+    async rpcStep(runId, logicalStepName, rpcName, data, rpcService, stepOptions) {
+        // Capture the predecessor before nextStepKey advances the lineage to us.
+        const fromStepName = this.lastStepName(runId);
+        const stepName = this.nextStepKey(runId, logicalStepName);
+        // Resolve the retry policy ONCE here so the value persisted on the step
+        // (which drives `retriesExhausted`) is the same one the queue dispatch turns
+        // into `attempts`. Without this the queue could retry N times while the
+        // engine thinks retries are already exhausted (or vice-versa).
+        const resolvedStepOptions = {
+            retries: stepOptions?.retries ?? DEFAULT_STEP_RETRIES,
+            retryDelay: stepOptions?.retryDelay,
+        };
         // Check if step already exists
         let stepState;
         try {
@@ -910,7 +1103,7 @@ export class PikkuWorkflowService {
         }
         catch {
             // Step doesn't exist - create it
-            stepState = await this.insertStepState(runId, stepName, rpcName, data, stepOptions);
+            stepState = await this.insertStepState(runId, stepName, rpcName, data, resolvedStepOptions, fromStepName);
         }
         if (stepState.status === 'succeeded') {
             // Return cached result
@@ -930,94 +1123,66 @@ export class PikkuWorkflowService {
             // Step is already scheduled, pause workflow
             throw new WorkflowAsyncException(runId, stepName);
         }
-        // Step is pending - schedule it
-        await this.setStepScheduled(stepState.stepId);
         // Hand off to subclass-overridable transport. Default behavior enqueues
         // via the queue service; DO-style subclasses RPC to a step worker.
-        const dispatched = await this.dispatchStep(runId, stepName, rpcName, data, stepOptions);
+        // Dispatch BEFORE marking the step `scheduled`: if the queue is down,
+        // dispatchStep throws WorkflowDispatchException and the step stays `pending`,
+        // so the orchestrator's next replay re-dispatches it. Marking `scheduled`
+        // first would strand the step (replay sees `scheduled`, pauses, never
+        // re-enqueues the job that was never created).
+        const dispatched = await this.dispatchStep(runId, stepName, rpcName, data, resolvedStepOptions, fromStepName);
         if (dispatched) {
+            await this.setStepScheduled(stepState.stepId);
             throw new WorkflowAsyncException(runId, stepName);
         }
-        {
-            // Inline (no transport available) - execute locally with retry loop
-            const retries = stepOptions?.retries ?? 0;
-            const retryDelay = stepOptions?.retryDelay;
-            let currentStepState = stepState;
-            while (true) {
-                try {
-                    await this.setStepRunning(currentStepState.stepId);
-                    // Check if the name refers to a workflow
-                    const workflowMeta = pikkuState(null, 'workflows', 'meta')[rpcName];
-                    let result;
-                    if (workflowMeta) {
-                        const childWire = {
-                            type: 'workflow',
-                            id: rpcName,
-                            parentRunId: runId,
-                            pikkuUserId: rpcService.wire?.pikkuUserId,
-                        };
-                        const { runId: childRunId } = await this.startWorkflow(rpcName, data, childWire, rpcService, { inline: true });
-                        await this.setStepChildRunId(currentStepState.stepId, childRunId);
-                        // Poll until child workflow completes
-                        while (true) {
-                            const childRun = await this.getRun(childRunId);
-                            if (!childRun) {
-                                throw new WorkflowRunNotFoundError(childRunId);
-                            }
-                            if (WORKFLOW_END_STATES.has(childRun.status)) {
-                                if (childRun.status === 'failed') {
-                                    throw new Error(childRun.error?.message || 'Sub-workflow failed');
-                                }
-                                if (childRun.status === 'cancelled') {
-                                    throw new Error('Sub-workflow was cancelled');
-                                }
-                                result = childRun.output;
-                                break;
-                            }
-                            await new Promise((resolve) => setTimeout(resolve, 500));
-                        }
-                    }
-                    else {
-                        result = await rpcService.rpcWithWire(rpcName, data, {
-                            workflowStep: {
-                                runId,
-                                stepId: currentStepState.stepId,
-                                attemptCount: currentStepState.attemptCount,
-                            },
-                        });
-                    }
-                    await this.setStepResult(currentStepState.stepId, result);
-                    return result;
-                }
-                catch (error) {
-                    if (error instanceof RPCNotFoundError) {
-                        await this.updateRunStatus(runId, 'suspended', undefined, {
-                            message: `RPC '${rpcName}' not found. Deploy the missing function and resume.`,
-                            code: 'RPC_NOT_FOUND',
-                        });
-                        throw error;
+        // Inline (no transport available) - execute locally with the shared retry
+        // loop. The body resolves to a sub-workflow result or a plain RPC result.
+        const retries = resolvedStepOptions.retries ?? this.getConfig().retries;
+        const retryDelay = resolvedStepOptions.retryDelay;
+        return this.runInlineRetryLoop(stepState, retries, retryDelay, async (currentStepState) => {
+            // Check if the name refers to a workflow
+            const workflowMeta = pikkuState(null, 'workflows', 'meta')[rpcName];
+            if (workflowMeta) {
+                const childWire = {
+                    type: 'workflow',
+                    id: rpcName,
+                    parentRunId: runId,
+                    pikkuUserId: rpcService.wire?.pikkuUserId,
+                };
+                const { runId: childRunId } = await this.startWorkflow(rpcName, data, childWire, rpcService, { inline: true });
+                await this.setStepChildRunId(currentStepState.stepId, childRunId);
+                // Poll until child workflow completes
+                while (true) {
+                    const childRun = await this.getRun(childRunId);
+                    if (!childRun) {
+                        throw new WorkflowRunNotFoundError(childRunId);
                     }
-                    // Record the error (marks step as failed)
-                    await this.setStepError(currentStepState.stepId, error);
-                    // Check if we should retry
-                    if (currentStepState.attemptCount < retries) {
-                        // Create a new pending retry attempt
-                        currentStepState = await this.createRetryAttempt(currentStepState.stepId, 'pending');
-                        // Wait for retry delay if specified
-                        if (retryDelay) {
-                            await new Promise((resolve) => setTimeout(resolve, getDurationInMilliseconds(retryDelay)));
+                    if (WORKFLOW_END_STATES.has(childRun.status)) {
+                        if (childRun.status === 'failed') {
+                            throw new Error(childRun.error?.message || 'Sub-workflow failed');
                         }
-                        // Continue loop to retry
-                    }
-                    else {
-                        // No more retries, fail the workflow
-                        throw error;
+                        if (childRun.status === 'cancelled') {
+                            throw new Error('Sub-workflow was cancelled');
+                        }
+                        return childRun.output;
                     }
+                    await new Promise((resolve) => setTimeout(resolve, 500));
                 }
             }
-        }
+            return this.invokeStepRpc(runId, stepName, currentStepState, rpcName, data, rpcService);
+        }, async (error) => {
+            if (error instanceof RPCNotFoundError) {
+                await this.updateRunStatus(runId, 'suspended', undefined, {
+                    message: `RPC '${rpcName}' not found. Deploy the missing function and resume.`,
+                    code: 'RPC_NOT_FOUND',
+                });
+                throw error;
+            }
+        });
     }
-    async inlineStep(runId, stepName, fn, stepOptions) {
+    async inlineStep(runId, logicalStepName, fn, stepOptions) {
+        const fromStepName = this.lastStepName(runId);
+        const stepName = this.nextStepKey(runId, logicalStepName);
         // Check if step already exists
         let stepState;
         try {
@@ -1025,7 +1190,7 @@ export class PikkuWorkflowService {
         }
         catch {
             // Step doesn't exist - create it (inline, no RPC)
-            stepState = await this.insertStepState(runId, stepName, null, null, stepOptions);
+            stepState = await this.insertStepState(runId, stepName, null, null, stepOptions, fromStepName);
         }
         if (stepState.status === 'succeeded') {
             // Return cached result
@@ -1034,39 +1199,14 @@ export class PikkuWorkflowService {
         // Execute inline function
         const retries = stepOptions?.retries ?? this.getConfig().retries;
         const retryDelay = stepOptions?.retryDelay ?? this.getConfig().retryDelay;
-        let currentStepState = stepState;
         // Check if we're running inline (in-memory) or remote (queue-based)
         if (this.isInline(runId)) {
-            // Inline mode - execute with retry loop
-            while (true) {
-                try {
-                    await this.setStepRunning(currentStepState.stepId);
-                    const result = await fn();
-                    await this.setStepResult(currentStepState.stepId, result);
-                    return result;
-                }
-                catch (error) {
-                    // Record the error (marks step as failed)
-                    await this.setStepError(currentStepState.stepId, error);
-                    // Check if we should retry
-                    if (currentStepState.attemptCount < retries) {
-                        // Create a new pending retry attempt
-                        currentStepState = await this.createRetryAttempt(currentStepState.stepId, 'pending');
-                        // Wait for retry delay if specified
-                        if (retryDelay) {
-                            await new Promise((resolve) => setTimeout(resolve, getDurationInMilliseconds(retryDelay)));
-                        }
-                        // Continue loop to retry
-                    }
-                    else {
-                        // No more retries, fail the workflow
-                        throw error;
-                    }
-                }
-            }
+            // Inline mode - execute with the shared in-process retry loop.
+            return this.runInlineRetryLoop(stepState, retries, retryDelay, () => fn());
         }
         else {
-            // Remote mode - use queue-based retry
+            // Remote mode - single attempt, then suspend for orchestrator-driven retry.
+            let currentStepState = stepState;
             try {
                 await this.setStepRunning(currentStepState.stepId);
                 const result = await fn();
@@ -1090,7 +1230,9 @@ export class PikkuWorkflowService {
             }
         }
     }
-    async sleepStep(runId, stepName, duration) {
+    async sleepStep(runId, logicalStepName, duration) {
+        const fromStepName = this.lastStepName(runId);
+        const stepName = this.nextStepKey(runId, logicalStepName);
         // Check if step already exists
         let stepState;
         try {
@@ -1098,9 +1240,7 @@ export class PikkuWorkflowService {
         }
         catch {
             // Step doesn't exist - create it (sleep step, no RPC)
-            stepState = await this.insertStepState(runId, stepName, null, {
-                duration,
-            });
+            stepState = await this.insertStepState(runId, stepName, null, { duration }, undefined, fromStepName);
         }
         if (stepState.status === 'succeeded') {
             // Sleep already completed, return immediately
@@ -1110,13 +1250,20 @@ export class PikkuWorkflowService {
             // Sleep is already scheduled, pause workflow
             throw new WorkflowAsyncException(runId, stepName);
         }
-        // Step is pending - schedule it
-        await this.setStepScheduled(stepState.stepId);
         // Hand off to subclass-overridable transport. Default behavior schedules
         // a delayed sleeper RPC via the scheduler service; DO-style subclasses
-        // override to use native timer primitives (e.g. setAlarm).
-        const scheduled = await this.scheduleSleep(runId, stepState.stepId, duration);
+        // override to use native timer primitives (e.g. setAlarm). Schedule BEFORE
+        // marking `scheduled` so a scheduler outage leaves the step `pending` for
+        // re-scheduling on replay instead of stranding it (see rpcStep).
+        let scheduled;
+        try {
+            scheduled = await this.scheduleSleep(runId, stepState.stepId, duration);
+        }
+        catch (cause) {
+            throw new WorkflowDispatchException(runId, stepName, { cause });
+        }
         if (scheduled) {
+            await this.setStepScheduled(stepState.stepId);
             throw new WorkflowAsyncException(runId, stepName);
         }
         // Inline mode - use setTimeout with actual duration
@@ -1138,7 +1285,8 @@ export class PikkuWorkflowService {
         return `__workflow_suspend:${reason}`;
     }
     async suspendStep(runId, reason) {
-        const suspendStepName = this.getSuspendStepName(reason);
+        const fromStepName = this.lastStepName(runId);
+        const suspendStepName = this.nextStepKey(runId, this.getSuspendStepName(reason));
         await this.withStepLock(runId, suspendStepName, async () => {
             let stepState;
             try {
@@ -1147,12 +1295,12 @@ export class PikkuWorkflowService {
             catch {
                 stepState = await this.insertStepState(runId, suspendStepName, 'pikkuWorkflowSuspend', {
                     reason,
-                });
+                }, undefined, fromStepName);
             }
             if (!stepState.stepId) {
                 stepState = await this.insertStepState(runId, suspendStepName, 'pikkuWorkflowSuspend', {
                     reason,
-                });
+                }, undefined, fromStepName);
             }
             if (stepState.status === 'succeeded') {
                 return;
@@ -1206,7 +1354,7 @@ export class PikkuWorkflowService {
         const singletonServices = getSingletonServices();
         const workflow = singletonServices.config?.workflow;
         return {
-            retries: workflow?.retries ?? 0,
+            retries: workflow?.retries ?? DEFAULT_STEP_RETRIES,
             retryDelay: workflow?.retryDelay ?? 0,
             orchestratorQueueName: workflow?.orchestratorQueueName ?? 'pikku-workflow-orchestrator',
             stepWorkerQueueName: workflow?.stepWorkerQueueName ?? 'pikku-workflow-step-worker',