@pikku/core 0.12.150 → 0.12.151

package/dist/wirings/workflow/pikku-workflow-service.js

@@ -1,5 +1,6 @@
 import { getSingletonServices, pikkuState } from '../../pikku-state.js';
 import { getDurationInMilliseconds } from '../../time-utils.js';
+import { buildWorkflowWire } from './workflow-wire.js';
 import { ChildWorkflowStartedException, continueGraph, executeGraphStep, runWorkflowGraph, stripInstanceOrdinal, } from './graph/graph-runner.js';
 import { isExpectedError } from '../../errors/error-handler.js';
 import { PikkuMissingMetaError } from '../../errors/errors.js';
@@ -7,8 +8,8 @@ import { RPCNotFoundError } from '../rpc/rpc-runner.js';
 import { deriveInvocationId } from './workflow-invocation-id.js';
 import { approvalDeciderFrom } from './workflow-approval-policy.js';
 import { buildRunTimeline, reconstructStateAt, } from './run-timeline.js';
-import { DEFAULT_STEP_RETRIES, RUN_LEASE_RETRY_MS, WORKFLOW_CHILD_POLL_MAX_MS, WORKFLOW_END_STATES, WORKFLOW_POLL_FACTOR, WORKFLOW_POLL_MIN_MS, WORKFLOW_TERMINAL_STATES, NO_COMPENSATE_STATE_PREFIX, forwardStepName, isCompensationStepName, milestoneStepName, isRunSettled, } from './workflow-constants.js';
-import { WorkflowAsyncException, WorkflowDispatchException, WorkflowNotFoundError, WorkflowRunCancelledError, WorkflowRunFailedError, WorkflowRunNotFoundError, WorkflowStepNameNotString, WorkflowStepSupersededError, } from './workflow-errors.js';
+import { DEFAULT_STEP_RETRIES, RUN_LEASE_RETRY_MS, WORKFLOW_CHILD_POLL_MAX_MS, WORKFLOW_END_STATES, WORKFLOW_POLL_FACTOR, WORKFLOW_POLL_MIN_MS, WORKFLOW_TERMINAL_STATES, NO_COMPENSATE_STATE_PREFIX, forwardStepName, isCompensationStepName, isRunSettled, isRunStopped, } from './workflow-constants.js';
+import { WorkflowAsyncException, WorkflowCancelledException, WorkflowDispatchException, WorkflowNotFoundError, WorkflowRunNotFoundError, WorkflowStepNameNotString, WorkflowStepSupersededError, } from './workflow-errors.js';
 import { resolveWorkflowMeta } from './workflow-meta-resolver.js';
 import { storeWorkflowVersion } from './workflow-version-store.js';
 import { jobGroupFor, orchestratorQueueName, requireQueueService, resolveWorkflowConfig, stepDispatchTarget, stepJobOptions, stepWorkerQueueName, } from './workflow-queue-routing.js';
@@ -16,16 +17,18 @@ import { wireWorkflowQueueWorkers } from './workflow-queue-wiring.js';
 import { approvalStepNameFor, evaluateApprovalStep, recordApprovalDecision, } from './workflow-approval.js';
 import { auditApprovalDecision } from './workflow-approval-audit.js';
 import { recordSuspension, suspendStepNameFor } from './workflow-suspend.js';
-import { claimStepByReadThenWrite } from './workflow-step-claim.js';
+import { claimStepByReadThenWrite, takeBackLostInlineStep, warnStepAttemptsExhausted, warnStepRerun, } from './workflow-step-claim.js';
 import { holdWorkflowLease, isRunLeaseError, nullWhenStepHeld, } from './workflow-run-lease.js';
-import { summarizeRunStatus } from './workflow-run-status.js';
+import { runOutput, summarizeRunStatus } from './workflow-run-status.js';
+import { writeRunMirror } from './workflow-run-mirror.js';
 import { runningStepLease, startStepLeaseRefresh, stepLeaseMsForQueue, } from './workflow-step-lease.js';
 import { runInlineRetryLoop } from './workflow-step-retry.js';
 import { runInlineChildStep, startQueuedChildStep, } from './workflow-child-step.js';
 import { runDslWorkflowPass } from './workflow-dsl-pass.js';
-import { childRunFailure, failRunOrUnwind, isUnwound, compensatingForStep, cancelAndUnwind, createUnwindHost, driveUnwind, recordHasCompensation, isChildWorkflowRecord, } from './workflow-compensation.js';
+import { runLogger, runTraceWire, withRunTrace } from './workflow-run-trace.js';
+import { childRunFailure, settleStoppedRun, failRunOrUnwind, isUnwound, compensatingForStep, cancelAndUnwind, createUnwindHost, driveUnwind, recordHasCompensation, isChildWorkflowRecord, } from './workflow-compensation.js';
 import { runVersionMismatchFallback } from './workflow-version-fallback.js';
-import { RedispatchBackoff, sweepStalledRuns, sweepUndispatchedSteps, } from './workflow-recovery.js';
+import { WorkflowRecovery, } from './workflow-recovery.js';
 /** The workflow engine; subclasses supply storage. Leases serialise runs and step claims over a shared store. */
 export class PikkuWorkflowService {
     runExtension;
@@ -68,6 +71,10 @@ export class PikkuWorkflowService {
     get logger() {
         return getSingletonServices()?.logger;
     }
+    lostStepLog = {
+        logger: () => this.logger,
+        traceparent: (runId) => this.getRunTraceparent(runId),
+    };
     mirror;
     /**
      * Serialises orchestration passes over a run and claims on a step. A service
@@ -91,19 +98,7 @@ export class PikkuWorkflowService {
     }
     async mirrored(write, mirror) {
         const written = await write();
-        if (this.mirror) {
-            try {
-                await mirror(this.mirror, written);
-            }
-            catch (err) {
-                try {
-                    this.logger?.warn?.(`[pikku] WorkflowRunMirror write failed: ${err?.message ?? err}`);
-                }
-                catch {
-                    // knowledge: decisions/internals/workflow-run-mirror-is-never-a-source-of-truth.md
-                }
-            }
-        }
+        await writeRunMirror(this.mirror, () => this.logger, (target) => mirror(target, written));
         return written;
     }
     wireQueueWorkers() {
@@ -144,6 +139,7 @@ export class PikkuWorkflowService {
         }
     }
     async createRun(workflowName, input, inline, graphHash, wire, options) {
+        wire = withRunTrace(wire);
         await storeWorkflowVersion(this, workflowName, graphHash);
         return this.mirrored(() => this.createRunImpl(workflowName, input, inline, graphHash, wire, options), (mirror, runId) => mirror.createRun(runId, workflowName, input, inline, graphHash, wire, options));
     }
@@ -252,77 +248,89 @@ export class PikkuWorkflowService {
         await this.enqueueOrchestration(runId, workflowName);
     }
     async enqueueOrchestration(runId, workflowName, delay) {
-        const queueService = requireQueueService();
-        workflowName ??= (await this.getRun(runId))?.workflow;
-        await queueService.add(this.getOrchestratorQueueName(workflowName), { runId }, {
+        await this.addOrchestratorJob(runId, workflowName, {
             ...this.resolveStepJobOptions(),
             ...(delay !== undefined ? { delay } : {}),
+        });
+    }
+    async addOrchestratorJob(runId, workflowName, options) {
+        const queueService = requireQueueService();
+        const run = await this.getRunIdentity(runId);
+        workflowName ??= run?.workflow;
+        await queueService.add(this.getOrchestratorQueueName(workflowName), { runId }, {
+            ...options,
             group: this.getJobGroup(workflowName),
+            traceparent: run?.wire?.traceparent,
         });
     }
     /**
-     * Ids of runs that are stalled: still `running`, with no step in a state that
-     * something is expected to complete (`scheduled`, `suspended`, or `running`
-     * under a live lease), and no step activity since `before`.
+     * Ids of runs that are stalled: still `running`, with no step something is
+     * expected to complete (`scheduled`, `suspended`, `running` under a live
+     * lease, or parked on a child run), and no step activity since `before`.
      *
-     * A `running` step whose lease has lapsed is the opposite of in flight — the
-     * dispatch that claimed it is gone — so it does not hold its run out of the
-     * sweep.
+     * A lapsed lease is a dispatch that died, so it holds nothing out of the
+     * sweep. Nor does a `running` step with no lease and no child run: that is
+     * work inside an orchestration pass, whose owner is the run's lease, which the
+     * sweep checks before it touches the run.
      *
-     * Returns nothing by default so a store that cannot express the query keeps
-     * working unchanged; a store that overrides it gains crash recovery through
-     * `recoverStalledRuns`.
+     * Returns nothing by default, for a store that cannot express the query.
      */
     async findStalledRunIds(_before, _limit) {
         return [];
     }
     /**
-     * Runs holding a step that has sat `pending` since before `before`, paired
-     * with the step that flagged them.
-     *
-     * Returns nothing by default so a store that cannot express the query keeps
-     * working unchanged — and, because it does not opt in, gains no re-dispatches
-     * either. A store must have an atomic `claimStepForExecution` before
-     * overriding this, or no concurrency for one to exclude: the relay makes
-     * duplicate dispatch routine, and the claim is what keeps a duplicate from
-     * becoming a second execution. Every `@pikku/kysely` dialect qualifies on its
-     * status-guarded claim, `mongodb` on the same claim expressed as a
-     * single-document update, `in-memory` on being inline and single-process —
-     * none of them overrides this yet.
+     * Runs holding a step `pending` since before `before`. A store overriding it
+     * needs an atomic `claimStepForExecution` (see `sweepUndispatchedSteps`).
      */
     async findUndispatchedSteps(_before, _limit) {
         return [];
     }
-    redispatchBackoff = new RedispatchBackoff();
-    get sweepDeps() {
-        return {
-            resume: (runId) => this.resumeWorkflow(runId),
-            logger: getSingletonServices()?.logger,
-        };
-    }
     /**
-     * Re-drive runs whose next move was lost. Not self-starting — call it from a
-     * scheduled task at whatever interval suits the workload.
+     * Steps parked on `scheduled` on a live run, oldest first, a page at a time.
+     * Returns nothing by default, for a store whose runs die with the process.
      */
-    async recoverStalledRuns(options) {
-        return sweepStalledRuns((before, limit) => this.findStalledRunIds(before, limit), this.redispatchBackoff, options, this.sweepDeps);
+    async findScheduledSteps(_limit, _offset) {
+        return [];
     }
-    /**
-     * Re-drive steps whose dispatch was lost. Not self-starting — call it from a
-     * scheduled task; ~30s suits a queue whose `pending`→`running` latency is
-     * well under that.
-     */
-    async relayUndispatchedSteps(options) {
-        return sweepUndispatchedSteps((before, limit) => this.findUndispatchedSteps(before, limit), this.redispatchBackoff, options, this.sweepDeps);
+    recovery = new WorkflowRecovery({
+        leases: () => this.leaseService,
+        listStepStates: (runId) => this.listStepStates(runId),
+        withRunLease: (runId, fn) => this.withRunLease(runId, fn),
+        findStalledRunIds: (before, limit) => this.findStalledRunIds(before, limit),
+        findUndispatchedSteps: this.findUndispatchedSteps.bind(this),
+        findScheduledSteps: this.findScheduledSteps.bind(this),
+        resumeWorkflow: (runId) => this.resumeWorkflow(runId),
+        wakeSleep: this.executeWorkflowSleepCompleted.bind(this),
+        rearmSleep: this.scheduleSleep.bind(this),
+        queueStepWorker: (...args) => this.queueStepWorker(...args),
+    });
+    /** One recovery pass, for a host driving it from a scheduled task. */
+    recoverWorkflows(options) {
+        return this.recovery.recover(options);
+    }
+    /** Sweep now and on an interval, once the queue is consuming. */
+    startRecovery(options) {
+        this.recovery.start(options);
+    }
+    stopRecovery() {
+        this.recovery.stop();
+    }
+    /** Re-drive runs whose next move was lost. */
+    recoverStalledRuns(options) {
+        return this.recovery.recoverStalledRuns(options);
+    }
+    /** Re-drive steps whose dispatch was lost. Not self-starting. */
+    relayUndispatchedSteps(options) {
+        return this.recovery.relayUndispatchedSteps(options);
     }
     resolveStepJobOptions(stepOptions) {
         return stepJobOptions(stepOptions);
     }
     async queueStepWorker(runId, stepName, rpcName, data, stepOptions, fromStepName) {
-        const queueService = requireQueueService();
-        await queueService.add(this.getStepWorkerQueueName(rpcName), { runId, stepName, rpcName, data, fromStepName }, {
+        await requireQueueService().add(this.getStepWorkerQueueName(rpcName), { runId, stepName, rpcName, data, fromStepName }, {
             ...this.resolveStepJobOptions(stepOptions),
             group: this.getJobGroup(rpcName),
+            traceparent: await this.getRunTraceparent(runId),
         });
     }
     async executeWorkflowSleepCompleted(runId, stepId) {
@@ -330,17 +338,7 @@ export class PikkuWorkflowService {
         await this.resumeWorkflow(runId);
     }
     async scheduleOrchestratorRetry(runId, retryDelay, workflowName) {
-        const queueService = requireQueueService();
-        if (!workflowName) {
-            const run = await this.getRun(runId);
-            workflowName = run?.workflow;
-        }
-        await queueService.add(this.getOrchestratorQueueName(workflowName), { runId }, {
-            ...(retryDelay
-                ? { delay: getDurationInMilliseconds(retryDelay) }
-                : undefined),
-            group: this.getJobGroup(workflowName),
-        });
+        await this.addOrchestratorJob(runId, workflowName, retryDelay ? { delay: getDurationInMilliseconds(retryDelay) } : {});
     }
     async dispatchStep(runId, stepName, rpcName, data, stepOptions, fromStepName) {
         const target = await stepDispatchTarget(rpcName, stepName, () => this.isInline(runId));
@@ -348,10 +346,7 @@ export class PikkuWorkflowService {
             return false;
         }
         try {
-            await getSingletonServices().queueService.add(this.getStepWorkerQueueName(rpcName), { runId, stepName, rpcName, data, fromStepName }, {
-                ...this.resolveStepJobOptions(stepOptions),
-                group: this.getJobGroup(rpcName),
-            });
+            await this.queueStepWorker(runId, stepName, rpcName, data, stepOptions, fromStepName);
         }
         catch (cause) {
             throw new WorkflowDispatchException(runId, stepName, { cause });
@@ -363,7 +358,7 @@ export class PikkuWorkflowService {
             !getSingletonServices()?.schedulerService) {
             return false;
         }
-        await getSingletonServices().schedulerService.scheduleRPC(duration, this.getConfig().sleeperRPCName, { runId, stepId });
+        await getSingletonServices().schedulerService.scheduleRPC(duration, this.getConfig().sleeperRPCName, { runId, stepId }, undefined, await this.getRunTraceparent(runId));
         return true;
     }
     setRunExtension(create) {
@@ -372,6 +367,7 @@ export class PikkuWorkflowService {
             updateRunStatus: this.updateRunStatus.bind(this),
             onChildWorkflowFailed: this.onChildWorkflowFailed.bind(this),
             verifyStepName: this.verifyStepName.bind(this),
+            getRunTraceparent: this.getRunTraceparent.bind(this),
         };
         const extension = create(engine);
         this.runExtension = extension;
@@ -381,6 +377,10 @@ export class PikkuWorkflowService {
         return this.runExtension;
     }
     async startWorkflow(name, input, wire, rpcService, options) {
+        wire = {
+            ...wire,
+            traceparent: wire?.traceparent ?? rpcService?.traceparent,
+        };
         const resolved = resolveWorkflowMeta(name);
         const workflowMeta = resolved?.meta;
         const packageName = resolved?.packageName ?? null;
@@ -401,7 +401,7 @@ export class PikkuWorkflowService {
                 throw new Error(`Workflow graph '${name}': '${options.startNode}' is not a declared entry node`);
             }
             const shouldInline = options?.inline || !getSingletonServices()?.queueService;
-            return runWorkflowGraph(this, name, input, rpcService, shouldInline, options?.startNode, wire, workflowMeta);
+            return runWorkflowGraph(this, name, input, rpcService, shouldInline, options?.startNode, wire, workflowMeta, options?.onRunCreated);
         }
         if (!workflowMeta.graphHash) {
             throw new Error(`Missing workflow graphHash for '${name}'`);
@@ -434,7 +434,7 @@ export class PikkuWorkflowService {
                         message: error.message,
                         stack: error.stack,
                     });
-                    getSingletonServices().logger.error(`Workflow ${name} (run ${runId}) failed:`, isExpectedError(error) ? error.message : error);
+                    runLogger(getSingletonServices().logger, wire).error(`Workflow ${name} (run ${runId}) failed:`, isExpectedError(error) ? error.message : error);
                     const settled = await this.getRun(runId);
                     if (settled?.wire?.type === 'workflow' && isUnwound(settled.status)) {
                         throw childRunFailure(settled) ?? error;
@@ -461,16 +461,7 @@ export class PikkuWorkflowService {
     }
     async runToCompletion(name, input, rpcService, options) {
         const { runId } = await this.startWorkflow(name, input, options?.wire ?? { type: 'internal' }, rpcService, { inline: true });
-        const run = await this.awaitRunEnd(runId, options?.pollIntervalMs ?? 1000);
-        if (run.status === 'failed' ||
-            run.status === 'compensated' ||
-            run.status === 'compensation_failed') {
-            throw new WorkflowRunFailedError(run.error?.message);
-        }
-        if (run.status === 'cancelled') {
-            throw new WorkflowRunCancelledError();
-        }
-        return run.output;
+        return runOutput(await this.awaitRunEnd(runId, options?.pollIntervalMs ?? 1000));
     }
     async awaitRunEnd(runId, maxIntervalMs) {
         let interval = Math.min(WORKFLOW_POLL_MIN_MS, maxIntervalMs);
@@ -547,6 +538,18 @@ export class PikkuWorkflowService {
         }
         return run;
     }
+    /** The trace every hop of the run stays on. */
+    async getRunTraceparent(runId) {
+        return (await this.getRunIdentity(runId))?.wire?.traceparent;
+    }
+    /** Warn that a step lost its worker and is running again, as `attempt`. */
+    logStepRerun(runId, stepName, attempt) {
+        return warnStepRerun(this.lostStepLog, runId, stepName, attempt);
+    }
+    /** Warn that a step lost its worker with no attempts left, and failed. */
+    async logStepAttemptsExhausted(runId, stepName, attempts) {
+        await warnStepAttemptsExhausted(this.lostStepLog, runId, stepName, attempts);
+    }
     lastStepName(runId) {
         return this.runContexts.get(runId)?.lastStep;
     }
@@ -616,6 +619,7 @@ export class PikkuWorkflowService {
             workflowWire: (addonNs) => this.createWorkflowWire(run.workflow, runId, rpcService, addonNs),
             extension: this.runExtension,
             updateRunStatus: (status, output, error) => this.updateRunStatus(runId, status, output, error),
+            settleIfStopped: () => this.settleStoppedRun(runId, rpcService),
             onChildCompleted: (result) => this.onChildWorkflowCompleted(run, result),
             onChildFailed: (error) => this.onChildWorkflowFailed(run, error),
             failOrUnwind: (error) => this.failOrUnwind(run, error, rpcService),
@@ -713,7 +717,7 @@ export class PikkuWorkflowService {
                         startWorkflow: (...args) => this.startWorkflow(...args),
                         setStepChildRunId: (...args) => this.setStepChildRunId(...args),
                         getRun: (id) => this.getRun(id),
-                    }, runId, stepState.stepId, rpcName, data, rpcService, run.wire?.pikkuUserId);
+                    }, runId, stepState.stepId, rpcName, data, rpcService, run.wire);
                 }
                 else {
                     result = await this.invokeStepRpc(runId, stepName, stepState, rpcName, data, rpcService, run, isCompensationStepName(stepName)
@@ -792,6 +796,7 @@ export class PikkuWorkflowService {
     async invokeStepRpc(runId, stepName, stepState, rpcName, data, rpcService, knownRun, compensatingFor) {
         const run = knownRun ?? (await this.getRunIdentity(runId));
         return rpcService.rpcWithWire(rpcName, data, {
+            ...runTraceWire(run?.wire?.traceparent),
             ...(run?.wire?.pikkuUserId ? { pikkuUserId: run.wire.pikkuUserId } : {}),
             ...(compensatingFor
                 ? {
@@ -855,7 +860,7 @@ export class PikkuWorkflowService {
         }
         const retries = resolvedStepOptions.retries ?? this.getConfig().retries;
         const retryDelay = resolvedStepOptions.retryDelay;
-        return runInlineRetryLoop(this, stepState, retries, retryDelay, async (currentStepState) => {
+        return runInlineRetryLoop(this, await takeBackLostInlineStep(this, runId, stepName, stepState), retries, retryDelay, async (currentStepState) => {
             if (resolvedStepOptions.virtualUser) {
                 return resolvedStepOptions.virtualUser.invoke(rpcName, data);
             }
@@ -864,7 +869,7 @@ export class PikkuWorkflowService {
                     startWorkflow: (...args) => this.startWorkflow(...args),
                     setStepChildRunId: (...args) => this.setStepChildRunId(...args),
                     awaitRunEnd: (id) => this.awaitRunEnd(id, WORKFLOW_CHILD_POLL_MAX_MS),
-                }, runId, currentStepState.stepId, rpcName, data, rpcService, (await this.getRunIdentity(runId))?.wire?.pikkuUserId);
+                }, runId, currentStepState.stepId, rpcName, data, rpcService, (await this.getRunIdentity(runId))?.wire);
             }
             return this.invokeStepRpc(runId, stepName, currentStepState, rpcName, data, rpcService);
         }, async (error) => {
@@ -875,7 +880,7 @@ export class PikkuWorkflowService {
                 });
                 throw error;
             }
-        });
+        }, () => this.throwIfRunStopped(runId));
     }
     async inlineStep(runId, logicalStepName, fn, stepOptions, data = null, rpcName = null) {
         const fromStepName = this.lastStepName(runId);
@@ -887,7 +892,7 @@ export class PikkuWorkflowService {
         const retries = stepOptions?.retries ?? this.getConfig().retries;
         const retryDelay = stepOptions?.retryDelay ?? this.getConfig().retryDelay;
         if (await this.isInline(runId)) {
-            return runInlineRetryLoop(this, stepState, retries, retryDelay, () => fn());
+            return runInlineRetryLoop(this, stepState, retries, retryDelay, () => fn(), undefined, () => this.throwIfRunStopped(runId));
         }
         else {
             let currentStepState = stepState;
@@ -899,7 +904,7 @@ export class PikkuWorkflowService {
             }
             catch (error) {
                 await this.setStepError(currentStepState.stepId, error);
-                if (currentStepState.attemptCount < retries) {
+                if (currentStepState.attemptCount <= retries) {
                     await this.createRetryAttempt(currentStepState.stepId, 'pending');
                     await this.scheduleOrchestratorRetry(runId, retryDelay);
                     throw new WorkflowAsyncException(runId, stepName);
@@ -971,41 +976,15 @@ export class PikkuWorkflowService {
         return await this.withStepLock(runId, approvalStepName, () => evaluateApprovalStep(this.approvalStore, runId, reason, approvalStepName, fromStepName, options));
     }
     createWorkflowWire(name, runId, rpcService, addonNamespace) {
-        const workflowWire = {
-            name,
-            runId,
-            getRun: async () => (await this.getRun(runId)),
-            do: async (stepName, rpcNameOrFn, dataOrOptions, options) => {
-                this.verifyStepName(stepName);
-                if (typeof rpcNameOrFn === 'string') {
-                    const resolvedRpcName = addonNamespace && !rpcNameOrFn.includes(':')
-                        ? `${addonNamespace}:${rpcNameOrFn}`
-                        : rpcNameOrFn;
-                    return await this.rpcStep(runId, stepName, resolvedRpcName, dataOrOptions, rpcService, options);
-                }
-                else {
-                    return await this.inlineStep(runId, stepName, rpcNameOrFn, dataOrOptions);
-                }
-            },
-            sleep: async (stepName, duration) => {
-                this.verifyStepName(stepName);
-                await this.sleepStep(runId, stepName, getDurationInMilliseconds(duration));
-            },
-            suspend: async (reason) => {
-                this.verifyStepName(reason);
-                await this.suspendStep(runId, reason);
-            },
-            milestone: async (name) => {
-                this.verifyStepName(name);
-                await this.inlineStep(runId, milestoneStepName(name), async () => ({
-                    name,
-                }));
-            },
-            approval: (async (reason, options) => {
-                this.verifyStepName(reason);
-                return await this.approvalStep(runId, reason, options);
-            }),
-        };
+        const workflowWire = buildWorkflowWire(name, runId, addonNamespace, {
+            getRun: () => this.getRun(runId),
+            verifyStepName: (stepName) => this.verifyStepName(stepName),
+            rpcStep: (stepName, rpcName, data, options) => this.rpcStep(runId, stepName, rpcName, data, rpcService, options),
+            inlineStep: (stepName, fn, options) => this.inlineStep(runId, stepName, fn, options),
+            sleepStep: (stepName, ms) => this.sleepStep(runId, stepName, ms),
+            suspendStep: (reason) => this.suspendStep(runId, reason),
+            approvalStep: (reason, options) => this.approvalStep(runId, reason, options),
+        });
         this.runExtension?.decorateWorkflowWire(workflowWire, {
             name,
             runId,
@@ -1038,13 +1017,23 @@ export class PikkuWorkflowService {
         const api = this.compensationApi(rpcService);
         return failRunOrUnwind(api, run, error, () => this.onChildWorkflowFailed(run, error));
     }
+    /** Ends a run's body at its next inline attempt once the run has been stopped. */
+    async throwIfRunStopped(runId) {
+        if (isRunStopped((await this.getRun(runId))?.status ?? '')) {
+            throw new WorkflowCancelledException(runId);
+        }
+    }
+    settleStoppedRun(runId, rpcService) {
+        return settleStoppedRun(this.compensationApi(rpcService), runId);
+    }
     /** Stops an unfinished run, children first, and unwinds what it completed. */
     async cancelRun(runId, rpcService, reason) {
         const api = this.compensationApi(rpcService);
         return cancelAndUnwind(api, createUnwindHost(api), runId, reason);
     }
     async markFailedUnlessUnwound(runId, error) {
-        if (isUnwound((await this.getRun(runId))?.status))
+        const status = (await this.getRun(runId))?.status;
+        if (status === 'cancelled' || isUnwound(status))
             return;
         await this.updateRunStatus(runId, 'failed', undefined, error);
     }

package/dist/wirings/workflow/workflow-child-step.d.ts

@@ -10,8 +10,8 @@ interface ChildStepApi {
 }
 export declare const runInlineChildStep: (api: ChildStepApi & {
     awaitRunEnd(childRunId: string): Promise<WorkflowRun>;
-}, runId: string, stepId: string, workflowName: string, data: unknown, rpcService: PikkuRPC, pikkuUserId: string | undefined) => Promise<unknown>;
+}, runId: string, stepId: string, workflowName: string, data: unknown, rpcService: PikkuRPC, parentWire: WorkflowRunWire | undefined) => Promise<unknown>;
 export declare const startQueuedChildStep: (api: ChildStepApi & {
     getRun(id: string): Promise<WorkflowRun | null>;
-}, runId: string, stepId: string, workflowName: string, data: unknown, rpcService: PikkuRPC, pikkuUserId: string | undefined) => Promise<unknown>;
+}, runId: string, stepId: string, workflowName: string, data: unknown, rpcService: PikkuRPC, parentWire: WorkflowRunWire | undefined) => Promise<unknown>;
 export {};

package/dist/wirings/workflow/workflow-child-step.js

@@ -1,8 +1,14 @@
 import { getSingletonServices } from '../../pikku-state.js';
 import { ChildWorkflowStartedException } from './graph/graph-runner.js';
 import { childRunFailure } from './workflow-compensation.js';
-export const runInlineChildStep = async (api, runId, stepId, workflowName, data, rpcService, pikkuUserId) => {
-    const { runId: childRunId } = await api.startWorkflow(workflowName, data, { type: 'workflow', id: workflowName, parentRunId: runId, pikkuUserId }, rpcService, { inline: true });
+export const runInlineChildStep = async (api, runId, stepId, workflowName, data, rpcService, parentWire) => {
+    const { runId: childRunId } = await api.startWorkflow(workflowName, data, {
+        type: 'workflow',
+        id: workflowName,
+        parentRunId: runId,
+        pikkuUserId: parentWire?.pikkuUserId,
+        traceparent: parentWire?.traceparent,
+    }, rpcService, { inline: true });
     await api.setStepChildRunId(stepId, childRunId);
     const childRun = await api.awaitRunEnd(childRunId);
     const failure = childRunFailure(childRun);
@@ -10,14 +16,15 @@ export const runInlineChildStep = async (api, runId, stepId, workflowName, data,
         throw failure;
     return childRun.output;
 };
-export const startQueuedChildStep = async (api, runId, stepId, workflowName, data, rpcService, pikkuUserId) => {
+export const startQueuedChildStep = async (api, runId, stepId, workflowName, data, rpcService, parentWire) => {
     const shouldInline = !getSingletonServices()?.queueService;
     const { runId: childRunId } = await api.startWorkflow(workflowName, data, {
         type: 'workflow',
         id: workflowName,
         parentRunId: runId,
         parentStepId: stepId,
-        pikkuUserId,
+        pikkuUserId: parentWire?.pikkuUserId,
+        traceparent: parentWire?.traceparent,
     }, rpcService, { inline: shouldInline });
     await api.setStepChildRunId(stepId, childRunId);
     if (!shouldInline) {

package/dist/wirings/workflow/workflow-compensation.d.ts

@@ -57,6 +57,12 @@ export type UnwindCause = SerializedError & {
 export declare const childRunFailure: (childRun: WorkflowRun) => Error | null;
 export declare const beginUnwind: (api: CompensationApi, host: UnwindHost, run: WorkflowRun, cause: UnwindCause) => Promise<'unwinding' | 'finished' | 'not-needed'>;
 export declare const isUnwound: (status: string | undefined) => boolean;
+/**
+ * The status of a run that can no longer take an outcome from a pass of its
+ * own. An unwind that stopped to wait on the steps in flight is finished here:
+ * those steps were this pass's, and nothing else will wake an inline run.
+ */
+export declare const settleStoppedRun: (api: CompensationApi, runId: string) => Promise<WorkflowStatus | undefined>;
 export declare const failRunOrUnwind: (api: CompensationApi, run: WorkflowRun, error: Error & {
     code?: string;
     childCompensation?: ChildCompensationInfo;

package/dist/wirings/workflow/workflow-compensation.js

@@ -1,4 +1,4 @@
-import { DEFAULT_STEP_RETRIES, NO_COMPENSATE_STATE_PREFIX, WORKFLOW_TERMINAL_STATES, compensationStepName, } from './workflow-constants.js';
+import { DEFAULT_STEP_RETRIES, NO_COMPENSATE_STATE_PREFIX, WORKFLOW_TERMINAL_STATES, isRunStopped, compensationStepName, } from './workflow-constants.js';
 import { pikkuState } from '../../pikku-state.js';
 import { resolveNamespace } from '../rpc/rpc-runner.js';
 import { runningStepLease } from './workflow-step-lease.js';
@@ -239,6 +239,20 @@ export const beginUnwind = async (api, host, run, cause) => {
 export const isUnwound = (status) => status === 'compensating' ||
     status === 'compensated' ||
     status === 'compensation_failed';
+/**
+ * The status of a run that can no longer take an outcome from a pass of its
+ * own. An unwind that stopped to wait on the steps in flight is finished here:
+ * those steps were this pass's, and nothing else will wake an inline run.
+ */
+export const settleStoppedRun = async (api, runId) => {
+    const status = (await api.getRun(runId))?.status;
+    if (!status || !isRunStopped(status))
+        return undefined;
+    if (status === 'compensating') {
+        await driveUnwind(createUnwindHost(api), runId);
+    }
+    return status;
+};
 export const failRunOrUnwind = async (api, run, error, onFailed) => {
     const cause = {
         message: error.message,

package/dist/wirings/workflow/workflow-constants.d.ts

@@ -30,6 +30,11 @@ export declare const isMilestoneStepName: (stepName: string) => boolean;
  * not the run, which resumes when its approval or signal arrives.
  */
 export declare const isRunSettled: (status: string) => boolean;
+/**
+ * True for a run its own pass must no longer write an outcome for. That is a
+ * settled run, or one unwinding, which `cancelRun` moves there on its own.
+ */
+export declare const isRunStopped: (status: string) => boolean;
 export declare const WORKFLOW_POLL_MIN_MS = 10;
 export declare const WORKFLOW_POLL_FACTOR = 1.6;
 export declare const WORKFLOW_CHILD_POLL_MAX_MS = 500;
@@ -45,6 +50,14 @@ export declare const DEFAULT_STALLED_RUN_LIMIT = 100;
 export declare const DEFAULT_UNDISPATCHED_STEP_MS = 30000;
 /** Steps re-driven per `relayUndispatchedSteps` call, so one tick is bounded. */
 export declare const DEFAULT_UNDISPATCHED_STEP_LIMIT = 100;
+/** How far past its wake time a sleep sits before a sweep decides its timer is lost. */
+export declare const SLEEP_WAKE_GRACE_MS = 5000;
+/** `scheduled` steps read per query, so a sweep pages rather than loads them all. */
+export declare const SCHEDULED_STEP_PAGE_SIZE = 200;
+/** How often a long-running host sweeps for lost work, after its startup pass. */
+export declare const DEFAULT_RECOVERY_INTERVAL_MS = 30000;
+/** Idle window `startRecovery` treats as stalled; the run's lease is what makes it safe. */
+export declare const DEFAULT_RECOVERY_STALLED_MS = 30000;
 /** First wait before a step already re-dispatched once is re-dispatched again. */
 export declare const REDISPATCH_BACKOFF_MS = 30000;
 /** Ceiling on the doubling backoff, so a permanently stuck step still gets swept. */

package/dist/wirings/workflow/workflow-constants.js

@@ -46,6 +46,11 @@ export const isMilestoneStepName = (stepName) => stepName.startsWith(MILESTONE_S
  * not the run, which resumes when its approval or signal arrives.
  */
 export const isRunSettled = (status) => WORKFLOW_TERMINAL_STATES.has(status);
+/**
+ * True for a run its own pass must no longer write an outcome for. That is a
+ * settled run, or one unwinding, which `cancelRun` moves there on its own.
+ */
+export const isRunStopped = (status) => isRunSettled(status) || status === 'compensating';
 export const WORKFLOW_POLL_MIN_MS = 10;
 export const WORKFLOW_POLL_FACTOR = 1.6;
 export const WORKFLOW_CHILD_POLL_MAX_MS = 500;
@@ -61,6 +66,14 @@ export const DEFAULT_STALLED_RUN_LIMIT = 100;
 export const DEFAULT_UNDISPATCHED_STEP_MS = 30_000;
 /** Steps re-driven per `relayUndispatchedSteps` call, so one tick is bounded. */
 export const DEFAULT_UNDISPATCHED_STEP_LIMIT = 100;
+/** How far past its wake time a sleep sits before a sweep decides its timer is lost. */
+export const SLEEP_WAKE_GRACE_MS = 5_000;
+/** `scheduled` steps read per query, so a sweep pages rather than loads them all. */
+export const SCHEDULED_STEP_PAGE_SIZE = 200;
+/** How often a long-running host sweeps for lost work, after its startup pass. */
+export const DEFAULT_RECOVERY_INTERVAL_MS = 30_000;
+/** Idle window `startRecovery` treats as stalled; the run's lease is what makes it safe. */
+export const DEFAULT_RECOVERY_STALLED_MS = 30_000;
 /** First wait before a step already re-dispatched once is re-dispatched again. */
 export const REDISPATCH_BACKOFF_MS = 30_000;
 /** Ceiling on the doubling backoff, so a permanently stuck step still gets swept. */

package/dist/wirings/workflow/workflow-dsl-pass.d.ts

@@ -5,6 +5,8 @@ export interface DslPassDeps {
     workflowWire: (addonNamespace: string | null) => PikkuWorkflowWire;
     extension: WorkflowRunExtension | undefined;
     updateRunStatus: (status: WorkflowStatus, output?: unknown, error?: SerializedError) => Promise<void>;
+    /** The run's status if something has ended it, finishing any unwind that waited on this pass. */
+    settleIfStopped: () => Promise<WorkflowStatus | undefined>;
     onChildCompleted: (result: unknown) => Promise<void>;
     onChildFailed: (error: Error) => Promise<void>;
     failOrUnwind: (error: Error) => Promise<'unwinding' | 'finished' | 'failed'>;