@pikku/core 0.12.148 → 0.12.151

package/dist/wirings/scheduler/remote-scheduled-job.js

@@ -6,9 +6,9 @@ import { ScheduledTaskNotFoundError, runScheduledTask, } from './scheduler-runne
  *
  * @throws {BadRequestError} When the named task is not wired in this unit.
  */
-export async function pikkuRemoteScheduledJobFunc({ logger }, { taskName }) {
+export async function pikkuRemoteScheduledJobFunc({ logger }, { taskName, traceparent }) {
     try {
-        await runScheduledTask({ name: taskName, traceId: `cron-${taskName}` });
+        await runScheduledTask({ name: taskName, traceparent });
     }
     catch (error) {
         if (error instanceof ScheduledTaskNotFoundError) {

package/dist/wirings/scheduler/scheduler-runner.d.ts

@@ -2,11 +2,11 @@ import type { CoreUserSession } from '../../types/core.types.js';
 import type { CoreScheduledTask } from './scheduler.types.js';
 import { PikkuError } from '../../errors/error-handler.js';
 import type { CorePikkuFunctionConfig, CorePikkuFunctionSessionless } from '../../function/functions.types.js';
-/** Parameters for {@link runScheduledTask}: the task's name, an optional session to run it with, and an optional trace ID. */
+/** Parameters for {@link runScheduledTask}: the task's name, an optional session to run it with, and an optional W3C trace to continue. */
 export type RunScheduledTasksParams = {
     name: string;
     session?: CoreUserSession;
-    traceId?: string;
+    traceparent?: string;
 };
 /**
  * Registers a sessionless function to run on a cron schedule.
@@ -35,10 +35,10 @@ export declare class ScheduledTaskNotFoundError extends PikkuError {
  * When `session` is given it is placed on the wire, so the function and its
  * middleware see that user; otherwise the task runs without one.
  *
- * @param params - See {@link RunScheduledTasksParams}. `traceId` defaults to `cron-<name>-<timestamp>`.
+ * @param params - See {@link RunScheduledTasksParams}. A missing or invalid `traceparent` starts a new trace.
  * @throws {ScheduledTaskNotFoundError} When the task or its metadata is missing.
  * @throws The error the function threw, including the one `scheduledTask.skip()` raises, so the caller can tell a skip from a failure.
  */
-export declare function runScheduledTask({ name, session, traceId, }: RunScheduledTasksParams): Promise<void>;
+export declare function runScheduledTask({ name, session, traceparent, }: RunScheduledTasksParams): Promise<void>;
 /** Every wired scheduled task, keyed by name. Scheduler services read this to know what to schedule. */
 export declare const getScheduledTasks: () => Map<string, CoreScheduledTask>;

package/dist/wirings/scheduler/scheduler-runner.js

@@ -1,6 +1,7 @@
 import { getErrorResponse, PikkuError } from '../../errors/error-handler.js';
 import { getSingletonServices, getCreateWireServices, pikkuState, } from '../../pikku-state.js';
 import { addFunction, runPikkuFunc } from '../../function/function-runner.js';
+import { continueTrace } from '../../trace.js';
 import { PikkuSessionService, createMiddlewareSessionWireProps, } from '../../services/user-session-service.js';
 /**
  * Registers a sessionless function to run on a cron schedule.
@@ -56,16 +57,15 @@ class ScheduledTaskSkippedError extends PikkuError {
  * When `session` is given it is placed on the wire, so the function and its
  * middleware see that user; otherwise the task runs without one.
  *
- * @param params - See {@link RunScheduledTasksParams}. `traceId` defaults to `cron-<name>-<timestamp>`.
+ * @param params - See {@link RunScheduledTasksParams}. A missing or invalid `traceparent` starts a new trace.
  * @throws {ScheduledTaskNotFoundError} When the task or its metadata is missing.
  * @throws The error the function threw, including the one `scheduledTask.skip()` raises, so the caller can tell a skip from a failure.
  */
-export async function runScheduledTask({ name, session, traceId, }) {
+export async function runScheduledTask({ name, session, traceparent, }) {
     const singletonServices = getSingletonServices();
     const createWireServices = getCreateWireServices();
-    const resolvedTraceId = traceId ?? `cron-${name}-${Date.now()}`;
-    const scopedLogger = singletonServices.logger.scope?.(resolvedTraceId) ??
-        singletonServices.logger;
+    const trace = continueTrace(traceparent);
+    const scopedLogger = singletonServices.logger.scope?.(trace.traceId) ?? singletonServices.logger;
     const task = pikkuState(null, 'scheduler', 'tasks').get(name);
     const meta = pikkuState(null, 'scheduler', 'meta')[name];
     const userSession = new PikkuSessionService(singletonServices.sessionStore);
@@ -79,7 +79,8 @@ export async function runScheduledTask({ name, session, traceId, }) {
         throw new ScheduledTaskNotFoundError(`Scheduled task meta not found: ${name}`);
     }
     const wire = {
-        traceId: resolvedTraceId,
+        traceId: trace.traceId,
+        trace,
         scheduledTask: {
             name,
             schedule: task.schedule,

package/dist/wirings/trigger/pikku-trigger-service.js

@@ -1,6 +1,7 @@
 import { rpcService } from '../rpc/rpc-runner.js';
 import { setupTrigger } from './trigger-runner.js';
 import { getSingletonServices, getCreateWireServices, pikkuState, } from '../../pikku-state.js';
+import { traceWire, tracedLogger } from '../../trace.js';
 /**
  * Base class for a {@link TriggerService}: pairs registered trigger sources with
  * the triggers listening to them and owns teardown. Subclasses decide where and
@@ -54,9 +55,11 @@ export class PikkuTriggerService {
         // The user rides on the wire, where the function runner resolves the
         // target's session and credentials from it and a started workflow records
         // it on the run.
-        const wire = options?.pikkuUserId
-            ? { pikkuUserId: options.pikkuUserId }
-            : {};
+        // One trace per event, shared by every target it fires.
+        const wire = {
+            ...traceWire(),
+            ...(options?.pikkuUserId ? { pikkuUserId: options.pikkuUserId } : {}),
+        };
         const wireServices = await createWireServices?.(singletonServices, wire);
         const services = { ...singletonServices, ...wireServices };
         const rpc = rpcService.getContextRPCService(services, wire, false);
@@ -74,7 +77,7 @@ export class PikkuTriggerService {
                 }
             }
             catch (error) {
-                singletonServices.logger.error(`Error invoking ${target.targetType} '${target.targetName}' from trigger '${triggerName}': ${error}`);
+                tracedLogger(singletonServices.logger, wire.traceId).error(`Error invoking ${target.targetType} '${target.targetName}' from trigger '${triggerName}': ${error}`);
             }
         }
     }

package/dist/wirings/trigger/webhook-source-runner.d.ts

@@ -3,6 +3,7 @@ import type { CoreSingletonServices } from '../../types/core.types.js';
 import type { PikkuHTTP } from '../http/http.types.js';
 import type { DeclaredTriggerSource } from '../../services/trigger-source-store.js';
 import type { CoreTriggerWebhookSource, WebhookCheckResult, WebhookSetupResult, WebhookSourceJob, WebhookSourceState, WebhookTeardownResult } from './webhook-source.types.js';
+import { type TraceContext } from '../../trace.js';
 /**
  * Registers a webhook trigger source: Pikku mounts its route (by default
  * `POST /webhooks/<name>`), runs `receive` to turn each request into events,
@@ -33,11 +34,16 @@ export declare const subscribedWebhookEvents: (source: string) => string[];
  */
 export declare const receiveWebhookSourceRequest: (sourceName: string, wire: {
     http?: PikkuHTTP;
+    trace?: TraceContext;
 }) => Promise<{
     received: number;
 } | void>;
 /** The `pikku-incoming-webhooks` worker: runs the trigger an event was queued for. Throws so the queue retries. */
-export declare const dispatchWebhookSourceJob: (job: WebhookSourceJob) => Promise<void>;
+export declare const dispatchWebhookSourceJob: (job: WebhookSourceJob, 
+/** The worker's wire, whose trace the trigger continues. */
+wire?: {
+    trace?: TraceContext;
+}) => Promise<void>;
 /** What one lifecycle pass (`check`, `setup` or `teardown`) did for one webhook source. */
 export type WebhookSourceOutcome = {
     source: string;

package/dist/wirings/trigger/webhook-source-runner.js

@@ -4,6 +4,7 @@ import { parseJson } from '../../utils.js';
 import { PikkuMissingMetaError, UnauthorizedError, } from '../../errors/errors.js';
 import { timingSafeStringEqual, verifyHmacSignature, verifyPublicKeySignature, } from '../../utils/hmac.js';
 import { webhookSecretCredentialName } from './webhook-source.types.js';
+import { childTraceWire, outgoingTraceparent, } from '../../trace.js';
 const LIFECYCLE = ['receive', 'check', 'setup', 'teardown'];
 /**
  * Registers a webhook trigger source: Pikku mounts its route (by default
@@ -195,11 +196,18 @@ export const receiveWebhookSourceRequest = async (sourceName, wire) => {
         throw new Error(`Webhook source '${sourceName}' received events but no incomingWebhookService is configured to queue them.`);
     }
     return {
-        received: await service.accept({ source: sourceName, request, events }),
+        received: await service.accept({
+            source: sourceName,
+            request,
+            events,
+            traceparent: outgoingTraceparent(wire),
+        }),
     };
 };
 /** The `pikku-incoming-webhooks` worker: runs the trigger an event was queued for. Throws so the queue retries. */
-export const dispatchWebhookSourceJob = async (job) => {
+export const dispatchWebhookSourceJob = async (job, 
+/** The worker's wire, whose trace the trigger continues. */
+wire = {}) => {
     const singletonServices = getSingletonServices();
     const name = triggerName(job.source, job.event.name);
     const trigger = pikkuState(null, 'trigger', 'meta')[name];
@@ -213,7 +221,7 @@ export const dispatchWebhookSourceJob = async (job) => {
             singletonServices,
             auth: false,
             data: () => job.event.data,
-            wire: {},
+            wire: childTraceWire(wire.trace),
         });
     }
     catch (e) {

package/dist/wirings/workflow/graph/graph-runner.d.ts

@@ -11,6 +11,6 @@ export declare function stripInstanceOrdinal(name: string): string;
 export declare function continueGraph(workflowService: PikkuWorkflowService, runId: string, graphName: string, overrideMeta?: WorkflowRuntimeMeta, rpcService?: any): Promise<void>;
 export declare function executeGraphStep(workflowService: PikkuWorkflowService, rpcService: any, runId: string, stepId: string, nodeId: string, rpcName: string, data: any, graphName: string): Promise<any>;
 export declare function runFromMeta(workflowService: PikkuWorkflowService, runId: string, meta: WorkflowRuntimeMeta, _rpcService: any): Promise<void>;
-export declare function runWorkflowGraph(workflowService: PikkuWorkflowService, graphName: string, triggerInput: any, rpcService?: any, inline?: boolean, startNode?: string, wire?: WorkflowRunWire, overrideMeta?: WorkflowRuntimeMeta): Promise<{
+export declare function runWorkflowGraph(workflowService: PikkuWorkflowService, graphName: string, triggerInput: any, rpcService?: any, inline?: boolean, startNode?: string, wire?: WorkflowRunWire, overrideMeta?: WorkflowRuntimeMeta, onRunCreated?: (runId: string) => void): Promise<{
     runId: string;
 }>;

package/dist/wirings/workflow/graph/graph-runner.js

@@ -1,9 +1,11 @@
-import { WorkflowAsyncException, WorkflowSuspendedException, } from '../workflow-errors.js';
-import { DEFAULT_STEP_RETRIES, NO_COMPENSATE_STATE_PREFIX, } from '../workflow-constants.js';
+import { WorkflowAsyncException, WorkflowCancelledException, WorkflowSuspendedException, } from '../workflow-errors.js';
+import { runInlineRetryLoop } from '../workflow-step-retry.js';
+import { DEFAULT_STEP_RETRIES, NO_COMPENSATE_STATE_PREFIX, isRunStopped, } from '../workflow-constants.js';
 import { childRunFailure } from '../workflow-compensation.js';
 import { runningStepLease } from '../workflow-step-lease.js';
 import { pikkuState, getSingletonServices } from '../../../pikku-state.js';
 import { RPCNotFoundError } from '../../rpc/rpc-runner.js';
+import { runTraceWire } from '../workflow-run-trace.js';
 export class ChildWorkflowStartedException extends Error {
     parentRunId;
     stepId;
@@ -610,6 +612,10 @@ async function failGraphRun(workflowService, runId, failedNode, rpcService) {
     await failGraphRunWith(workflowService, runId, new Error(`Graph node '${failedNode}' failed after exhausting retries`), rpcService);
 }
 async function failGraphRunWith(workflowService, runId, cause, rpcService) {
+    if (rpcService &&
+        (await workflowService.settleStoppedRun(runId, rpcService))) {
+        return;
+    }
     const error = Object.assign(cause, { code: 'GRAPH_NODE_FAILED' });
     const run = await workflowService.getRun(runId);
     if (run && rpcService) {
@@ -664,7 +670,8 @@ export async function continueGraph(workflowService, runId, graphName, overrideM
         return;
     }
     const currentRun = await workflowService.getRun(runId);
-    if (currentRun?.status === 'suspended') {
+    if (currentRun?.status === 'suspended' ||
+        (currentRun && isRunStopped(currentRun.status))) {
         return;
     }
     const instances = await workflowService.getStepInstances(runId);
@@ -726,6 +733,7 @@ async function invokeGraphNodeRpc(workflowService, rpcService, runId, stepId, no
         recoveringFrom: (await workflowService.getRunState(runId))[`${RECOVERING_FROM_STATE_PREFIX}${nodeId}`],
     };
     const result = await rpcService.rpcWithWire(rpcName, input, {
+        ...runTraceWire(await workflowService.getRunTraceparent(runId)),
         graph: graphWire,
         workflow: workflowService.createWorkflowWire(graphName, runId, rpcService),
     });
@@ -747,6 +755,9 @@ export async function executeGraphStep(workflowService, rpcService, runId, stepI
                 id: rpcName,
                 parentRunId: runId,
                 parentStepId: stepId,
+                pikkuUserId: (await workflowService.getRunIdentity(runId))?.wire
+                    ?.pikkuUserId,
+                traceparent: await workflowService.getRunTraceparent(runId),
             };
             const shouldInline = !getSingletonServices()?.queueService;
             const { runId: childRunId } = await workflowService.startWorkflow(rpcName, data, childWire, rpcService, { inline: shouldInline });
@@ -798,9 +809,9 @@ async function executeGraphNodeInline(workflowService, rpcService, runId, graphN
         return;
     const rpcName = node.rpcName;
     const stepState = await workflowService.insertStepState(runId, instanceKey, rpcName, input, { retries: node.retries ?? 0, retryDelay: node.retryDelay }, fromStepName);
-    await workflowService.setStepRunning(stepState.stepId);
-    try {
-        let result;
+    let attempt = stepState;
+    const runNode = async (current) => {
+        attempt = current;
         const subWorkflowMeta = getWorkflowMeta(rpcName);
         const agentMeta = subWorkflowMeta
             ? undefined
@@ -810,42 +821,48 @@ async function executeGraphNodeInline(workflowService, rpcService, runId, graphN
                 type: 'workflow',
                 id: rpcName,
                 parentRunId: runId,
-                parentStepId: stepState.stepId,
+                parentStepId: attempt.stepId,
+                pikkuUserId: (await workflowService.getRunIdentity(runId))?.wire
+                    ?.pikkuUserId,
+                traceparent: await workflowService.getRunTraceparent(runId),
             };
             const { runId: childRunId } = await workflowService.startWorkflow(rpcName, input, childWire, rpcService, { inline: true });
-            await workflowService.setStepChildRunId(stepState.stepId, childRunId);
+            await workflowService.setStepChildRunId(attempt.stepId, childRunId);
             const childRun = await workflowService.getRun(childRunId);
             const failure = childRun && childRunFailure(childRun);
             if (failure)
                 throw failure;
-            result = childRun?.output;
+            return childRun?.output;
         }
-        else if (agentMeta) {
+        if (agentMeta) {
             const agentRun = await rpcService.agent.run(rpcName, input);
-            result = agentRun.result;
+            return agentRun.result;
         }
-        else {
-            result = await invokeGraphNodeRpc(workflowService, rpcService, runId, stepState.stepId, nodeId, rpcName, input, graphName);
-        }
-        await workflowService.setStepResult(stepState.stepId, result);
+        return invokeGraphNodeRpc(workflowService, rpcService, runId, attempt.stepId, nodeId, rpcName, input, graphName);
+    };
+    try {
+        await runInlineRetryLoop(workflowService, stepState, node.retries ?? 0, node.retryDelay, runNode, async (error) => {
+            if (error instanceof WorkflowAsyncException ||
+                error instanceof WorkflowSuspendedException) {
+                throw error;
+            }
+            if (error instanceof RPCNotFoundError) {
+                await workflowService.setStepError(attempt.stepId, error);
+                await workflowService.updateRunStatus(runId, 'suspended', undefined, {
+                    message: `RPC '${rpcName}' not found. Deploy the missing function and resume.`,
+                    code: 'RPC_NOT_FOUND',
+                });
+                throw new WorkflowSuspendedException(runId, 'RPC_NOT_FOUND');
+            }
+        }, () => workflowService.throwIfRunStopped(runId));
     }
     catch (error) {
         if (error instanceof WorkflowAsyncException ||
-            error instanceof WorkflowSuspendedException) {
+            error instanceof WorkflowSuspendedException ||
+            error instanceof WorkflowCancelledException ||
+            !recoversFailure(node)) {
             throw error;
         }
-        if (error instanceof RPCNotFoundError) {
-            await workflowService.setStepError(stepState.stepId, error);
-            await workflowService.updateRunStatus(runId, 'suspended', undefined, {
-                message: `RPC '${rpcName}' not found. Deploy the missing function and resume.`,
-                code: 'RPC_NOT_FOUND',
-            });
-            throw new WorkflowSuspendedException(runId, 'RPC_NOT_FOUND');
-        }
-        await workflowService.setStepError(stepState.stepId, error);
-        if (recoversFailure(node))
-            return;
-        throw error;
     }
 }
 async function continueGraphInline(workflowService, rpcService, runId, graphName, nodes, triggerInput, entryNodeIds) {
@@ -859,7 +876,8 @@ async function continueGraphInline(workflowService, rpcService, runId, graphName
             return;
         }
         const run = await workflowService.getRun(runId);
-        if (run?.status === 'suspended') {
+        if (run?.status === 'suspended' ||
+            (await workflowService.settleStoppedRun(runId, rpcService))) {
             return;
         }
         const instances = await workflowService.getStepInstances(runId);
@@ -916,7 +934,7 @@ function planEntryFirings(graphName, nodeId, node, triggerNodeResults, triggerIn
     }
     return firings;
 }
-export async function runWorkflowGraph(workflowService, graphName, triggerInput, rpcService, inline, startNode, wire, overrideMeta) {
+export async function runWorkflowGraph(workflowService, graphName, triggerInput, rpcService, inline, startNode, wire, overrideMeta, onRunCreated) {
     const meta = overrideMeta ?? getWorkflowMeta(graphName);
     if (!meta?.nodes) {
         throw new Error(`Workflow graph '${graphName}' not found`);
@@ -946,6 +964,7 @@ export async function runWorkflowGraph(workflowService, graphName, triggerInput,
         deterministic: meta.deterministic,
         plannedSteps: meta.plannedSteps,
     });
+    onRunCreated?.(runId);
     if (inline) {
         workflowService.registerInlineRun(runId);
     }

package/dist/wirings/workflow/index.d.ts

@@ -7,6 +7,7 @@ export { WorkflowApprovalForbiddenError } from './workflow-approval-policy.js';
 export { streamWorkflowRunStatus, type WorkflowStatusStreamParams, } from './workflow-status-stream.js';
 export type { WorkflowRunEngine, WorkflowRunExtension, } from './workflow-run-engine.types.js';
 export { deriveInvocationId, uuidv5 } from './workflow-invocation-id.js';
+export { runTraceWire } from './workflow-run-trace.js';
 export { isRef } from './graph/workflow-graph.types.js';
 export type { RefValue } from './graph/workflow-graph.types.js';
 export { buildRunTimeline, reconstructStateAt, reconstructFinalState, } from './run-timeline.js';
@@ -15,7 +16,8 @@ export { addWorkflow } from './dsl/workflow-runner.js';
 export { template, type TemplateString } from './graph/template.js';
 export { pikkuWorkflowGraph, type PikkuWorkflowGraphConfig, type PikkuWorkflowGraphResult, } from './graph/wire-workflow-graph.js';
 export type { WorkflowStepInput as WorkflowStepQueueInput, PikkuWorkflowOrchestratorInput, PikkuWorkflowSleeperInput, } from './workflow-queue-workers.js';
-export type { WorkflowService, WorkflowQueueOptions, WorkflowServiceOptions, WorkflowServiceConfig, WorkflowPlannedStep, WorkflowRunWire, WorkflowStatus, WorkflowVersionStatus, StepStatus, WorkflowRun, WorkflowRunStatus, StepState, WorkflowRunService, WorkflowRunMirror, CoreWorkflow, PikkuWorkflow, ContextVariable, WorkflowContext, WorkflowsMeta, WorkflowsRuntimeMeta, } from './workflow.types.js';
+export type { WorkflowService, WorkflowQueueOptions, WorkflowServiceOptions, WorkflowRecoveryOptions, WorkflowServiceConfig, WorkflowPlannedStep, WorkflowRunWire, WorkflowStatus, WorkflowVersionStatus, StepStatus, WorkflowRun, WorkflowRunStatus, StepState, WorkflowRunService, WorkflowRunMirror, CoreWorkflow, PikkuWorkflow, ContextVariable, WorkflowContext, WorkflowsMeta, WorkflowsRuntimeMeta, } from './workflow.types.js';
 export type { WorkflowStepOptions, WorkflowWireDoRPC, WorkflowApprovalOptions, ApprovalOutcome, InputSource, OutputBinding, RpcStepMeta, Condition, BranchStepMeta, ParallelGroupStepMeta, FanoutStepMeta, ReturnStepMeta, InlineStepMeta, SleepStepMeta, CancelStepMeta, SuspendStepMeta, MilestoneStepMeta, ApprovalStepMeta, SetStepMeta, SwitchCaseMeta, SwitchStepMeta, FilterStepMeta, ArrayPredicateStepMeta, WorkflowStepMeta, WorkflowStepWire, PikkuWorkflowWire, } from './workflow.types.js';
 export { createGraph } from './graph/graph-node.js';
 export type { GraphNodeConfig, ForEachConfig, ForEachMode, ItemFn, TemplateFn, } from './graph/workflow-graph.types.js';
+export type { ScheduledStep, StalledSweepOptions, UndispatchedSweepOptions, WorkflowRecoveryPass, } from './workflow-recovery.js';

package/dist/wirings/workflow/index.js

@@ -6,6 +6,7 @@ export { assertWorkflowRunOwner, WorkflowRunForbiddenError, } from './workflow-r
 export { WorkflowApprovalForbiddenError } from './workflow-approval-policy.js';
 export { streamWorkflowRunStatus, } from './workflow-status-stream.js';
 export { deriveInvocationId, uuidv5 } from './workflow-invocation-id.js';
+export { runTraceWire } from './workflow-run-trace.js';
 export { isRef } from './graph/workflow-graph.types.js';
 export { buildRunTimeline, reconstructStateAt, reconstructFinalState, } from './run-timeline.js';
 export { addWorkflow } from './dsl/workflow-runner.js';

package/dist/wirings/workflow/pikku-scenario-service.js

@@ -1,4 +1,5 @@
 import { runPikkuFunc } from '../../function/function-runner.js';
+import { runTraceWire } from './workflow-run-trace.js';
 import { getSingletonServices, getCreateWireServices, pikkuState, } from '../../pikku-state.js';
 import { getDurationInMilliseconds } from '../../time-utils.js';
 import { closeWireServices } from '../../utils.js';
@@ -667,6 +668,7 @@ export class PikkuScenarioService {
             const runOnSurface = async (surface) => {
                 // No `rpc` yet — runPikkuFunc attaches it lazily for the invocation.
                 const wire = {
+                    ...runTraceWire(await this.engine.getRunTraceparent(runId)),
                     workflow: workflowWire,
                     scenario: workflowWire,
                     pikkuUserId: workflowWire.pikkuUserId,

package/dist/wirings/workflow/pikku-workflow-service.d.ts

@@ -1,13 +1,14 @@
 import type { CoreUserSession } from '../../types/core.types.js';
 import type { SerializedError } from '../../errors/serialized-error.js';
 import type { GroupConcurrencyConfig, JobGroup, JobOptions } from '../queue/queue.types.js';
-import type { PikkuWorkflowWire, StepState, StepStatus, WorkflowPlannedStep, WorkflowRun, WorkflowRunMirror, WorkflowRunStatus, WorkflowRunWire, WorkflowStatus, WorkflowVersionStatus, WorkflowQueueOptions, WorkflowStepOptions } from './workflow.types.js';
+import type { PikkuWorkflowWire, StepState, StepStatus, WorkflowPlannedStep, WorkflowRun, WorkflowRunMirror, WorkflowRunStatus, WorkflowRunWire, WorkflowStatus, WorkflowVersionStatus, WorkflowQueueOptions, WorkflowRecoveryOptions, WorkflowStepOptions } from './workflow.types.js';
 import type { WorkflowService } from '../../services/workflow-service.js';
 import type { ScenarioVirtualUsers } from '../../services/virtual-users-service.js';
 import type { PikkuRPC } from '../rpc/rpc-types.js';
 import { type RunTimeline, type ReconstructedRunState } from './run-timeline.js';
 import type { WorkflowRunEngine, WorkflowRunExtension } from './workflow-run-engine.types.js';
 import type { LeaseService } from '../../services/lease-service.js';
+import { type ScheduledStep, type StalledSweepOptions, type UndispatchedSweepOptions, type WorkflowRecoveryPass } from './workflow-recovery.js';
 /** The workflow engine; subclasses supply storage. Leases serialise runs and step claims over a shared store. */
 export declare abstract class PikkuWorkflowService implements WorkflowService {
     private runExtension?;
@@ -17,6 +18,7 @@ export declare abstract class PikkuWorkflowService implements WorkflowService {
     private enterExecution;
     private exitExecution;
     protected get logger(): import("../../services/logger.js").Logger;
+    private readonly lostStepLog;
     protected mirror?: WorkflowRunMirror;
     /**
      * Serialises orchestration passes over a run and claims on a step. A service
@@ -129,59 +131,45 @@ export declare abstract class PikkuWorkflowService implements WorkflowService {
     } | null>;
     resumeWorkflow(runId: string, workflowName?: string): Promise<void>;
     private enqueueOrchestration;
+    private addOrchestratorJob;
     /**
-     * Ids of runs that are stalled: still `running`, with no step in a state that
-     * something is expected to complete (`scheduled`, `suspended`, or `running`
-     * under a live lease), and no step activity since `before`.
+     * Ids of runs that are stalled: still `running`, with no step something is
+     * expected to complete (`scheduled`, `suspended`, `running` under a live
+     * lease, or parked on a child run), and no step activity since `before`.
      *
-     * A `running` step whose lease has lapsed is the opposite of in flight — the
-     * dispatch that claimed it is gone — so it does not hold its run out of the
-     * sweep.
+     * A lapsed lease is a dispatch that died, so it holds nothing out of the
+     * sweep. Nor does a `running` step with no lease and no child run: that is
+     * work inside an orchestration pass, whose owner is the run's lease, which the
+     * sweep checks before it touches the run.
      *
-     * Returns nothing by default so a store that cannot express the query keeps
-     * working unchanged; a store that overrides it gains crash recovery through
-     * `recoverStalledRuns`.
+     * Returns nothing by default, for a store that cannot express the query.
      */
     protected findStalledRunIds(_before: Date, _limit: number): Promise<string[]>;
     /**
-     * Runs holding a step that has sat `pending` since before `before`, paired
-     * with the step that flagged them.
-     *
-     * Returns nothing by default so a store that cannot express the query keeps
-     * working unchanged — and, because it does not opt in, gains no re-dispatches
-     * either. A store must have an atomic `claimStepForExecution` before
-     * overriding this, or no concurrency for one to exclude: the relay makes
-     * duplicate dispatch routine, and the claim is what keeps a duplicate from
-     * becoming a second execution. Every `@pikku/kysely` dialect qualifies on its
-     * status-guarded claim, `mongodb` on the same claim expressed as a
-     * single-document update, `in-memory` on being inline and single-process —
-     * none of them overrides this yet.
+     * Runs holding a step `pending` since before `before`. A store overriding it
+     * needs an atomic `claimStepForExecution` (see `sweepUndispatchedSteps`).
      */
     protected findUndispatchedSteps(_before: Date, _limit: number): Promise<Array<{
         runId: string;
         stepId: string;
     }>>;
-    private readonly redispatchBackoff;
-    private get sweepDeps();
     /**
-     * Re-drive runs whose next move was lost. Not self-starting — call it from a
-     * scheduled task at whatever interval suits the workload.
+     * Steps parked on `scheduled` on a live run, oldest first, a page at a time.
+     * Returns nothing by default, for a store whose runs die with the process.
      */
-    recoverStalledRuns(options?: {
-        stalledAfterMs?: number;
-        limit?: number;
-    }): Promise<{
+    protected findScheduledSteps(_limit: number, _offset: number): Promise<ScheduledStep[]>;
+    private readonly recovery;
+    /** One recovery pass, for a host driving it from a scheduled task. */
+    recoverWorkflows(options?: WorkflowRecoveryPass): Promise<void>;
+    /** Sweep now and on an interval, once the queue is consuming. */
+    startRecovery(options?: WorkflowRecoveryOptions | false): void;
+    stopRecovery(): void;
+    /** Re-drive runs whose next move was lost. */
+    recoverStalledRuns(options?: StalledSweepOptions): Promise<{
         resumed: string[];
     }>;
-    /**
-     * Re-drive steps whose dispatch was lost. Not self-starting — call it from a
-     * scheduled task; ~30s suits a queue whose `pending`→`running` latency is
-     * well under that.
-     */
-    relayUndispatchedSteps(options?: {
-        undispatchedAfterMs?: number;
-        limit?: number;
-    }): Promise<{
+    /** Re-drive steps whose dispatch was lost. Not self-starting. */
+    relayUndispatchedSteps(options?: UndispatchedSweepOptions): Promise<{
         redispatched: string[];
     }>;
     protected resolveStepJobOptions(stepOptions?: WorkflowStepOptions): JobOptions;
@@ -212,7 +200,13 @@ export declare abstract class PikkuWorkflowService implements WorkflowService {
     private beginReplay;
     private endReplay;
     private loadOrCreateStep;
-    private getRunIdentity;
+    getRunIdentity(runId: string): Promise<WorkflowRun | null>;
+    /** The trace every hop of the run stays on. */
+    getRunTraceparent(runId: string): Promise<string | undefined>;
+    /** Warn that a step lost its worker and is running again, as `attempt`. */
+    logStepRerun(runId: string, stepName: string, attempt: number): Promise<void>;
+    /** Warn that a step lost its worker with no attempts left, and failed. */
+    logStepAttemptsExhausted(runId: string, stepName: string, attempts: number): Promise<void>;
     private lastStepName;
     private nextStepKey;
     runWorkflowJob(runId: string, rpcService: PikkuRPC): Promise<void>;
@@ -256,6 +250,9 @@ export declare abstract class PikkuWorkflowService implements WorkflowService {
     failOrUnwind(run: WorkflowRun, error: Error & {
         code?: string;
     }, rpcService: PikkuRPC): Promise<'unwinding' | 'finished' | 'failed'>;
+    /** Ends a run's body at its next inline attempt once the run has been stopped. */
+    throwIfRunStopped(runId: string): Promise<void>;
+    settleStoppedRun(runId: string, rpcService: PikkuRPC): Promise<WorkflowStatus | undefined>;
     /** Stops an unfinished run, children first, and unwinds what it completed. */
     cancelRun(runId: string, rpcService: PikkuRPC, reason?: string): Promise<WorkflowStatus | undefined>;
     protected markFailedUnlessUnwound(runId: string, error: SerializedError): Promise<void>;