@pikku/core 0.12.148 → 0.12.151

package/dist/wirings/workflow/workflow-dsl-pass.js

@@ -1,4 +1,5 @@
 import { runPikkuFunc } from '../../function/function-runner.js';
+import { runTraceWire } from './workflow-run-trace.js';
 import { getCreateWireServices, getSingletonServices, } from '../../pikku-state.js';
 import { WorkflowAsyncException, WorkflowCancelledException, WorkflowSuspendedException, } from './workflow-errors.js';
 export const runDslWorkflowPass = async (deps, run, workflowMeta, workflow, pkgName) => {
@@ -9,6 +10,7 @@ export const runDslWorkflowPass = async (deps, run, workflowMeta, workflow, pkgN
     const workflowWire = deps.workflowWire(addonNs);
     workflowWire.pikkuUserId = run.wire?.pikkuUserId;
     const wire = {
+        ...runTraceWire(run.wire?.traceparent),
         workflow: workflowWire,
         pikkuUserId: run.wire?.pikkuUserId,
     };
@@ -36,6 +38,11 @@ export const runDslWorkflowPass = async (deps, run, workflowMeta, workflow, pkgN
             data: () => run.input,
             packageName: pkgName ?? undefined,
         });
+        // The run has been in the hands of its steps, and an inline one never went
+        // back through the queue, so what it was told is only known by rereading.
+        if (await deps.settleIfStopped()) {
+            throw new WorkflowCancelledException(runId);
+        }
         await deps.updateRunStatus('completed', result);
         await deps.onChildCompleted(result);
     }
@@ -47,24 +54,33 @@ export const runDslWorkflowPass = async (deps, run, workflowMeta, workflow, pkgN
         }
         if (error instanceof WorkflowCancelledException) {
             outcome = 'failed';
-            await deps.updateRunStatus('cancelled', undefined, {
-                message: error.message || 'Workflow cancelled',
-                stack: '',
-                code: 'WORKFLOW_CANCELLED',
-            });
-            await deps.onChildFailed(error);
+            const stopped = await deps.settleIfStopped();
+            if (!stopped) {
+                await deps.updateRunStatus('cancelled', undefined, {
+                    message: error.message || 'Workflow cancelled',
+                    stack: '',
+                    code: 'WORKFLOW_CANCELLED',
+                });
+            }
+            // An unwind tells the parent itself, when it finishes.
+            if (!stopped || stopped === 'cancelled')
+                await deps.onChildFailed(error);
             throw error;
         }
         if (error instanceof WorkflowSuspendedException) {
             outcome = 'interrupted';
-            await deps.updateRunStatus('suspended', undefined, {
-                message: error.message || 'Workflow suspended',
-                stack: '',
-                code: 'WORKFLOW_SUSPENDED',
-            });
+            if (!(await deps.settleIfStopped())) {
+                await deps.updateRunStatus('suspended', undefined, {
+                    message: error.message || 'Workflow suspended',
+                    stack: '',
+                    code: 'WORKFLOW_SUSPENDED',
+                });
+            }
             throw error;
         }
         outcome = 'failed';
+        if (await deps.settleIfStopped())
+            throw error;
         if ((await deps.failOrUnwind(error)) === 'unwinding')
             return;
         throw error;

package/dist/wirings/workflow/workflow-recovery.d.ts

@@ -1,4 +1,6 @@
+import type { LeaseService } from '../../services/lease-service.js';
 import type { Logger } from '../../services/logger.js';
+import type { StepState, WorkflowRecoveryOptions, WorkflowStepOptions } from './workflow.types.js';
 /**
  * Per-process, advisory record of when a run may next be re-dispatched.
  *
@@ -22,10 +24,13 @@ export declare class RedispatchBackoff {
     private readonly eligibleAt;
     private readonly delays;
     isEligible(runId: string, now: number): boolean;
+    /** Drop a record, for a re-drive that turned out not to happen. */
+    forget(runId: string): void;
     note(runId: string, now: number): void;
 }
 type SweepDeps = {
-    resume: (runId: string) => Promise<void>;
+    /** Resolves `false` when the run was left alone rather than re-driven. */
+    resume: (runId: string) => Promise<boolean | void>;
     logger?: Logger;
 };
 /**
@@ -41,8 +46,9 @@ type SweepDeps = {
  * Replay is the recovery — `resumeWorkflow` re-orchestrates from persisted step
  * state, and every settled step is memoized, so resuming a run that was not
  * actually stuck costs an orchestration pass and changes nothing. That
- * idempotence is what makes an idle-time heuristic safe here; a run that is
- * legitimately mid-sleep is excluded anyway, since its step is `scheduled`.
+ * idempotence is what makes an idle-time heuristic safe here. A run parked on a
+ * `scheduled` step is not this sweep's: `sweepScheduledSteps` owns those, since
+ * replay cannot move them.
  *
  * Idempotent is not free, though: a run stays stalled until something clears
  * the reason it stalled, so an unconditional sweep re-queues the same runs on
@@ -72,6 +78,13 @@ export declare const sweepStalledRuns: (findStalledRunIds: (before: Date, limit:
  * Re-dispatches back off per run (doubling from 30s, capped at 10m) so a
  * genuine queue backlog is not amplified by a tick that keeps firing at the
  * steps the backlog is already delaying.
+ *
+ * A store opts in by overriding `findUndispatchedSteps`, and needs an atomic
+ * `claimStepForExecution` first: the relay makes duplicate dispatch routine, and
+ * the claim is what keeps a duplicate from becoming a second execution. Every
+ * `@pikku/kysely` dialect qualifies on its status-guarded claim, `mongodb` on
+ * the same claim as a single-document update, `in-memory` on being inline and
+ * single-process.
  */
 export declare const sweepUndispatchedSteps: (findUndispatchedSteps: (before: Date, limit: number) => Promise<Array<{
     runId: string;
@@ -82,4 +95,119 @@ export declare const sweepUndispatchedSteps: (findUndispatchedSteps: (before: Da
 } | undefined, deps: SweepDeps) => Promise<{
     redispatched: string[];
 }>;
+/** A step parked on `scheduled`, waiting for a timer or a queue job. */
+export type ScheduledStep = {
+    runId: string;
+    stepId: string;
+    stepName: string;
+    /** `null` is a `workflow.sleep`; anything else is a dispatched step. */
+    rpcName: string | null;
+    data: unknown;
+    fromStepName?: string;
+    retries?: number;
+    retryDelay?: string | number;
+    /** When the step went `scheduled`, on the writer's clock. */
+    scheduledAt: Date;
+};
+type ScheduledStepDeps = {
+    /** Set the sleep's result and re-drive its run; `false` if it was left alone. */
+    wake: (step: ScheduledStep) => Promise<boolean>;
+    /** Arm a fresh wake-up `remainingMs` from now. */
+    rearm: (step: ScheduledStep, remainingMs: number) => Promise<void>;
+    /** Send a dispatched step to the step queue again. */
+    redispatch: (step: ScheduledStep) => Promise<void>;
+    logger?: Logger;
+};
+/**
+ * Recover steps parked on `scheduled` whose timer or queue job may have died
+ * with the process that armed them.
+ *
+ * Replay cannot do this: it reads `scheduled` as "something is coming" and
+ * stops, which is right for every pass except the one after the something was
+ * lost. Sleeps are judged by the clock. A sleep past its wake time is woken, and
+ * `rearm` (set at startup, when this process has armed nothing) gives one that
+ * is not yet due a timer for what remains of it, so a restart neither loses a
+ * wake-up nor restarts the wait. Past startup a sleep is left alone until it is
+ * `wakeGraceMs` overdue, since only then is its timer known to be lost.
+ *
+ * Both are safe to repeat across instances. Every instance that starts arms its
+ * own timer for a sleep it cannot tell is someone else's, but a wake-up only
+ * sets a sleep's result, which is the same `null` however many times it lands,
+ * and resuming is idempotent. Dispatched steps are safe on the same terms as
+ * the relay: age is the only signal, a redundant message costs one queue job,
+ * and the claim is what keeps it from being a second execution.
+ */
+export declare const sweepScheduledSteps: (findScheduledSteps: (limit: number, offset: number) => Promise<ScheduledStep[]>, backoff: RedispatchBackoff, options: {
+    rearm?: boolean;
+    wakeGraceMs?: number;
+    undispatchedAfterMs?: number;
+}, deps: ScheduledStepDeps) => Promise<{
+    woken: string[];
+    rearmed: string[];
+    redispatched: string[];
+}>;
+export type StalledSweepOptions = {
+    stalledAfterMs?: number;
+    limit?: number;
+};
+export type UndispatchedSweepOptions = {
+    undispatchedAfterMs?: number;
+    limit?: number;
+};
+/** `startup` is a process that has armed nothing yet. */
+export type WorkflowRecoveryPass = WorkflowRecoveryOptions & {
+    startup?: boolean;
+};
+/** What recovery needs from the workflow service. */
+export type WorkflowRecoveryHost = {
+    leases: () => LeaseService | undefined;
+    listStepStates: (runId: string) => Promise<StepState[] | null>;
+    withRunLease: (runId: string, fn: () => Promise<void>) => Promise<void>;
+    findStalledRunIds: (before: Date, limit: number) => Promise<string[]>;
+    findUndispatchedSteps: (before: Date, limit: number) => Promise<Array<{
+        runId: string;
+        stepId: string;
+    }>>;
+    findScheduledSteps: (limit: number, offset: number) => Promise<ScheduledStep[]>;
+    resumeWorkflow: (runId: string) => Promise<void>;
+    wakeSleep: (runId: string, stepId: string) => Promise<void>;
+    rearmSleep: (runId: string, stepId: string, remainingMs: number) => Promise<unknown>;
+    queueStepWorker: (runId: string, stepName: string, rpcName: string, data: any, stepOptions?: WorkflowStepOptions, fromStepName?: string) => Promise<void>;
+};
+/** Every way a workflow service brings back work a dead process left behind. */
+export declare class WorkflowRecovery {
+    private readonly host;
+    private readonly backoff;
+    private timer;
+    private sweeping;
+    constructor(host: WorkflowRecoveryHost);
+    private get logger();
+    /**
+     * Do `drive` to a run unless something else owns it, reporting whether it
+     * did. The run's lease is the ownership: a pass running anywhere holds it, and
+     * it outlives a crashed holder by its TTL, so a sweep on one instance cannot
+     * take a run from another and a dead instance's runs come free on their own.
+     */
+    private driveUnlessOwned;
+    private get sweepDeps();
+    recoverStalledRuns(options?: StalledSweepOptions): Promise<{
+        resumed: string[];
+    }>;
+    relayUndispatchedSteps(options?: UndispatchedSweepOptions): Promise<{
+        redispatched: string[];
+    }>;
+    /**
+     * Bring back everything a dead process left behind, once: steps parked on
+     * `scheduled`, runs with nothing in flight, and steps whose dispatch was
+     * lost. `startup` is a process that has armed nothing yet, so its sleeps get
+     * timers and nothing has to look idle first.
+     *
+     * One pass per lease, so a fleet does not sweep the same rows in step.
+     */
+    recover(options?: WorkflowRecoveryPass): Promise<void>;
+    /** `false` is the `workflow.recovery` config saying not to. */
+    start(options?: WorkflowRecoveryOptions | false): void;
+    stop(): void;
+    private sweep;
+}
 export {};

package/dist/wirings/workflow/workflow-recovery.js

@@ -1,4 +1,8 @@
-import { DEFAULT_STALLED_RUN_LIMIT, DEFAULT_STALLED_RUN_MS, DEFAULT_UNDISPATCHED_STEP_LIMIT, DEFAULT_UNDISPATCHED_STEP_MS, REDISPATCH_BACKOFF_MAX_ENTRIES, REDISPATCH_BACKOFF_MAX_MS, REDISPATCH_BACKOFF_MS, } from './workflow-constants.js';
+import { holdLease, LeaseTakenError } from '../../services/lease-service.js';
+import { getSingletonServices } from '../../pikku-state.js';
+import { getDurationInMilliseconds } from '../../time-utils.js';
+import { DEFAULT_RECOVERY_INTERVAL_MS, DEFAULT_RECOVERY_STALLED_MS, DEFAULT_STALLED_RUN_LIMIT, DEFAULT_STALLED_RUN_MS, DEFAULT_UNDISPATCHED_STEP_LIMIT, DEFAULT_UNDISPATCHED_STEP_MS, REDISPATCH_BACKOFF_MAX_ENTRIES, REDISPATCH_BACKOFF_MAX_MS, REDISPATCH_BACKOFF_MS, SCHEDULED_STEP_PAGE_SIZE, SLEEP_WAKE_GRACE_MS, } from './workflow-constants.js';
+import { isRunLeaseError } from './workflow-run-lease.js';
 /**
  * Per-process, advisory record of when a run may next be re-dispatched.
  *
@@ -25,6 +29,11 @@ export class RedispatchBackoff {
         const at = this.eligibleAt.get(runId);
         return at === undefined || at <= now;
     }
+    /** Drop a record, for a re-drive that turned out not to happen. */
+    forget(runId) {
+        this.eligibleAt.delete(runId);
+        this.delays.delete(runId);
+    }
     note(runId, now) {
         const previous = this.delays.get(runId);
         const delay = Math.min(previous === undefined ? REDISPATCH_BACKOFF_MS : previous * 2, REDISPATCH_BACKOFF_MAX_MS);
@@ -41,11 +50,16 @@ export class RedispatchBackoff {
         this.eligibleAt.set(runId, now + delay);
     }
 }
-const resumeEach = async (runIds, { resume, logger }, failure) => {
+const resumeEach = async (runIds, backoff, { resume, logger }, failure) => {
     const succeeded = [];
     for (const runId of runIds) {
         try {
-            await resume(runId);
+            if ((await resume(runId)) === false) {
+                // Nothing was re-driven, so nothing has earned a wait: a run owned by
+                // a live instance now may have lost that instance by the next tick.
+                backoff.forget(runId);
+                continue;
+            }
             succeeded.push(runId);
         }
         catch (err) {
@@ -68,8 +82,9 @@ const resumeEach = async (runIds, { resume, logger }, failure) => {
  * Replay is the recovery — `resumeWorkflow` re-orchestrates from persisted step
  * state, and every settled step is memoized, so resuming a run that was not
  * actually stuck costs an orchestration pass and changes nothing. That
- * idempotence is what makes an idle-time heuristic safe here; a run that is
- * legitimately mid-sleep is excluded anyway, since its step is `scheduled`.
+ * idempotence is what makes an idle-time heuristic safe here. A run parked on a
+ * `scheduled` step is not this sweep's: `sweepScheduledSteps` owns those, since
+ * replay cannot move them.
  *
  * Idempotent is not free, though: a run stays stalled until something clears
  * the reason it stalled, so an unconditional sweep re-queues the same runs on
@@ -94,7 +109,7 @@ export const sweepStalledRuns = async (findStalledRunIds, backoff, options, deps
         runIds.push(runId);
     }
     return {
-        resumed: await resumeEach(runIds, deps, (runId, detail) => `Failed to resume stalled workflow run ${runId}: ${detail}`),
+        resumed: await resumeEach(runIds, backoff, deps, (runId, detail) => `Failed to resume stalled workflow run ${runId}: ${detail}`),
     };
 };
 /**
@@ -111,6 +126,13 @@ export const sweepStalledRuns = async (findStalledRunIds, backoff, options, deps
  * Re-dispatches back off per run (doubling from 30s, capped at 10m) so a
  * genuine queue backlog is not amplified by a tick that keeps firing at the
  * steps the backlog is already delaying.
+ *
+ * A store opts in by overriding `findUndispatchedSteps`, and needs an atomic
+ * `claimStepForExecution` first: the relay makes duplicate dispatch routine, and
+ * the claim is what keeps a duplicate from becoming a second execution. Every
+ * `@pikku/kysely` dialect qualifies on its status-guarded claim, `mongodb` on
+ * the same claim as a single-document update, `in-memory` on being inline and
+ * single-process.
  */
 export const sweepUndispatchedSteps = async (findUndispatchedSteps, backoff, options, deps) => {
     const before = new Date(Date.now() - (options?.undispatchedAfterMs ?? DEFAULT_UNDISPATCHED_STEP_MS));
@@ -124,6 +146,210 @@ export const sweepUndispatchedSteps = async (findUndispatchedSteps, backoff, opt
         runIds.add(runId);
     }
     return {
-        redispatched: await resumeEach(runIds, deps, (runId, detail) => `Failed to re-dispatch workflow run ${runId}: ${detail}`),
+        redispatched: await resumeEach(runIds, backoff, deps, (runId, detail) => `Failed to re-dispatch workflow run ${runId}: ${detail}`),
     };
 };
+/**
+ * Recover steps parked on `scheduled` whose timer or queue job may have died
+ * with the process that armed them.
+ *
+ * Replay cannot do this: it reads `scheduled` as "something is coming" and
+ * stops, which is right for every pass except the one after the something was
+ * lost. Sleeps are judged by the clock. A sleep past its wake time is woken, and
+ * `rearm` (set at startup, when this process has armed nothing) gives one that
+ * is not yet due a timer for what remains of it, so a restart neither loses a
+ * wake-up nor restarts the wait. Past startup a sleep is left alone until it is
+ * `wakeGraceMs` overdue, since only then is its timer known to be lost.
+ *
+ * Both are safe to repeat across instances. Every instance that starts arms its
+ * own timer for a sleep it cannot tell is someone else's, but a wake-up only
+ * sets a sleep's result, which is the same `null` however many times it lands,
+ * and resuming is idempotent. Dispatched steps are safe on the same terms as
+ * the relay: age is the only signal, a redundant message costs one queue job,
+ * and the claim is what keeps it from being a second execution.
+ */
+export const sweepScheduledSteps = async (findScheduledSteps, backoff, options, deps) => {
+    const outcome = {
+        woken: [],
+        rearmed: [],
+        redispatched: [],
+    };
+    const grace = options.wakeGraceMs ?? SLEEP_WAKE_GRACE_MS;
+    const lostBefore = Date.now() - (options.undispatchedAfterMs ?? DEFAULT_UNDISPATCHED_STEP_MS);
+    const recover = async (step) => {
+        const now = Date.now();
+        if (step.rpcName !== null) {
+            if (step.scheduledAt.getTime() >= lostBefore)
+                return;
+            if (!backoff.isEligible(step.stepId, now))
+                return;
+            backoff.note(step.stepId, now);
+            await deps.redispatch(step);
+            outcome.redispatched.push(step.runId);
+            return;
+        }
+        const duration = step.data
+            ?.duration;
+        if (duration === undefined)
+            return;
+        const remaining = step.scheduledAt.getTime() + getDurationInMilliseconds(duration) - now;
+        if (options.rearm && remaining > 0) {
+            await deps.rearm(step, remaining);
+            outcome.rearmed.push(step.runId);
+        }
+        else if (remaining <= (options.rearm ? 0 : -grace)) {
+            if (await deps.wake(step))
+                outcome.woken.push(step.runId);
+        }
+    };
+    for (let offset = 0;; offset += SCHEDULED_STEP_PAGE_SIZE) {
+        const page = await findScheduledSteps(SCHEDULED_STEP_PAGE_SIZE, offset);
+        for (const step of page) {
+            try {
+                await recover(step);
+            }
+            catch (err) {
+                // One unrecoverable step must not stop the sweep from reaching the rest.
+                deps.logger?.error(`Failed to recover scheduled step ${step.stepName} of workflow run ${step.runId}: ${err instanceof Error ? err.message : String(err)}`);
+            }
+        }
+        if (page.length < SCHEDULED_STEP_PAGE_SIZE)
+            return outcome;
+    }
+};
+/** A store may hand a numeric duration back as the string it wrote. */
+const storedDurationMs = (duration) => /^\d+$/.test(String(duration))
+    ? Number(duration)
+    : getDurationInMilliseconds(duration);
+/** A retry sitting out its `retryDelay` is idle on purpose, not stalled. */
+const isWaitingOutRetryDelay = (steps) => {
+    const now = Date.now();
+    return (steps ?? []).some(({ status, attemptCount, retryDelay, updatedAt }) => status === 'pending' &&
+        attemptCount > 1 &&
+        retryDelay != null &&
+        updatedAt.getTime() + storedDurationMs(retryDelay) > now);
+};
+/** Every way a workflow service brings back work a dead process left behind. */
+export class WorkflowRecovery {
+    host;
+    backoff = new RedispatchBackoff();
+    timer;
+    sweeping = false;
+    constructor(host) {
+        this.host = host;
+    }
+    get logger() {
+        return getSingletonServices()?.logger;
+    }
+    /**
+     * Do `drive` to a run unless something else owns it, reporting whether it
+     * did. The run's lease is the ownership: a pass running anywhere holds it, and
+     * it outlives a crashed holder by its TTL, so a sweep on one instance cannot
+     * take a run from another and a dead instance's runs come free on their own.
+     */
+    async driveUnlessOwned(runId, drive) {
+        if (isWaitingOutRetryDelay(await this.host.listStepStates(runId))) {
+            return false;
+        }
+        try {
+            await this.host.withRunLease(runId, drive);
+            return true;
+        }
+        catch (error) {
+            if (isRunLeaseError(error))
+                return false;
+            throw error;
+        }
+    }
+    get sweepDeps() {
+        return {
+            resume: (runId) => this.driveUnlessOwned(runId, () => this.host.resumeWorkflow(runId)),
+            logger: this.logger,
+        };
+    }
+    recoverStalledRuns(options) {
+        return sweepStalledRuns(this.host.findStalledRunIds, this.backoff, options, this.sweepDeps);
+    }
+    relayUndispatchedSteps(options) {
+        return sweepUndispatchedSteps(this.host.findUndispatchedSteps, this.backoff, options, this.sweepDeps);
+    }
+    /**
+     * Bring back everything a dead process left behind, once: steps parked on
+     * `scheduled`, runs with nothing in flight, and steps whose dispatch was
+     * lost. `startup` is a process that has armed nothing yet, so its sleeps get
+     * timers and nothing has to look idle first.
+     *
+     * One pass per lease, so a fleet does not sweep the same rows in step.
+     */
+    async recover(options = {}) {
+        const { host } = this;
+        const { startup = false } = options;
+        const undispatchedAfterMs = startup
+            ? 0
+            : (options.undispatchedAfterMs ?? DEFAULT_UNDISPATCHED_STEP_MS);
+        const pass = async () => {
+            const scheduled = await sweepScheduledSteps(host.findScheduledSteps, this.backoff, { rearm: startup, undispatchedAfterMs }, {
+                wake: ({ runId, stepId }) => this.driveUnlessOwned(runId, () => host.wakeSleep(runId, stepId)),
+                rearm: async ({ runId, stepId }, remainingMs) => {
+                    await host.rearmSleep(runId, stepId, remainingMs);
+                },
+                redispatch: (step) => host.queueStepWorker(step.runId, step.stepName, step.rpcName, step.data, { retries: step.retries, retryDelay: step.retryDelay }, step.fromStepName),
+                logger: this.logger,
+            });
+            const { resumed } = await this.recoverStalledRuns({
+                stalledAfterMs: startup
+                    ? 0
+                    : (options.stalledAfterMs ?? DEFAULT_RECOVERY_STALLED_MS),
+            });
+            const { redispatched } = await this.relayUndispatchedSteps({
+                undispatchedAfterMs,
+            });
+            const { woken, rearmed } = scheduled;
+            if (woken.length +
+                rearmed.length +
+                scheduled.redispatched.length +
+                resumed.length +
+                redispatched.length >
+                0) {
+                this.logger?.info(`Workflow recovery: ${woken.length} sleeps woken, ${rearmed.length} re-armed, ${scheduled.redispatched.length} steps re-dispatched, ${resumed.length} stalled runs and ${redispatched.length} undispatched runs resumed`);
+            }
+        };
+        const leases = host.leases();
+        if (!leases)
+            return pass();
+        try {
+            await holdLease(leases, 'workflow-recovery', pass);
+        }
+        catch (error) {
+            if (!(error instanceof LeaseTakenError))
+                throw error;
+        }
+    }
+    /** `false` is the `workflow.recovery` config saying not to. */
+    start(options) {
+        if (options === false || this.timer)
+            return;
+        this.timer = setInterval(() => void this.sweep(options, false), options?.intervalMs ?? DEFAULT_RECOVERY_INTERVAL_MS);
+        this.timer.unref?.();
+        // Fire-and-forget: `sweep` logs every failure itself.
+        void this.sweep(options, true);
+    }
+    stop() {
+        clearInterval(this.timer);
+        this.timer = undefined;
+    }
+    async sweep(options, startup) {
+        if (this.sweeping)
+            return;
+        this.sweeping = true;
+        try {
+            await this.recover({ ...options, startup });
+        }
+        catch (error) {
+            this.logger?.error(`Workflow recovery sweep failed: ${error instanceof Error ? error.message : String(error)}`);
+        }
+        finally {
+            this.sweeping = false;
+        }
+    }
+}

package/dist/wirings/workflow/workflow-run-engine.types.d.ts

@@ -18,6 +18,7 @@ export interface WorkflowRunEngine {
     updateRunStatus(runId: string, status: WorkflowStatus, output?: any, error?: SerializedError): Promise<void>;
     onChildWorkflowFailed(run: WorkflowRun, error: unknown): Promise<void>;
     verifyStepName(stepName: unknown): void;
+    getRunTraceparent(runId: string): Promise<string | undefined>;
 }
 /**
  * Hooks a host may install to decorate runs it did not start. Used by the

package/dist/wirings/workflow/workflow-run-mirror.d.ts

@@ -0,0 +1,4 @@
+import type { Logger } from '../../services/logger.js';
+import type { WorkflowRunMirror } from './workflow.types.js';
+/** Writes to the mirror, if there is one, without ever failing the run. */
+export declare const writeRunMirror: (mirror: WorkflowRunMirror | undefined, getLogger: () => Logger | undefined, write: (mirror: WorkflowRunMirror) => Promise<void>) => Promise<void>;

package/dist/wirings/workflow/workflow-run-mirror.js

@@ -0,0 +1,16 @@
+/** Writes to the mirror, if there is one, without ever failing the run. */
+export const writeRunMirror = async (mirror, getLogger, write) => {
+    if (!mirror)
+        return;
+    try {
+        await write(mirror);
+    }
+    catch (err) {
+        try {
+            getLogger()?.warn?.(`[pikku] WorkflowRunMirror write failed: ${err?.message ?? err}`);
+        }
+        catch {
+            // knowledge: decisions/internals/workflow-run-mirror-is-never-a-source-of-truth.md
+        }
+    }
+};

package/dist/wirings/workflow/workflow-run-status.d.ts

@@ -2,3 +2,5 @@ import type { HistoryEntry } from './run-timeline.js';
 import type { WorkflowRun, WorkflowRunStatus } from './workflow.types.js';
 /** A run's status with each step folded to its latest attempt. */
 export declare function summarizeRunStatus(run: WorkflowRun, history: HistoryEntry[]): WorkflowRunStatus;
+/** What a finished run returned, or the error for a run that did not succeed. */
+export declare function runOutput(run: WorkflowRun): unknown;

package/dist/wirings/workflow/workflow-run-status.js

@@ -1,4 +1,5 @@
 import { WORKFLOW_TERMINAL_STATES } from './workflow-constants.js';
+import { WorkflowRunCancelledError, WorkflowRunFailedError, } from './workflow-errors.js';
 /** A run's status with each step folded to its latest attempt. */
 export function summarizeRunStatus(run, history) {
     const terminalStatuses = WORKFLOW_TERMINAL_STATES;
@@ -47,3 +48,15 @@ function compensationOutcome(run) {
     }
     return {};
 }
+/** What a finished run returned, or the error for a run that did not succeed. */
+export function runOutput(run) {
+    if (run.status === 'failed' ||
+        run.status === 'compensated' ||
+        run.status === 'compensation_failed') {
+        throw new WorkflowRunFailedError(run.error?.message);
+    }
+    if (run.status === 'cancelled') {
+        throw new WorkflowRunCancelledError();
+    }
+    return run.output;
+}

package/dist/wirings/workflow/workflow-run-trace.d.ts

@@ -0,0 +1,22 @@
+import { type TraceContext } from '../../trace.js';
+import type { Logger } from '../../services/logger.js';
+import type { WorkflowRunWire } from './workflow.types.js';
+/**
+ * The wire a run is stored with. It always holds a valid `traceparent`, so a
+ * run started from a wire with no trace still has one to keep through every
+ * pass, restart and recovery.
+ *
+ * knowledge: decisions/internals/a-workflow-run-owns-its-trace.md
+ */
+export declare const withRunTrace: (wire: WorkflowRunWire) => WorkflowRunWire;
+/**
+ * The wire fields for one hop of a run: the run's trace with a new span, so
+ * each attempt of a step is its own span in the same trace. Empty for a run
+ * that was stored without a trace.
+ */
+export declare const runTraceWire: (traceparent: string | undefined) => {
+    traceId?: string;
+    trace?: TraceContext;
+};
+/** `logger` on the trace of the run stored with `wire`. */
+export declare const runLogger: (logger: Logger, wire: WorkflowRunWire) => Logger;

package/dist/wirings/workflow/workflow-run-trace.js

@@ -0,0 +1,24 @@
+import { continueTrace, createTrace, formatTraceparent, parseTraceparent, tracedLogger, } from '../../trace.js';
+/**
+ * The wire a run is stored with. It always holds a valid `traceparent`, so a
+ * run started from a wire with no trace still has one to keep through every
+ * pass, restart and recovery.
+ *
+ * knowledge: decisions/internals/a-workflow-run-owns-its-trace.md
+ */
+export const withRunTrace = (wire) => parseTraceparent(wire?.traceparent)
+    ? wire
+    : { ...wire, traceparent: formatTraceparent(createTrace()) };
+/**
+ * The wire fields for one hop of a run: the run's trace with a new span, so
+ * each attempt of a step is its own span in the same trace. Empty for a run
+ * that was stored without a trace.
+ */
+export const runTraceWire = (traceparent) => {
+    if (!traceparent)
+        return {};
+    const trace = continueTrace(traceparent);
+    return { traceId: trace.traceId, trace };
+};
+/** `logger` on the trace of the run stored with `wire`. */
+export const runLogger = (logger, wire) => tracedLogger(logger, parseTraceparent(wire?.traceparent)?.traceId);

package/dist/wirings/workflow/workflow-step-claim.d.ts

@@ -1,4 +1,5 @@
 import type { StepState } from './workflow.types.js';
+import type { Logger } from '../../services/logger.js';
 /** What claiming a step needs from the workflow service. */
 export type StepClaimStore = {
     getStepState(runId: string, stepName: string): Promise<StepState>;
@@ -11,12 +12,29 @@ export type StepClaimStore = {
      * for lease exhaustion leaves the run `running` until a sweep notices.
      */
     resumeWorkflow?(runId: string): Promise<void>;
+    logStepRerun?(runId: string, stepName: string, attempt: number): Promise<void>;
+    logStepAttemptsExhausted?(runId: string, stepName: string, attempts: number): Promise<void>;
 };
+/** Where the warnings about a lost step go, and the trace of its run. */
+export type LostStepLog = {
+    logger(): Logger | undefined;
+    traceparent(runId: string): Promise<string | undefined>;
+};
+export declare const warnStepRerun: (log: LostStepLog, runId: string, stepName: string, attempt: number) => Promise<void>;
+export declare const warnStepAttemptsExhausted: (log: LostStepLog, runId: string, stepName: string, attempts: number) => Promise<void>;
 /**
  * Whether a step whose worker vanished may be handed to another one, or has
  * run out of attempts to spend on that.
  */
 export declare const leaseAttemptsExhausted: (stepState: StepState) => boolean;
+/**
+ * A pass holds its run's lease for as long as it works a step inline, so one
+ * that meets its own step already `running` is looking at a pass that died
+ * mid-step. Running it again is a new attempt, counted as one so a step that
+ * takes its process down every time runs out instead of looping forever. The
+ * invocation id comes from the step's name, so the attempt keeps it.
+ */
+export declare const takeBackLostInlineStep: (store: Pick<StepClaimStore, 'setStepError' | 'createRetryAttempt' | 'logStepRerun' | 'logStepAttemptsExhausted'>, runId: string, stepName: string, stepState: StepState) => Promise<StepState>;
 /**
  * Decide whether this dispatch owns the step, by reading its state and then
  * writing it — which only excludes a concurrent dispatch when the caller holds