@pikku/core 0.12.130 → 0.12.134

package/dist/wirings/workflow/workflow-errors.js

@@ -109,6 +109,44 @@ addError(WorkflowStepFunctionMismatchError, {
     status: 409,
     message: 'Workflow step was dispatched with a different function.',
 });
+/**
+ * Every dispatch that took this step lost its worker before finishing it, and
+ * the step has no attempts left to hand out. Failing here is the loud end of
+ * the loop: without it the step would be re-claimed and abandoned forever.
+ */
+export class WorkflowStepLeaseExpiredError extends PikkuError {
+    runId;
+    stepName;
+    attemptCount;
+    constructor(runId, stepName, attemptCount) {
+        super(`Workflow step '${stepName}' (run ${runId}) lost its worker on every one of its ${attemptCount} attempts: the last lease expired with the step still running`);
+        this.runId = runId;
+        this.stepName = stepName;
+        this.attemptCount = attemptCount;
+    }
+}
+addError(WorkflowStepLeaseExpiredError, {
+    status: 500,
+    message: 'Workflow step lost its worker and has no attempts left.',
+});
+/**
+ * A write from a dispatch that no longer owns its step: its lease lapsed and
+ * another dispatch claimed the step as a newer attempt. The newer attempt owns
+ * the outcome, so the stale one is dropped rather than recorded over it.
+ */
+export class WorkflowStepSupersededError extends PikkuError {
+    stepId;
+    attempt;
+    constructor(stepId, attempt) {
+        super(`Workflow step ${stepId}: attempt ${attempt} was superseded by a newer claim`);
+        this.stepId = stepId;
+        this.attempt = attempt;
+    }
+}
+addError(WorkflowStepSupersededError, {
+    status: 409,
+    message: 'Workflow step was claimed by a newer attempt.',
+});
 export class WorkflowStepNameNotString extends Error {
     constructor(stepName) {
         super(`Workflow step name must be a string. Received: ${typeof stepName}`);

package/dist/wirings/workflow/workflow-queue-routing.d.ts

@@ -1,4 +1,4 @@
-import type { JobGroup, JobOptions } from '../queue/queue.types.js';
+import type { JobGroup, JobOptions, QueueService } from '../queue/queue.types.js';
 import type { WorkflowServiceConfig, WorkflowStepOptions } from './workflow.types.js';
 export type WorkflowQueueStrategy = 'per-workflow' | 'shared-groups';
 export declare const resolveWorkflowConfig: () => WorkflowServiceConfig;
@@ -8,7 +8,7 @@ export declare const stepWorkerQueueName: (strategy: WorkflowQueueStrategy, rpcN
  * How a step reaches its worker: on the queue, or here in the orchestrator.
  *
  * A step naming a workflow queues whenever a queue exists, even unmarked. Run
- * here, it holds the parent's run lock — and its lock connection — until the
+ * here, it holds the parent's run lease until the
  * child ends, and marks the child inline so the child's own `sleep` degrades
  * from a suspension into a real in-process wait. Workflows cannot opt in
  * through `workflowQueued`: that flag is read off `rpc` meta, and `addWorkflow`
@@ -23,4 +23,6 @@ export declare const stepWorkerQueueName: (strategy: WorkflowQueueStrategy, rpcN
  */
 export declare const stepDispatchTarget: (rpcName: string, stepName: string, parentIsInline: () => Promise<boolean>) => Promise<'queue' | 'inline'>;
 export declare const jobGroupFor: (strategy: WorkflowQueueStrategy, id?: string) => JobGroup | undefined;
+/** The queue a remote run is driven through, or a clear account of its absence. */
+export declare const requireQueueService: () => QueueService;
 export declare const stepJobOptions: (stepOptions?: WorkflowStepOptions) => JobOptions;

package/dist/wirings/workflow/workflow-queue-routing.js

@@ -29,7 +29,7 @@ export const stepWorkerQueueName = (strategy, rpcName) => dedicatedQueueName('wf
  * How a step reaches its worker: on the queue, or here in the orchestrator.
  *
  * A step naming a workflow queues whenever a queue exists, even unmarked. Run
- * here, it holds the parent's run lock — and its lock connection — until the
+ * here, it holds the parent's run lease until the
  * child ends, and marks the child inline so the child's own `sleep` degrades
  * from a suspension into a real in-process wait. Workflows cannot opt in
  * through `workflowQueued`: that flag is read off `rpc` meta, and `addWorkflow`
@@ -61,6 +61,14 @@ export const stepDispatchTarget = async (rpcName, stepName, parentIsInline) => {
     return (await parentIsInline()) ? 'inline' : 'queue';
 };
 export const jobGroupFor = (strategy, id) => id && strategy === 'shared-groups' ? { id, tier: id } : undefined;
+/** The queue a remote run is driven through, or a clear account of its absence. */
+export const requireQueueService = () => {
+    const queueService = getSingletonServices()?.queueService;
+    if (!queueService) {
+        throw new Error('QueueService not configured. Remote workflows require a queue service.');
+    }
+    return queueService;
+};
 export const stepJobOptions = (stepOptions) => {
     const retries = stepOptions?.retries ?? DEFAULT_STEP_RETRIES;
     const retryDelay = stepOptions?.retryDelay;

package/dist/wirings/workflow/workflow-run-lease.d.ts

@@ -0,0 +1,10 @@
+import { type LeaseService } from '../../services/lease-service.js';
+/** The run is, or was meanwhile, orchestrated by someone else. */
+export declare const isRunLeaseError: (error: unknown) => boolean;
+/**
+ * Run `fn` holding `key` on the lease service a workflow service was built
+ * with. It never waits: a key held elsewhere throws `LeaseTakenError`.
+ */
+export declare const holdWorkflowLease: <T>(service: object, leases: LeaseService | undefined, key: string, fn: () => Promise<T>) => Promise<T>;
+/** A step whose lease is held elsewhere belongs to another dispatch. */
+export declare const nullWhenStepHeld: <T>(claim: () => Promise<T>) => Promise<T | null>;

package/dist/wirings/workflow/workflow-run-lease.js

@@ -0,0 +1,25 @@
+import { holdLease, LeaseLostError, LeaseTakenError, } from '../../services/lease-service.js';
+/** The run is, or was meanwhile, orchestrated by someone else. */
+export const isRunLeaseError = (error) => error instanceof LeaseTakenError || error instanceof LeaseLostError;
+/**
+ * Run `fn` holding `key` on the lease service a workflow service was built
+ * with. It never waits: a key held elsewhere throws `LeaseTakenError`.
+ */
+export const holdWorkflowLease = (service, leases, key, fn) => {
+    if (!leases) {
+        throw new Error(`${service.constructor.name} was constructed without a leaseService, so it cannot exclude a second process from a run or step. Pass the app's leaseService to its constructor.`);
+    }
+    return holdLease(leases, key, () => fn());
+};
+/** A step whose lease is held elsewhere belongs to another dispatch. */
+export const nullWhenStepHeld = async (claim) => {
+    try {
+        return await claim();
+    }
+    catch (error) {
+        if (error instanceof LeaseTakenError) {
+            return null;
+        }
+        throw error;
+    }
+};

package/dist/wirings/workflow/workflow-run-status.d.ts

@@ -0,0 +1,4 @@
+import type { HistoryEntry } from './run-timeline.js';
+import type { WorkflowRun, WorkflowRunStatus } from './workflow.types.js';
+/** A run's status with each step folded to its latest attempt. */
+export declare function summarizeRunStatus(run: WorkflowRun, history: HistoryEntry[]): WorkflowRunStatus;

package/dist/wirings/workflow/workflow-run-status.js

@@ -0,0 +1,37 @@
+/** A run's status with each step folded to its latest attempt. */
+export function summarizeRunStatus(run, history) {
+    const terminalStatuses = new Set(['completed', 'failed', 'cancelled']);
+    const stepMap = new Map();
+    for (const step of history) {
+        const existing = stepMap.get(step.stepName);
+        if (!existing || step.updatedAt > existing.completedAt) {
+            stepMap.set(step.stepName, {
+                status: step.status,
+                startedAt: step.runningAt ?? step.createdAt,
+                completedAt: step.succeededAt ?? step.failedAt,
+                attempts: step.attemptCount,
+            });
+        }
+    }
+    const steps = [...stepMap.entries()].map(([name, s]) => ({
+        name,
+        status: s.status,
+        duration: s.startedAt && s.completedAt
+            ? s.completedAt.getTime() - s.startedAt.getTime()
+            : undefined,
+        attempts: s.attempts,
+    }));
+    return {
+        id: run.id,
+        status: run.status,
+        startedAt: run.createdAt,
+        completedAt: terminalStatuses.has(run.status) ? run.updatedAt : undefined,
+        deterministic: run.deterministic,
+        plannedSteps: run.plannedSteps,
+        steps,
+        output: run.status === 'completed' ? run.output : undefined,
+        error: run.error
+            ? { message: run.error.message ?? 'Unknown error' }
+            : undefined,
+    };
+}

package/dist/wirings/workflow/workflow-status-stream.d.ts

@@ -15,7 +15,8 @@ export interface WorkflowStatusStreamParams {
     pollIntervalMs?: number;
 }
 /**
- * Streams one run's progress until it reaches a terminal state.
+ * Streams one run's progress until it reaches a terminal state. A suspended
+ * run is not terminal: it gets a `suspended` frame and the stream stays open.
  *
  * Polled rather than subscribed because a run's steps are written by whichever
  * worker picked them up, in whichever process — there is no in-memory event to

package/dist/wirings/workflow/workflow-status-stream.js

@@ -14,7 +14,8 @@ const TERMINAL = new Set([
 ]);
 const DEFAULT_POLL_INTERVAL_MS = 500;
 /**
- * Streams one run's progress until it reaches a terminal state.
+ * Streams one run's progress until it reaches a terminal state. A suspended
+ * run is not terminal: it gets a `suspended` frame and the stream stays open.
  *
  * Polled rather than subscribed because a run's steps are written by whichever
  * worker picked them up, in whichever process — there is no in-memory event to
@@ -27,6 +28,7 @@ const DEFAULT_POLL_INTERVAL_MS = 500;
 export const streamWorkflowRunStatus = async ({ workflowRunService, runId, channel, session, detailed = false, pollIntervalMs = DEFAULT_POLL_INTERVAL_MS, }) => {
     let lastHash = '';
     let initSent = false;
+    let announcedSuspend;
     const poll = async () => {
         const run = await workflowRunService.getRun(runId);
         if (!run) {
@@ -73,6 +75,21 @@ export const streamWorkflowRunStatus = async ({ workflowRunService, runId, chann
                 })),
             });
         }
+        // A suspended run is waiting for a decision or a signal, not finished, so
+        // the stream stays open for the resume. The frame is what tells a client
+        // that nothing more arrives until someone acts. `reason` is the string the
+        // workflow author gave `suspend()` for a person to read, so it goes to both
+        // routes, unlike `error`.
+        if (run.status === 'suspended') {
+            const reason = run.error?.message ?? 'Workflow suspended';
+            if (announcedSuspend !== reason) {
+                announcedSuspend = reason;
+                await channel.send({ type: 'suspended', reason });
+            }
+        }
+        else {
+            announcedSuspend = undefined;
+        }
         if (TERMINAL.has(run.status)) {
             await channel.send({ type: 'done' });
             await channel.close();

package/dist/wirings/workflow/workflow-step-claim.d.ts

@@ -3,8 +3,20 @@ import type { StepState } from './workflow.types.js';
 export type StepClaimStore = {
     getStepState(runId: string, stepName: string): Promise<StepState>;
     setStepRunning(stepId: string): Promise<void>;
+    setStepError(stepId: string, error: Error, attempt?: number): Promise<void>;
+    refreshStepLease(stepId: string, leaseMs: number | null): Promise<boolean>;
     createRetryAttempt(failedStepId: string, status: 'pending' | 'running'): Promise<StepState>;
+    /**
+     * Requeue the orchestrator so it sees the failure. Without it a step failed
+     * for lease exhaustion leaves the run `running` until a sweep notices.
+     */
+    resumeWorkflow?(runId: string): Promise<void>;
 };
+/**
+ * Whether a step whose worker vanished may be handed to another one, or has
+ * run out of attempts to spend on that.
+ */
+export declare const leaseAttemptsExhausted: (stepState: StepState) => boolean;
 /**
  * Decide whether this dispatch owns the step, by reading its state and then
  * writing it — which only excludes a concurrent dispatch when the caller holds
@@ -13,4 +25,4 @@ export type StepClaimStore = {
  * A store that can express the whole decision as a single conditional write
  * should do that instead of calling this.
  */
-export declare const claimStepByReadThenWrite: (store: StepClaimStore, runId: string, stepName: string, rpcName: string) => Promise<StepState | null>;
+export declare const claimStepByReadThenWrite: (store: StepClaimStore, runId: string, stepName: string, rpcName: string, leaseMs: number) => Promise<StepState | null>;

package/dist/wirings/workflow/workflow-step-claim.js

@@ -1,4 +1,10 @@
-import { WorkflowStepFunctionMismatchError } from './workflow-errors.js';
+import { DEFAULT_STEP_RETRIES, isStepLeaseLive } from './workflow-constants.js';
+import { WorkflowStepFunctionMismatchError, WorkflowStepLeaseExpiredError, } from './workflow-errors.js';
+/**
+ * Whether a step whose worker vanished may be handed to another one, or has
+ * run out of attempts to spend on that.
+ */
+export const leaseAttemptsExhausted = (stepState) => stepState.attemptCount >= (stepState.retries ?? DEFAULT_STEP_RETRIES) + 1;
 /**
  * Decide whether this dispatch owns the step, by reading its state and then
  * writing it — which only excludes a concurrent dispatch when the caller holds
@@ -7,21 +13,39 @@ import { WorkflowStepFunctionMismatchError } from './workflow-errors.js';
  * A store that can express the whole decision as a single conditional write
  * should do that instead of calling this.
  */
-export const claimStepByReadThenWrite = async (store, runId, stepName, rpcName) => {
+export const claimStepByReadThenWrite = async (store, runId, stepName, rpcName, leaseMs) => {
     const stepState = await store.getStepState(runId, stepName);
     // knowledge: decisions/security/a-step-runs-the-function-the-workflow-dispatched-it-with.md
     if (stepState.rpcName !== undefined &&
         stepState.rpcName !== (rpcName ?? null)) {
         throw new WorkflowStepFunctionMismatchError(runId, stepName);
     }
-    if (stepState.status === 'succeeded' || stepState.status === 'running') {
+    if (stepState.status === 'succeeded') {
+        return null;
+    }
+    if (stepState.status === 'running') {
+        if (isStepLeaseLive(stepState.leaseExpiresAt)) {
+            return null;
+        }
+        if (leaseAttemptsExhausted(stepState)) {
+            await store.setStepError(stepState.stepId, new WorkflowStepLeaseExpiredError(runId, stepName, stepState.attemptCount), stepState.attemptCount);
+            await store.resumeWorkflow?.(runId);
+            return null;
+        }
+    }
+    // A failed step that has spent every attempt is settled, however it failed;
+    // a redelivered message must not buy it another one.
+    if (stepState.status === 'failed' && leaseAttemptsExhausted(stepState)) {
         return null;
     }
-    if (stepState.status === 'failed') {
-        return store.createRetryAttempt(stepState.stepId, 'running');
+    if (stepState.status === 'failed' || stepState.status === 'running') {
+        const attempt = await store.createRetryAttempt(stepState.stepId, 'running');
+        await store.refreshStepLease(attempt.stepId, leaseMs);
+        return attempt;
     }
     if (stepState.status === 'pending' || stepState.status === 'scheduled') {
         await store.setStepRunning(stepState.stepId);
+        await store.refreshStepLease(stepState.stepId, leaseMs);
     }
     return stepState;
 };

package/dist/wirings/workflow/workflow-step-lease.d.ts

@@ -0,0 +1,25 @@
+import type { StepState } from './workflow.types.js';
+/**
+ * How long a claim on a step dispatched through `queueName` is good for, taken
+ * from that queue's own lock so the two never disagree about who owns the job.
+ */
+export declare const stepLeaseMsForQueue: (queueName: string) => number;
+/**
+ * Keep a dispatch's claim alive for as long as it is working, and report how to
+ * stop once it is not. It renews on the same loop as `holdLease`.
+ *
+ * Stopping waits for a refresh already in flight, so a caller that releases the
+ * lease after stopping cannot have that release overwritten by a late renewal.
+ *
+ * A refresh that resolves `false` means another dispatch has claimed the step
+ * since: renewing stops there, and says so, because the step's outcome will
+ * now be refused rather than recorded.
+ */
+export declare const startStepLeaseRefresh: (stepId: string, leaseMs: number, refresh: () => Promise<boolean>) => (() => Promise<void>);
+/**
+ * Who a `running` step belongs to, for a resumed run that meets it. `held` is a
+ * dispatch still working it. `lapsed` is one that died: the step is dispatched
+ * again but left `running`, so the claim counts it as another attempt rather
+ * than a first run — a step that kills its worker every time then runs out.
+ */
+export declare const runningStepLease: (stepState: Pick<StepState, 'status' | 'leaseExpiresAt'>) => 'held' | 'lapsed' | undefined;

package/dist/wirings/workflow/workflow-step-lease.js

@@ -0,0 +1,53 @@
+import { getSingletonServices, pikkuState } from '../../pikku-state.js';
+import { DEFAULT_STEP_LEASE_MS, STEP_LEASE_REFRESH_MIN_MS, isStepLeaseLive, } from './workflow-constants.js';
+import { keepLeaseAlive, leaseRenewalIntervalMs, RENEWALS_PER_LEASE, } from '../../services/lease-service.js';
+/**
+ * How long a claim on a step dispatched through `queueName` is good for, taken
+ * from that queue's own lock so the two never disagree about who owns the job.
+ */
+export const stepLeaseMsForQueue = (queueName) => {
+    const worker = pikkuState(null, 'queue', 'registrations').get(queueName);
+    const { lockDuration, visibilityTimeout } = worker?.config ?? {};
+    if (lockDuration !== undefined) {
+        return lockDuration;
+    }
+    if (visibilityTimeout !== undefined) {
+        return visibilityTimeout * 1000;
+    }
+    return DEFAULT_STEP_LEASE_MS;
+};
+/**
+ * Keep a dispatch's claim alive for as long as it is working, and report how to
+ * stop once it is not. It renews on the same loop as `holdLease`.
+ *
+ * Stopping waits for a refresh already in flight, so a caller that releases the
+ * lease after stopping cannot have that release overwritten by a late renewal.
+ *
+ * A refresh that resolves `false` means another dispatch has claimed the step
+ * since: renewing stops there, and says so, because the step's outcome will
+ * now be refused rather than recorded.
+ */
+export const startStepLeaseRefresh = (stepId, leaseMs, refresh) => {
+    const interval = leaseRenewalIntervalMs(leaseMs);
+    if (interval < STEP_LEASE_REFRESH_MIN_MS) {
+        getSingletonServices()?.logger?.warn(`Workflow step ${stepId}: a ${leaseMs}ms lease is refreshed every ${interval}ms. Raise the queue's lockDuration or visibilityTimeout above ${STEP_LEASE_REFRESH_MIN_MS * RENEWALS_PER_LEASE}ms.`);
+    }
+    const renewal = keepLeaseAlive(`workflow-step:${stepId}`, leaseMs, () => refresh().catch((error) => {
+        getSingletonServices()?.logger?.warn(`Workflow step ${stepId}: could not refresh its lease; another worker may take the step`, error);
+        throw error;
+    }));
+    renewal.signal.addEventListener('abort', () => getSingletonServices()?.logger?.warn(`Workflow step ${stepId}: lost its lease, so another dispatch may claim it; this one stops renewing and its outcome will be refused if it does`));
+    return renewal.stop;
+};
+/**
+ * Who a `running` step belongs to, for a resumed run that meets it. `held` is a
+ * dispatch still working it. `lapsed` is one that died: the step is dispatched
+ * again but left `running`, so the claim counts it as another attempt rather
+ * than a first run — a step that kills its worker every time then runs out.
+ */
+export const runningStepLease = (stepState) => {
+    if (stepState.status !== 'running' || stepState.leaseExpiresAt == null) {
+        return undefined;
+    }
+    return isStepLeaseLive(stepState.leaseExpiresAt) ? 'held' : 'lapsed';
+};

package/dist/wirings/workflow/workflow-step-retry.d.ts

@@ -0,0 +1,14 @@
+import type { StepState, WorkflowStepOptions } from './workflow.types.js';
+/** What running a step's retries in-process needs from the workflow service. */
+export type StepRetryStore = {
+    setStepRunning(stepId: string): Promise<void>;
+    setStepResult(stepId: string, result: any): Promise<void>;
+    setStepError(stepId: string, error: Error): Promise<void>;
+    createRetryAttempt(failedStepId: string, status: 'pending' | 'running'): Promise<StepState>;
+};
+/**
+ * Spend a step's retry budget here and now, rather than by handing the run back
+ * to the orchestrator between attempts. For an inline run there is nothing to
+ * hand it back to.
+ */
+export declare const runInlineRetryLoop: (store: StepRetryStore, stepState: StepState, retries: number, retryDelay: WorkflowStepOptions['retryDelay'], doWork: (currentStepState: StepState) => Promise<any>, onError?: (error: any) => Promise<void>) => Promise<any>;

package/dist/wirings/workflow/workflow-step-retry.js

@@ -0,0 +1,33 @@
+import { getDurationInMilliseconds } from '../../time-utils.js';
+/**
+ * Spend a step's retry budget here and now, rather than by handing the run back
+ * to the orchestrator between attempts. For an inline run there is nothing to
+ * hand it back to.
+ */
+export const runInlineRetryLoop = async (store, stepState, retries, retryDelay, doWork, onError) => {
+    let currentStepState = stepState;
+    while (true) {
+        let result;
+        try {
+            await store.setStepRunning(currentStepState.stepId);
+            result = await doWork(currentStepState);
+        }
+        catch (error) {
+            if (onError)
+                await onError(error);
+            await store.setStepError(currentStepState.stepId, error);
+            if (currentStepState.attemptCount >= retries) {
+                throw error;
+            }
+            currentStepState = await store.createRetryAttempt(currentStepState.stepId, 'pending');
+            if (retryDelay) {
+                await new Promise((resolve) => setTimeout(resolve, getDurationInMilliseconds(retryDelay)));
+            }
+            continue;
+        }
+        // Outside the try: the work is done, so failing to record it must not
+        // count as a failed attempt and run the work again.
+        await store.setStepResult(currentStepState.stepId, result);
+        return result;
+    }
+};

package/dist/wirings/workflow/workflow-version-fallback.d.ts

@@ -0,0 +1,11 @@
+import type { PikkuRPC } from '../rpc/rpc-types.js';
+import type { PikkuWorkflowService } from './pikku-workflow-service.js';
+import type { WorkflowRun } from './workflow.types.js';
+/**
+ * Resume a run whose workflow changed since it started. A graph workflow
+ * resumes from the version it started on; a complex one, whose inline steps
+ * cannot be replayed against a different definition, fails.
+ */
+export declare const runVersionMismatchFallback: (service: PikkuWorkflowService, run: WorkflowRun, currentMeta: {
+    source: string;
+}, rpcService: PikkuRPC) => Promise<void>;

package/dist/wirings/workflow/workflow-version-fallback.js

@@ -0,0 +1,30 @@
+import { runFromMeta } from './graph/graph-runner.js';
+/**
+ * Resume a run whose workflow changed since it started. A graph workflow
+ * resumes from the version it started on; a complex one, whose inline steps
+ * cannot be replayed against a different definition, fails.
+ */
+export const runVersionMismatchFallback = async (service, run, currentMeta, rpcService) => {
+    // What can be resumed depends on the version the run started on, not on what
+    // the workflow has since become. The current source only stands in when that
+    // version was never stored.
+    const version = await service.getWorkflowVersion(run.workflow, run.graphHash);
+    const source = version?.source ?? currentMeta.source;
+    if (source === 'complex') {
+        await service.updateRunStatus(run.id, 'failed', undefined, {
+            message: `Workflow '${run.workflow}' definition changed. Complex workflows with inline steps cannot be migrated.`,
+            stack: '',
+            code: 'VERSION_CONFLICT',
+        });
+        return;
+    }
+    if (!version) {
+        await service.updateRunStatus(run.id, 'failed', undefined, {
+            message: `Workflow '${run.workflow}' version '${run.graphHash}' not found. Cannot resume with changed definition.`,
+            stack: '',
+            code: 'VERSION_NOT_FOUND',
+        });
+        return;
+    }
+    await runFromMeta(service, run.id, version.graph, rpcService);
+};

package/dist/wirings/workflow/workflow.types.d.ts

@@ -2,6 +2,7 @@ import type { CommonWireMeta } from '../../types/core.types.js';
 import type { SerializedError } from '../../errors/serialized-error.js';
 import type { CorePikkuFunctionConfig } from '../../function/functions.types.js';
 import type { GroupConcurrencyConfig } from '../queue/queue.types.js';
+import type { LeaseService } from '../../services/lease-service.js';
 export type { WorkflowService } from '../../services/workflow-service.js';
 export type { WorkflowStepOptions, WorkflowWireDoRPC, WorkflowApprovalOptions, ApprovalOutcome, InputSource, OutputBinding, RpcStepMeta, Condition, BranchStepMeta, ParallelGroupStepMeta, FanoutStepMeta, ReturnStepMeta, InlineStepMeta, SleepStepMeta, CancelStepMeta, SuspendStepMeta, ApprovalStepMeta, SetStepMeta, SwitchCaseMeta, SwitchStepMeta, FilterStepMeta, ArrayPredicateStepMeta, WorkflowStepMeta, WorkflowStepWire, PikkuWorkflowWire, } from './dsl/workflow-dsl.types.js';
 import type { WorkflowStepMeta } from './dsl/workflow-dsl.types.js';
@@ -24,6 +25,15 @@ export interface WorkflowQueueOptions {
     queueConcurrency?: number;
     queueGroupConcurrency?: number | GroupConcurrencyConfig;
 }
+/**
+ * What a workflow service over a shared store is built with. Every process that
+ * can reach the store may orchestrate the same run or claim the same step, so
+ * the service serialises both on `leaseService` — the same one the app
+ * registers, whatever store it is backed by.
+ */
+export interface WorkflowServiceOptions extends WorkflowQueueOptions {
+    leaseService: LeaseService;
+}
 export interface WorkflowPlannedStep {
     stepName: string;
     displayName?: string;
@@ -65,6 +75,13 @@ export interface StepState {
     createdAt: Date;
     updatedAt: Date;
     childRunId?: string;
+    /**
+     * When the dispatch that claimed this step stops owning it. The holder pushes
+     * it forward while it is still working, so a lapsed lease means the worker is
+     * gone and the step may be claimed again. `undefined` is a store that keeps no
+     * lease, where a `running` step is owned until it moves on its own.
+     */
+    leaseExpiresAt?: Date;
     runningAt?: Date;
     scheduledAt?: Date;
     succeededAt?: Date;

package/knowledge/decisions/internals/a-held-run-is-woken-later-not-retried.md

@@ -0,0 +1,28 @@
+---
+type: decision
+title: A held run is woken later, not retried
+description: An orchestrator message that finds its run held enqueues a fresh wake-up a second later instead of failing, because the queue's retry budget is for real failures and a long pass outlasts it
+tags: [workflows, leases, queues]
+---
+
+# A held run is woken later, not retried
+
+Every message on the orchestrator queue carries news the run has to see — a step
+finished, a child ended — so a message that finds the run held by another pass
+cannot simply be dropped. Throwing it back to the queue looks equivalent and is
+not: it spends the retries the queue keeps for genuine failures. pg-boss gives an
+orchestrator message six attempts on an exponential backoff, the last about 55
+seconds in; a pass that holds the run longer than that leaves the message dead
+and the run `running` forever.
+
+So a `LeaseTakenError` or `LeaseLostError` enqueues a new orchestrator message
+with a one-second delay (`RUN_LEASE_RETRY_MS`) and the current message succeeds.
+`verifiers/workflows/src/runners/run-lease.runner.ts` holds a run's lease for 90
+seconds and checks the run still completes.
+
+The same errors never fail an inline run: its body already wrote its outcome,
+and losing the lease afterwards says only that the outcome was not produced
+alone — the run keeps the status it wrote.
+
+**What this rules out:** treating a taken run lease as a queue failure, and
+marking a run failed because its lease was lost.

package/knowledge/decisions/internals/index.md

@@ -11,6 +11,7 @@ caller is entitled to assume.
 
 <!-- pikku:knowledge-index -->
 
+- [A held run is woken later, not retried](a-held-run-is-woken-later-not-retried.md) — An orchestrator message that finds its run held enqueues a fresh wake-up a second later instead of failing, because the queue's retry budget is for real failures and a long pass outlasts it
 - [A non-streaming agent run registers with aiRunState on the same terms as a streaming one](a-non-streaming-agent-run-registers-with-airunstate-too.md) — Otherwise interruptAIAgent finds the run, passes the ownership check, then cannot stop it — and reports that as if the run were on another host
 - [With scenarios.reset on, the dev seed is the fixture every assertion is written against](a-reset-suite-makes-the-dev-seed-the-fixture.md) — The rollback restores the database as the suite found it, so seed rows are the shared baseline — which makes an absolute date in the seed a test that expires
 - [A resumed turn is as interruptible as the first one](a-resumed-agent-turn-is-as-interruptible-as-the-first.md) — It is the same person listening to the same voice, and after an approval it is where most of the reply actually gets spoken
@@ -112,7 +113,7 @@ caller is entitled to assume.
 - [The dev queue copies prod timing and serialization semantics](the-dev-queue-copies-prod-timing-and-serialization-semantics.md) — InMemoryQueueService dispatches via setTimeout, retries with backoff, and JSON round-trips every payload so dev behaviour matches a real backend
 - [One door per name — the ecosystem tier and the package root are both gone](the-ecosystem-entry-point-carries-the-adapter-surface.md) — the adapter surface was split onto @pikku/core/ecosystem to keep a stability promise at the root; publishing every module twice cost more than the promise was worth, so both the facades and the root barrel were deleted
 - [The embedding model is pinned per service and doc/query embedding is split](the-embedding-model-is-pinned-per-service-and-doc-query-embedding-is-split.md) — AIEmbeddingService fixes its model at construction so index and query share a vector space, and separates embedDocuments from embedQuery for asymmetric models
-- [The in-memory workflow service is inline-only and single-process](the-in-memory-workflow-service-is-inline-only-and-single-process.md) — InMemoryWorkflowService wires no queues and implements withRunLock/withStepLock as pass-throughs, because inline execution has no second holder to exclude
+- [The in-memory workflow service is inline-only and single-process](the-in-memory-workflow-service-is-inline-only-and-single-process.md) — InMemoryWorkflowService wires no queues and implements withRunLease/withStepLock as pass-throughs, because inline execution has no second holder to exclude
 - [The KEK salt is scoped to the key version, not the secret](the-kek-salt-is-scoped-to-the-key-version.md) — One stored salt per key version means N secrets cost one derivation, which is the point of envelope encryption
 - [The MCP handshake is challenged only when every target is gated](the-mcp-handshake-is-challenged-only-when-every-target-is-gated.md) — A client decides whether a server speaks OAuth from the handshake alone, so a fully-gated server that answers initialize with a 200 is detected as needing no sign-in
 - [The middleware resolution cache is deliberately unbounded](the-middleware-resolution-cache-is-deliberately-unbounded.md) — Its keyspace is the set of registered wires, not request traffic, and middleware is dynamic — so eviction would buy nothing and cost the dedupe guarantee

package/knowledge/decisions/internals/the-in-memory-workflow-service-is-inline-only-and-single-process.md

@@ -1,14 +1,14 @@
 ---
 type: decision
 title: The in-memory workflow service is inline-only and single-process
-description: InMemoryWorkflowService wires no queues and implements withRunLock/withStepLock as pass-throughs, because inline execution has no second holder to exclude
+description: InMemoryWorkflowService wires no queues and implements withRunLease/withStepLock as pass-throughs, because inline execution has no second holder to exclude
 tags: services
 ---
 
 # The in-memory workflow service is inline-only and single-process
 
 `InMemoryWorkflowService` (`packages/core/src/services/in-memory-workflow-service.ts`)
-calls `super({ ...options, wireQueues: false })` and implements `withRunLock` and
+calls `super({ ...options, wireQueues: false })` and implements `withRunLease` and
 `withStepLock` as bare `return fn()`. Both look like unfinished work and neither
 is.
 

package/knowledge/decisions/internals/workflow-step-lock-is-held-only-to-claim-the-step.md

@@ -1,7 +1,7 @@
 ---
 type: decision
 title: A workflow step lock is held only to claim the step, never across its execution
-description: Holding the advisory lock — and its pooled connection — across step work exhausted the connection pool and self-deadlocked
+description: Holding the step lock across step work exhausted the connection pool and self-deadlocked when it was an advisory lock; it stays claim-only on leases
 tags: workflow
 ---
 
@@ -15,10 +15,17 @@ lock is then released, and the actual work plus result persistence run outside
 it.
 
 The guard is what makes that safe — once a step is `running`, any concurrent
-worker returns early. The alternative was tried and failed: holding the advisory
-lock, and therefore its pooled connection, across `executeGraphStep` (network
-I/O plus further pool queries) let concurrent steps exhaust the connection pool
-and self-deadlock.
+worker returns early. The alternative was tried and failed: holding the
+then-advisory lock, and therefore its pooled connection, across
+`executeGraphStep` (network I/O plus further pool queries) let concurrent steps
+exhaust the connection pool and self-deadlock.
+
+`withStepLock` is now a lease on the app's `leaseService`
+(`workflow-step:<runId>:<step>`), so it pins no connection. It still stays
+claim-only: a lease held across the work would have to outlive it, and a taken
+step lease already means "another dispatch owns it", so the claim answers
+`null`. Kysely and MongoDB claim with a status-guarded conditional update and do
+not take it at all.
 
 **What this rules out:** widening the `withStepLock` callback to cover RPC
 invocation, child-workflow start, `setStepResult` or `resumeWorkflow` — the

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pikku/core",
-  "version": "0.12.130",
+  "version": "0.12.134",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/pikkujs/pikku.git",