bunqueue 2.8.45 → 2.8.47

package/dist/client/workflow/loops.js

@@ -3,6 +3,99 @@
  * Handles doUntil, doWhile, forEach, and map node types.
  */
 import { executeStepWithRetry, buildContext } from './runner';
+import { clock } from './clock';
+/**
+ * Run one iteration of a loop body step, unless it already ran.
+ *
+ * Two properties depend on this, and both were broken:
+ *
+ * TRANSCRIPT. `exec.steps[name]` is overwritten every iteration, so on its own a
+ * loop keeps a sliding window of exactly one turn — by turn 3 the transcript of
+ * turn 1 is gone and an agent is asked to reason without its own history
+ * (test/workflow-ai-sdk-agent.test.ts measured prompt sizes 1, 3, 3 where they must
+ * be 1, 3, 5). Each iteration is therefore also kept under `name:iteration`.
+ *
+ * RESUME. A loop is a single job, so re-entering the node used to replay every
+ * iteration from zero: an agent killed at turn 7 re-paid for turns 1-6, and the
+ * restarted in-memory counter overwrote `turn:0` and `turn:1` with turn 2's content,
+ * corrupting the very transcript above. Memoising against the persisted per-iteration
+ * record fixes both at once — the counter realigns naturally, because iteration 0
+ * finds `turn:0` already complete and skips it.
+ *
+ * Only `completed` is skipped. An iteration left `running` by a crash is genuinely
+ * unfinished and must run again.
+ *
+ * The base name keeps holding the LAST iteration, which is the documented contract
+ * for downstream steps.
+ *
+ * The indexed copies ARE the unwind set: `findStepDef` resolves `turn:0` through a
+ * second, iteration-only pass, and `unwindSet` excludes the bare `turn` mirror so the
+ * last iteration is not compensated twice. An earlier version of this comment said
+ * the opposite, which was true only while a loop compensated its last iteration
+ * alone.
+ */
+async function runIteration(step, exec, iteration, emitter, updateFn) {
+    const indexed = `${step.name}:${iteration}`;
+    const already = exec.steps[indexed];
+    if (already?.status === 'completed') {
+        // Restore the base name so the loop condition and downstream steps see this
+        // iteration's result without the body running a second time.
+        exec.steps[step.name] = { ...already };
+        updateFn(exec);
+        return;
+    }
+    let thrown;
+    let threw = false;
+    try {
+        await executeStepWithRetry(step, buildContext(exec), exec, { emitter, updateFn }, iteration);
+    }
+    catch (error) {
+        // Caught rather than left to a `finally`, because the mirror write below must not be
+        // able to REPLACE this error. An exception thrown from a `finally` supersedes the one
+        // in flight, so a store that refused the mirror write turned "provider timeout after
+        // the charge settled" into "SQLITE_BUSY": that message is what `failureReason`
+        // records, so the operator lost the only account of what actually went wrong
+        // (`test/repro-workflow-loop-failed-iteration.test.ts`).
+        thrown = error;
+        threw = true;
+    }
+    {
+        // Writing the mirror on BOTH paths is load-bearing, for the same reason
+        // `executeForEach` writes its own in a `finally`.
+        //
+        // The indexed copies ARE the unwind set, and `unwindSet` drops the bare mirror
+        // whenever a `${name}:0` sibling exists, because that mirror is the LAST iteration
+        // and compensating it as well would undo that iteration twice. Written only on
+        // success, the iteration that THREW existed under the bare name alone, so the two
+        // rules combined to lose it: a `doUntil` that charged on every turn and failed on
+        // turn 2 refunded turns 0 and 1, left turn 2's charge standing, and still reported
+        // `rollbackStatus: 'completed'`. That record was unreachable even by
+        // `abandonCompensation`, which walks the same set
+        // (`test/repro-workflow-loop-failed-iteration.test.ts`).
+        //
+        // A failed iteration is the one MOST likely to need undoing: a charge that reached
+        // the provider and then lost its response is recorded failed while the money has
+        // already moved.
+        const record = exec.steps[step.name];
+        if (record) {
+            // The in-memory copy is what the unwind reads, so it happens regardless. Only the
+            // PERSIST can fail, and when the step itself failed, that step's error is the cause
+            // and a write that fails while reporting it is downstream. With no step error to
+            // protect, a write that cannot be persisted is the failure and must surface: an
+            // iteration whose record never reached disk is one whose reversal runs again.
+            exec.steps[indexed] = { ...record };
+            try {
+                updateFn(exec);
+            }
+            catch (writeError) {
+                if (!threw)
+                    throw writeError;
+            }
+        }
+    }
+    if (threw)
+        throw thrown;
+}
 /** Execute a doUntil loop: run steps, then check condition. Repeat until condition returns true. */
 export async function executeDoUntil(def, exec, emitter, updateFn) {
     let iteration = 0;
@@ -12,8 +105,7 @@ export async function executeDoUntil(def, exec, emitter, updateFn) {
             throw new Error(`doUntil exceeded maxIterations (${def.maxIterations})`);
         }
         for (const step of def.steps) {
-            const ctx = buildContext(exec);
-            await executeStepWithRetry(step, ctx, exec, emitter, updateFn);
+            await runIteration(step, exec, iteration, emitter, updateFn);
         }
         iteration++;
         const ctx = buildContext(exec);
@@ -31,8 +123,7 @@ export async function executeDoWhile(def, exec, emitter, updateFn) {
             throw new Error(`doWhile exceeded maxIterations (${def.maxIterations})`);
         }
         for (const step of def.steps) {
-            const stepCtx = buildContext(exec);
-            await executeStepWithRetry(step, stepCtx, exec, emitter, updateFn);
+            await runIteration(step, exec, iteration, emitter, updateFn);
         }
     }
 }
@@ -40,6 +131,16 @@ export async function executeDoWhile(def, exec, emitter, updateFn) {
 export async function executeForEach(def, exec, emitter, updateFn) {
     const ctx = buildContext(exec);
     const items = def.items(ctx);
+    // Anything with a `length` used to be accepted, and JavaScript is generous about
+    // what has one. A number iterated ZERO times and the run reported `completed`, so a
+    // batch that processed nothing was indistinguishable from a batch with nothing to
+    // do. A string iterated its CHARACTERS, so an id list that arrived as `'u1,u2'`
+    // silently processed five "items" nobody passed. `null`/`undefined` were caught only
+    // by accident, because reading `.length` throws. The likely production shapes were
+    // exactly the ones that passed (`test/repro-workflow-foreach-non-array.test.ts`).
+    if (!Array.isArray(items)) {
+        throw new Error(`forEach items must be an array, got ${items === null ? 'null' : typeof items}`);
+    }
     if (items.length > def.maxIterations) {
         throw new Error(`forEach items (${items.length}) exceeds maxIterations (${def.maxIterations})`);
     }
@@ -57,15 +158,47 @@ export async function executeForEach(def, exec, emitter, updateFn) {
                 return def.step.handler(enrichedCtx);
             },
         };
+        // Memoised the same way as doUntil/doWhile: an item already provisioned before a
+        // crash must not be provisioned again when the node is re-entered.
+        if (exec.steps[indexedName]?.status === 'completed')
+            continue;
         const stepCtx = buildContext(exec);
-        await executeStepWithRetry(indexedStep, stepCtx, exec, emitter, updateFn);
-        // Persist this iteration's __item/__index alongside the step record so
-        // saga compensation can restore the correct per-iteration context later.
-        // executeStepWithRetry has just written this record (it throws otherwise).
-        const record = exec.steps[indexedName];
-        record.loopItem = item;
-        record.loopIndex = i;
-        updateFn(exec);
+        let thrown;
+        let threw = false;
+        try {
+            await executeStepWithRetry(indexedStep, stepCtx, exec, { emitter, updateFn }, i);
+        }
+        catch (error) {
+            // Same reason `runIteration` catches instead of using a `finally`: a write that
+            // fails below must not REPLACE the step's own error, which is what `failureReason`
+            // records and the only account of what went wrong.
+            thrown = error;
+            threw = true;
+        }
+        {
+            // Persist this iteration's __item/__index alongside the step record so saga
+            // compensation can restore the correct per-iteration context later.
+            //
+            // Recording on BOTH paths is load-bearing: the iteration that THROWS is itself
+            // eligible for compensation, and without its item recorded its compensate handler
+            // is handed an undefined `__item` and silently releases nothing — the reservation
+            // it had already taken at the warehouse leaks
+            // (test/workflow-saga-extreme.test.ts).
+            const record = exec.steps[indexedName];
+            if (record) {
+                record.loopItem = item;
+                record.loopIndex = i;
+                try {
+                    updateFn(exec);
+                }
+                catch (writeError) {
+                    if (!threw)
+                        throw writeError;
+                }
+            }
+        }
+        if (threw)
+            throw thrown;
     }
 }
 /** Execute a map node: transform step results into a new value */
@@ -76,8 +209,8 @@ export async function executeMap(def, exec, emitter, updateFn) {
     exec.steps[def.name] = {
         status: 'completed',
         result,
-        startedAt: Date.now(),
-        completedAt: Date.now(),
+        startedAt: clock().now(),
+        completedAt: clock().now(),
     };
     updateFn(exec);
     emitter?.emitStep('step:completed', exec.id, exec.workflowName, def.name, { result });

package/dist/client/workflow/recovery.d.ts

@@ -11,12 +11,21 @@ import type { Workflow } from './workflow';
 import type { WorkflowStore } from './store';
 import type { WorkflowEmitter } from './emitter';
 import type { RecoverResult } from './types';
+import { type TimerHandle } from './clock';
 export interface RecoverDeps {
     store: WorkflowStore;
     queue: Queue;
     workflows: Map<string, Workflow>;
     emitter: WorkflowEmitter | null;
-    timeoutTimers: Map<string, ReturnType<typeof setTimeout>>;
+    timeoutTimers: Map<string, TimerHandle>;
     scheduleTimeoutCheck: (id: string, wfName: string, nodeIdx: number, ms: number) => void;
+    /**
+     * Nodes this engine is executing right now. Recovery consults it for the same reason
+     * the `compensating` branch consults the unwind claim: re-enqueueing a node that is
+     * already in flight produces a job the admission check will reject, and counting it
+     * as a recovery reports work that did not happen. A step running without a bound is
+     * the ordinary way to reach this.
+     */
+    nodesInFlight: ReadonlySet<string>;
 }
 export declare function recoverExecutions(deps: RecoverDeps): Promise<RecoverResult>;

package/dist/client/workflow/recovery.js

@@ -7,25 +7,47 @@
  * - 'compensating': re-run compensation from the beginning (must be idempotent)
  */
 import { runCompensation } from './compensator';
+import { hasSignal } from './storeSignals';
+import { clock } from './clock';
+import { decideAdmission } from './admission';
 export async function recoverExecutions(deps) {
     const { store, workflows } = deps;
     const executions = store.listRecoverable();
     const result = { running: 0, waiting: 0, compensating: 0, total: 0 };
-    for (const exec of executions) {
-        const wf = workflows.get(exec.workflowName);
+    for (const snapshot of executions) {
+        const wf = workflows.get(snapshot.workflowName);
         if (!wf)
             continue;
+        // Re-read before driving it. The list above is a single snapshot taken before the
+        // loop, and each iteration awaits: driving a parent runs its sub-workflow's unwind
+        // and settles the CHILD's records in the store, but this loop still holds the
+        // child's pre-unwind copy. Using that copy, every `if (record.compensation)`
+        // guard reads an unsettled snapshot and dispatches the same handlers a second
+        // time, issuing a duplicate refund and leaving no trace: both rows end `failed`
+        // with `rollbackStatus: 'completed'`. The in-flight `inFlight` claim does not
+        // help here, because the two unwinds are sequential, not concurrent.
+        const exec = store.get(snapshot.id);
+        if (!exec)
+            continue;
         if (exec.state === 'running') {
-            await enqueueExecution(exec, deps.queue);
-            result.running++;
+            const admission = decideAdmission(exec, exec.currentNodeIndex, deps.nodesInFlight);
+            // Only `already-in-flight` is reachable here: the state is `running` by the
+            // branch and the index is the execution's own cursor.
+            if (admission.kind === 'run') {
+                await enqueueExecution(exec, deps.queue);
+                result.running++;
+            }
         }
         else if (exec.state === 'waiting') {
             await recoverWaiting(exec, wf, deps);
             result.waiting++;
         }
         else if (exec.state === 'compensating') {
-            await runCompensation(exec, wf, store, deps.emitter);
-            result.compensating++;
+            // A lost claim means another driver owns this unwind, so nothing happened here
+            // and counting it as a recovery would overstate what recover() did.
+            const outcome = await runCompensation(exec, wf, store, deps.emitter, deps.workflows);
+            if (outcome === 'ran')
+                result.compensating++;
         }
     }
     result.total = result.running + result.waiting + result.compensating;
@@ -37,6 +59,10 @@ async function enqueueExecution(exec, queue) {
         workflowName: exec.workflowName,
         nodeIndex: exec.currentNodeIndex,
     };
+    // No deterministic jobId, for the reason documented in WorkflowExecutor.enqueue:
+    // the duplicate this path can create is neutralised by the cursor guard in
+    // processStep, and dedup here risks swallowing a legitimate re-enqueue after a
+    // restart, which wedges the run for good.
     await queue.add('wf:step', jobData);
 }
 async function recoverWaiting(exec, wf, deps) {
@@ -45,8 +71,9 @@ async function recoverWaiting(exec, wf, deps) {
         await enqueueExecution(exec, deps.queue);
         return;
     }
-    // Check if signal already arrived while we were down
-    if (exec.signals[node.event] !== undefined) {
+    // Check if signal already arrived while we were down. Key presence, not value:
+    // a payload-less signal records the key with an `undefined` value (see hasSignal).
+    if (hasSignal(exec.signals, node.event)) {
         exec.state = 'running';
         deps.store.update(exec);
         await enqueueExecution(exec, deps.queue);
@@ -59,7 +86,7 @@ async function recoverWaiting(exec, wf, deps) {
         const waitKey = `__waitFor:${node.event}`;
         const waitRecord = exec.steps[waitKey];
         const waitingSince = waitRecord?.startedAt ?? exec.updatedAt;
-        const elapsed = Date.now() - waitingSince;
+        const elapsed = clock().now() - waitingSince;
         const remaining = node.timeout - elapsed;
         if (remaining <= 0) {
             exec.state = 'running';

package/dist/client/workflow/rollbackControl.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Operator actions on a parked unwind.
+ *
+ * A `compensation-stuck` run is waiting for a human decision, which is why the state
+ * is not terminal: the engine has done everything it safely can, and the remaining
+ * choice, was the reversal fixed or do we accept a partial rollback, is not one it can
+ * make. Both exits are explicit, and both leave the execution record saying exactly
+ * what happened.
+ */
+import type { Workflow } from './workflow';
+import type { WorkflowStore } from './store';
+import type { WorkflowEmitter } from './emitter';
+export interface RollbackDeps {
+    store: WorkflowStore;
+    emitter: WorkflowEmitter | null;
+    workflows: Map<string, Workflow>;
+}
+/**
+ * Retry the compensation that parked the run, then carry on with the rest.
+ *
+ * The retry is asked for with a FLAG rather than by clearing the failed outcome first.
+ * Clearing worked, and it cost more than it bought: it wiped the operator's
+ * `compensation-failed` record in memory and persisted that wipe BEFORE running
+ * anything, so a resume that then met a failing store left a run whose durable row had
+ * lost the very diagnostic the operator was acting on, sitting in `compensating` and
+ * therefore handed back by `listRecoverable()` at every subsequent startup. Guarding
+ * that needed a deep snapshot, a restore path, and a second write which could itself
+ * throw and mask the original error.
+ *
+ * None of that machinery is needed if nothing is destroyed in the first place. The
+ * record stays until a real outcome replaces it, and a store that fails simply leaves
+ * the parked run exactly as the operator found it.
+ */
+export declare function resumeCompensation(deps: RollbackDeps, executionId: string): Promise<void>;
+/** Accept a partial rollback: the outstanding steps are recorded as skipped. */
+export declare function abandonParkedCompensation(deps: RollbackDeps, executionId: string): void;

package/dist/client/workflow/rollbackControl.js

@@ -0,0 +1,51 @@
+/**
+ * Operator actions on a parked unwind.
+ *
+ * A `compensation-stuck` run is waiting for a human decision, which is why the state
+ * is not terminal: the engine has done everything it safely can, and the remaining
+ * choice, was the reversal fixed or do we accept a partial rollback, is not one it can
+ * make. Both exits are explicit, and both leave the execution record saying exactly
+ * what happened.
+ */
+import { abandonCompensation, runCompensation } from './compensator';
+/**
+ * Retry the compensation that parked the run, then carry on with the rest.
+ *
+ * The retry is asked for with a FLAG rather than by clearing the failed outcome first.
+ * Clearing worked, and it cost more than it bought: it wiped the operator's
+ * `compensation-failed` record in memory and persisted that wipe BEFORE running
+ * anything, so a resume that then met a failing store left a run whose durable row had
+ * lost the very diagnostic the operator was acting on, sitting in `compensating` and
+ * therefore handed back by `listRecoverable()` at every subsequent startup. Guarding
+ * that needed a deep snapshot, a restore path, and a second write which could itself
+ * throw and mask the original error.
+ *
+ * None of that machinery is needed if nothing is destroyed in the first place. The
+ * record stays until a real outcome replaces it, and a store that fails simply leaves
+ * the parked run exactly as the operator found it.
+ */
+export async function resumeCompensation(deps, executionId) {
+    const { exec, wf } = parked(deps, executionId);
+    const outcome = await runCompensation(exec, wf, deps.store, deps.emitter, deps.workflows, { retryFailed: true });
+    if (outcome === 'claim-lost') {
+        // Nothing was mutated: `runCompensation` returns this before the unwind begins.
+        throw new Error(`execution "${executionId}" is already being rolled back by another driver`);
+    }
+}
+/** Accept a partial rollback: the outstanding steps are recorded as skipped. */
+export function abandonParkedCompensation(deps, executionId) {
+    const { exec, wf } = parked(deps, executionId);
+    abandonCompensation(exec, wf, deps.store, deps.emitter);
+}
+function parked(deps, executionId) {
+    const exec = deps.store.get(executionId);
+    if (!exec)
+        throw new Error(`Execution "${executionId}" not found`);
+    if (exec.state !== 'compensation-stuck') {
+        throw new Error(`Execution "${executionId}" is "${exec.state}", not a parked unwind ("compensation-stuck")`);
+    }
+    const wf = deps.workflows.get(exec.workflowName);
+    if (!wf)
+        throw new Error(`Workflow "${exec.workflowName}" not registered`);
+    return { exec, wf };
+}

package/dist/client/workflow/runner.d.ts

@@ -6,14 +6,34 @@ import type { Workflow } from './workflow';
 import type { WorkflowEmitter } from './emitter';
 /** Run a promise with a timeout */
 export declare function runWithTimeout<T>(promise: Promise<T> | T, timeoutMs: number): Promise<T>;
+/** Engine hooks a step needs; they always travel together. */
+export interface StepHooks {
+    emitter: WorkflowEmitter | null;
+    updateFn: (exec: Execution) => void;
+}
 /** Execute a step with retry logic and exponential backoff */
-export declare function executeStepWithRetry(def: StepDefinition, ctx: StepContext, exec: Execution, emitter: WorkflowEmitter | null, updateFn: (exec: Execution) => void): Promise<void>;
+export declare function executeStepWithRetry(def: StepDefinition, ctx: StepContext, exec: Execution, hooks: StepHooks, occurrence?: number): Promise<void>;
 /** Execute multiple steps in parallel via Promise.allSettled */
 export declare function executeParallelSteps(steps: StepDefinition[], ctx: StepContext, exec: Execution, emitter: WorkflowEmitter | null, updateFn: (exec: Execution) => void): Promise<void>;
 /** Execute a sub-workflow by starting it and polling for completion */
 export declare function executeSubWorkflow(workflowName: string, input: unknown, startFn: (name: string, input: unknown) => Promise<{
     id: string;
-}>, getFn: (id: string) => Execution | null, pollIntervalMs?: number): Promise<Record<string, unknown>>;
+}>, getFn: (id: string) => Execution | null, pollIntervalMs?: number, 
+/**
+ * A child this node already started, from an earlier entry into the same node.
+ *
+ * Without it the node started a BRAND NEW child every time it was re-entered, and
+ * re-entry is routine: a restart followed by `recover()` re-enqueues the parent's
+ * current node. Measured across one restart, the child ran twice and both rows were
+ * left `running` forever, since a child is excluded from recovery while its parent
+ * exists and `cleanup`/`archive` only reap terminal states. Duplicated work, not just
+ * a leaked row: a child that provisions a resource provisioned it twice
+ * (`test/repro-workflow-orphan-child.test.ts`).
+ */
+existingChildId?: string): Promise<{
+    results: Record<string, unknown>;
+    executionId: string;
+}>;
 /** Find a step definition by name across all node types */
 export declare function findStepDef(wf: Workflow, name: string): StepDefinition | null;
 /** Build a StepContext from the current execution state */