@multiplier-labs/stepflow 0.3.3 → 0.3.4

package/src/core/engine/concurrency.ts

@@ -0,0 +1,229 @@
+/**
+ * Concurrency control for the WorkflowEngine.
+ *
+ * Owns:
+ *   - the priority-ordered queue of pending runs (`runQueue`)
+ *   - the set of in-flight run abort controllers (`activeRuns`)
+ *   - the timer handles used to defer launches via setTimeout/setImmediate
+ *
+ * Responsibilities:
+ *   - `hasCapacity()` against the configured `maxConcurrency` ceiling
+ *   - priority-ordered enqueue (higher priority first, FIFO within a priority)
+ *   - dispatch loop that drains the queue while capacity exists, with a
+ *     try/catch around each dispatch so a single bad run cannot tear down
+ *     the loop and strand the rest of the queue (preserves the audit
+ *     2026-04-27 C3 fix in `processQueue`).
+ *   - dispatch-failure telemetry: log + emit `run.failed` + best-effort
+ *     storage update with `QUEUE_DISPATCH_FAILED`.
+ *
+ * NOT responsible for:
+ *   - building the `executeWorkflow` invocation (that lives in WorkflowEngine
+ *     so the orchestrator/spawnChild wiring stays where it is)
+ *   - storage initialization / shutdown sequencing (see `lifecycle.ts`)
+ */
+
+import type { StorageAdapter } from '../../storage/types';
+import type { EventTransport } from '../../events/types';
+import type { Logger, WorkflowDefinition } from '../types';
+
+/**
+ * Queued run waiting to be executed when capacity is available.
+ */
+export interface QueuedRun {
+  runId: string;
+  definition: WorkflowDefinition;
+  input: Record<string, unknown>;
+  metadata?: Record<string, unknown>;
+  priority: number;
+  queuedAt: Date;
+}
+
+type TimerHandle = ReturnType<typeof setTimeout> | ReturnType<typeof setImmediate>;
+
+export interface ConcurrencyManagerDeps {
+  storage: StorageAdapter;
+  events: EventTransport;
+  logger: Logger;
+  /** Maximum concurrent runs; undefined or <=0 means unlimited. */
+  maxConcurrency?: number;
+}
+
+export class ConcurrencyManager {
+  private readonly activeRuns = new Map<string, AbortController>();
+  private readonly runQueue: QueuedRun[] = [];
+  private readonly timerHandles = new Set<TimerHandle>();
+  private readonly storage: StorageAdapter;
+  private readonly events: EventTransport;
+  private readonly logger: Logger;
+  private readonly maxConcurrency: number | undefined;
+
+  constructor(deps: ConcurrencyManagerDeps) {
+    this.storage = deps.storage;
+    this.events = deps.events;
+    this.logger = deps.logger;
+    this.maxConcurrency = deps.maxConcurrency;
+  }
+
+  hasCapacity(): boolean {
+    if (this.maxConcurrency === undefined || this.maxConcurrency <= 0) {
+      return true;
+    }
+    return this.activeRuns.size < this.maxConcurrency;
+  }
+
+  getActiveCount(): number {
+    return this.activeRuns.size;
+  }
+
+  getQueuedCount(): number {
+    return this.runQueue.length;
+  }
+
+  isActive(runId: string): boolean {
+    return this.activeRuns.has(runId);
+  }
+
+  /** Insert a run in priority order (higher priority first, FIFO within same). */
+  enqueue(run: QueuedRun): void {
+    let inserted = false;
+    for (let i = 0; i < this.runQueue.length; i++) {
+      if (run.priority > this.runQueue[i].priority) {
+        this.runQueue.splice(i, 0, run);
+        inserted = true;
+        break;
+      }
+    }
+    if (!inserted) {
+      this.runQueue.push(run);
+    }
+  }
+
+  /** Remove a run from the queue if present. Returns true if it was queued. */
+  removeFromQueue(runId: string): boolean {
+    const idx = this.runQueue.findIndex((q) => q.runId === runId);
+    if (idx === -1) return false;
+    this.runQueue.splice(idx, 1);
+    return true;
+  }
+
+  registerActive(runId: string, controller: AbortController): void {
+    this.activeRuns.set(runId, controller);
+  }
+
+  unregisterActive(runId: string): void {
+    this.activeRuns.delete(runId);
+  }
+
+  abortActive(runId: string): boolean {
+    const controller = this.activeRuns.get(runId);
+    if (!controller) return false;
+    controller.abort();
+    return true;
+  }
+
+  /**
+   * Abort every in-flight run. `onAbort` is called for each run id before its
+   * controller is signaled (so callers can log, emit, etc.).
+   */
+  abortAllActive(onAbort?: (runId: string) => void): void {
+    for (const [runId, controller] of this.activeRuns) {
+      if (onAbort) onAbort(runId);
+      controller.abort();
+    }
+  }
+
+  /** Forget all active-run bookkeeping without aborting (used during shutdown). */
+  clearActive(): void {
+    this.activeRuns.clear();
+  }
+
+  /**
+   * Schedule a launch via setTimeout (when `delay` is truthy) or setImmediate
+   * (otherwise). Tracks the handle so it can be cleared during shutdown.
+   */
+  scheduleLaunch(fn: () => void, delay?: number): void {
+    if (delay) {
+      const handle = setTimeout(fn, delay);
+      this.timerHandles.add(handle);
+    } else {
+      const handle = setImmediate(fn);
+      this.timerHandles.add(handle);
+    }
+  }
+
+  clearTimers(): void {
+    for (const handle of this.timerHandles) {
+      if (typeof handle === 'object' && 'ref' in handle) {
+        clearImmediate(handle as ReturnType<typeof setImmediate>);
+      } else {
+        clearTimeout(handle as ReturnType<typeof setTimeout>);
+      }
+    }
+    this.timerHandles.clear();
+  }
+
+  /**
+   * Drain the queue while capacity is available, dispatching each item via
+   * `dispatch(run)`. Each iteration is wrapped in try/catch so that a failure
+   * dispatching one queued item (e.g., an event-transport that throws) cannot
+   * tear down the loop and leave remaining items stuck in the queue. Failed
+   * dispatches are logged, the run is marked as failed in storage as
+   * best-effort telemetry, and the loop advances to the next item.
+   *
+   * Emits `run.dequeued` for each dispatched run (this is what the C3
+   * resilience test exercises by having the emit throw).
+   */
+  processQueue(dispatch: (run: QueuedRun) => void): void {
+    while (this.hasCapacity() && this.runQueue.length > 0) {
+      const next = this.runQueue.shift()!;
+
+      try {
+        this.logger.debug(`Starting queued run ${next.runId}`);
+
+        this.events.emit({
+          runId: next.runId,
+          kind: next.definition.kind,
+          eventType: 'run.dequeued',
+          timestamp: new Date(),
+        });
+
+        dispatch(next);
+      } catch (err) {
+        this.recordQueueDispatchFailure(next, err);
+      }
+    }
+  }
+
+  /**
+   * Record a queue-dispatch failure: log it, emit a failure event, and
+   * best-effort mark the run as failed in storage. All side-effects are
+   * defensively wrapped so this method itself never throws.
+   */
+  private recordQueueDispatchFailure(queued: QueuedRun, err: unknown): void {
+    const message = err instanceof Error ? err.message : String(err);
+    this.logger.error(`Failed to dispatch queued run ${queued.runId}`, err);
+
+    try {
+      this.events.emit({
+        runId: queued.runId,
+        kind: queued.definition.kind,
+        eventType: 'run.failed',
+        timestamp: new Date(),
+        payload: { error: { code: 'QUEUE_DISPATCH_FAILED', message } },
+      });
+    } catch (emitErr) {
+      this.logger.error(`Failed to emit dispatch-failure event for ${queued.runId}`, emitErr);
+    }
+
+    // Fire-and-forget; we don't want to block the queue loop on storage I/O.
+    this.storage
+      .updateRun(queued.runId, {
+        status: 'failed',
+        finishedAt: new Date(),
+        error: { code: 'QUEUE_DISPATCH_FAILED', message },
+      })
+      .catch((updateErr) => {
+        this.logger.error(`Failed to record dispatch failure for ${queued.runId}`, updateErr);
+      });
+  }
+}

package/src/core/engine/lifecycle.ts

@@ -0,0 +1,176 @@
+/**
+ * Lifecycle and resume orchestration for the WorkflowEngine.
+ *
+ * Owns the cross-cutting "engine itself" lifecycle:
+ *   - `initialize()` — bring up storage adapters that need explicit init
+ *     (e.g., PostgresStorageAdapter)
+ *   - `shutdown()` — drain timers, abort in-flight runs, close transports
+ *   - `resumeRun()` / `getResumableRuns()` / `resumeAllInterrupted()` —
+ *     restart interrupted runs from their persisted checkpoints
+ *
+ * NOT responsible for:
+ *   - per-step execution (that lives in `orchestrator.ts`)
+ *   - queue/active/timer bookkeeping (that lives in `engine/concurrency.ts`;
+ *     this module delegates to a `ConcurrencyManager`)
+ *   - building the `executeWorkflow` invocation (the engine still owns that
+ *     so the orchestrator/spawnChild wiring stays in one place; this module
+ *     calls it via the injected `launchRun` callback)
+ */
+
+import type { StorageAdapter, WorkflowRunRecord } from '../../storage/types';
+import type { EventTransport } from '../../events/types';
+import type { Logger, WorkflowDefinition, WorkflowKind } from '../types';
+import { RunNotFoundError, WorkflowNotFoundError } from '../../utils/errors';
+import type { ConcurrencyManager } from './concurrency';
+
+/**
+ * Callback the engine provides for actually launching a run. Lifecycle uses
+ * it to start a resumed run with its checkpoint; the engine's implementation
+ * registers the abort controller, schedules the launch via the concurrency
+ * manager, and invokes the orchestrator.
+ */
+export type LaunchRunFn = (
+  runId: string,
+  definition: WorkflowDefinition,
+  input: Record<string, unknown>,
+  metadata?: Record<string, unknown>,
+  delay?: number,
+  checkpoint?: { completedStepKeys: Set<string>; results: Record<string, unknown> }
+) => void;
+
+export interface LifecycleManagerDeps {
+  storage: StorageAdapter;
+  events: EventTransport;
+  logger: Logger;
+  registry: Map<WorkflowKind, WorkflowDefinition>;
+  concurrency: ConcurrencyManager;
+  launchRun: LaunchRunFn;
+}
+
+export class LifecycleManager {
+  private readonly storage: StorageAdapter;
+  private readonly events: EventTransport;
+  private readonly logger: Logger;
+  private readonly registry: Map<WorkflowKind, WorkflowDefinition>;
+  private readonly concurrency: ConcurrencyManager;
+  private readonly launchRun: LaunchRunFn;
+
+  constructor(deps: LifecycleManagerDeps) {
+    this.storage = deps.storage;
+    this.events = deps.events;
+    this.logger = deps.logger;
+    this.registry = deps.registry;
+    this.concurrency = deps.concurrency;
+    this.launchRun = deps.launchRun;
+  }
+
+  /**
+   * Initialize the engine's storage adapter if it requires it.
+   * Safe to call when the adapter has no `initialize` hook.
+   */
+  async initialize(): Promise<void> {
+    if (this.storage.initialize) {
+      await this.storage.initialize();
+    }
+  }
+
+  /**
+   * Shutdown the engine gracefully:
+   *   1. clear pending timer handles (so no further launches fire)
+   *   2. abort all in-flight runs
+   *   3. close the event transport
+   *   4. close the storage adapter
+   *   5. drop active-run bookkeeping
+   */
+  async shutdown(): Promise<void> {
+    this.logger.info('Shutting down workflow engine...');
+
+    this.concurrency.clearTimers();
+
+    this.concurrency.abortAllActive((runId) => {
+      this.logger.debug(`Canceling active run: ${runId}`);
+    });
+
+    if (this.events.close) {
+      this.events.close();
+    }
+
+    if (this.storage.close) {
+      await this.storage.close();
+    }
+
+    this.concurrency.clearActive();
+    this.logger.info('Workflow engine shutdown complete');
+  }
+
+  /**
+   * Resume an interrupted workflow run from its checkpoint.
+   * The run must be in 'queued' or 'running' status.
+   */
+  async resumeRun(runId: string): Promise<string> {
+    const run = await this.storage.getRun(runId);
+    if (!run) {
+      throw new RunNotFoundError(runId);
+    }
+
+    if (!['queued', 'running'].includes(run.status)) {
+      throw new Error(`Cannot resume run ${runId}: status is "${run.status}"`);
+    }
+
+    const definition = this.registry.get(run.kind);
+    if (!definition) {
+      throw new WorkflowNotFoundError(run.kind);
+    }
+
+    if (this.concurrency.isActive(runId)) {
+      this.logger.warn(`Run ${runId} is already active, skipping resume`);
+      return runId;
+    }
+
+    this.logger.info(`Resuming run ${runId} from checkpoint`);
+
+    const completedStepKeys = new Set(run.completedSteps ?? []);
+
+    this.launchRun(runId, definition, run.input, run.metadata, undefined, {
+      completedStepKeys,
+      results: run.context,
+    });
+
+    return runId;
+  }
+
+  /**
+   * Get all runs that were interrupted and can be resumed.
+   * Returns runs with status 'queued' or 'running'.
+   */
+  async getResumableRuns(): Promise<WorkflowRunRecord[]> {
+    const result = await this.storage.listRuns({
+      status: ['queued', 'running'],
+    });
+    return result.items;
+  }
+
+  /**
+   * Resume every interrupted run whose workflow kind is currently registered.
+   * Useful for recovering after a server restart.
+   */
+  async resumeAllInterrupted(): Promise<string[]> {
+    const resumableRuns = await this.getResumableRuns();
+    const resumedIds: string[] = [];
+
+    for (const run of resumableRuns) {
+      if (this.registry.has(run.kind)) {
+        try {
+          await this.resumeRun(run.id);
+          resumedIds.push(run.id);
+        } catch (error) {
+          this.logger.error(`Failed to resume run ${run.id}:`, error);
+        }
+      } else {
+        this.logger.warn(`Cannot resume run ${run.id}: workflow "${run.kind}" not registered`);
+      }
+    }
+
+    return resumedIds;
+  }
+}

package/src/core/engine.test.ts

@@ -1271,6 +1271,76 @@ describe('WorkflowEngine', () => {
     });
   });
 
+  describe('processQueue resilience (audit 2026-04-27 C3)', () => {
+    it('continues processing remaining queued items when one item fails to dispatch', async () => {
+      // Drive a dispatch failure by injecting an event transport whose
+      // run.dequeued emit throws once. Pre-fix, the throw escaped processQueue
+      // and the remaining queued items were stranded; post-fix, the loop logs,
+      // records telemetry, and advances to the next item.
+      const baseEvents = new MemoryEventTransport();
+      let dequeueCount = 0;
+      const flakyEvents: typeof baseEvents = Object.create(baseEvents);
+      flakyEvents.emit = (event) => {
+        if (event.eventType === 'run.dequeued') {
+          dequeueCount++;
+          if (dequeueCount === 1) {
+            throw new Error('boom: dequeue handler exploded');
+          }
+        }
+        return baseEvents.emit.call(baseEvents, event);
+      };
+      flakyEvents.subscribe = baseEvents.subscribe.bind(baseEvents);
+      flakyEvents.subscribeAll = baseEvents.subscribeAll.bind(baseEvents);
+
+      const customStorage = new MemoryStorageAdapter();
+      const eng = new WorkflowEngine({
+        storage: customStorage,
+        events: flakyEvents,
+        logger: new SilentLogger(),
+        settings: { maxConcurrency: 1 },
+      });
+
+      const completionOrder: string[] = [];
+      eng.registerWorkflow({
+        kind: 'test.queue.resilient',
+        name: 'Test Queue Resilient',
+        steps: [
+          {
+            key: 's1',
+            name: 'S1',
+            handler: async (ctx) => {
+              completionOrder.push(ctx.input.id as string);
+              return 'done';
+            },
+          },
+        ],
+      });
+
+      // First runs immediately (no dequeue event emitted for it).
+      const firstId = await eng.startRun({ kind: 'test.queue.resilient', input: { id: 'first' } });
+      // These two get queued behind it; their run.dequeued events fire when
+      // the queue advances.
+      const secondId = await eng.startRun({ kind: 'test.queue.resilient', input: { id: 'second' } });
+      const thirdId = await eng.startRun({ kind: 'test.queue.resilient', input: { id: 'third' } });
+
+      // First and third must complete; second's dispatch fails and is recorded
+      // as failed instead of stranding the queue.
+      await eng.waitForRun(firstId, { timeout: 2000 });
+      await eng.waitForRun(thirdId, { timeout: 2000 });
+
+      expect(completionOrder).toContain('first');
+      expect(completionOrder).toContain('third');
+      expect(completionOrder).not.toContain('second');
+
+      // Telemetry: the dropped run should be marked failed in storage.
+      const secondRun = await customStorage.getRun(secondId);
+      expect(secondRun?.status).toBe('failed');
+      expect(secondRun?.error?.code).toBe('QUEUE_DISPATCH_FAILED');
+
+      await eng.shutdown();
+    });
+  });
+
   describe('orchestrator: raceWithAbort pre-aborted signal', () => {
     it('should throw WorkflowCanceledError immediately when signal is already aborted', async () => {
       engine.registerWorkflow({