@multiplier-labs/stepflow 0.3.1 → 0.3.4

package/dist/{types-DzkEqnQJ.d.ts → types-DZUdWmCc.d.ts}

@@ -1,4 +1,4 @@
-import { W as WorkflowKind } from './types-BVShq7zT.js';
+import { W as WorkflowKind } from './types-K5Gjk3H_.js';
 
 /**
  * Event system types for the workflow engine.
@@ -7,7 +7,7 @@ import { W as WorkflowKind } from './types-BVShq7zT.js';
 /**
  * Built-in event types emitted by the workflow engine.
  */
-type BuiltInEventType = "run.created" | "run.queued" | "run.dequeued" | "run.started" | "run.resumed" | "run.completed" | "run.failed" | "run.canceled" | "run.timeout" | "step.started" | "step.completed" | "step.failed" | "step.skipped" | "step.retry";
+type BuiltInEventType = 'run.created' | 'run.queued' | 'run.dequeued' | 'run.started' | 'run.resumed' | 'run.completed' | 'run.failed' | 'run.canceled' | 'run.timeout' | 'step.started' | 'step.completed' | 'step.failed' | 'step.skipped' | 'step.retry';
 /**
  * Event types emitted by the workflow engine.
  * Includes built-in types and allows custom events via string.

package/dist/{types-BVShq7zT.d.ts → types-K5Gjk3H_.d.ts}

@@ -10,18 +10,18 @@ type WorkflowKind = string;
 /**
  * Status of a workflow run.
  */
-type RunStatus = "queued" | "running" | "succeeded" | "failed" | "canceled";
+type RunStatus = 'queued' | 'running' | 'succeeded' | 'failed' | 'canceled';
 /**
  * Status of a workflow step.
  */
-type StepStatus = "pending" | "running" | "succeeded" | "failed" | "skipped" | "canceled";
+type StepStatus = 'pending' | 'running' | 'succeeded' | 'failed' | 'skipped' | 'canceled';
 /**
  * Error handling strategy for a step.
  * - 'fail': Stop the workflow immediately
  * - 'retry': Retry the step up to maxRetries times
  * - 'skip': Mark as skipped and continue to next step
  */
-type StepErrorStrategy = "fail" | "retry" | "skip";
+type StepErrorStrategy = 'fail' | 'retry' | 'skip';
 /**
  * Structured error information stored with runs and steps.
  */
@@ -35,7 +35,7 @@ interface WorkflowError {
  * Result of a completed workflow run.
  */
 interface RunResult {
-    status: "succeeded" | "failed" | "canceled";
+    status: 'succeeded' | 'failed' | 'canceled';
     results: Record<string, unknown>;
     error?: WorkflowError;
     duration: number;

package/docs/stepflow-api-reference.md

@@ -496,7 +496,6 @@ await scheduler.addSchedule({
 await scheduler.removeSchedule(scheduleId);
 await scheduler.updateSchedule(scheduleId, { enabled: false });
 const schedules = await scheduler.getSchedules();
-const schedule = scheduler.getSchedule(scheduleId);
 const runId = await scheduler.triggerNow(scheduleId);
 
 await scheduler.stop();
@@ -780,7 +779,7 @@ const storage = new SQLiteStorageAdapter({
   tablePrefix: 'workflow',   // default: 'workflow'
 });
 
-// Additional methods
+// Additional methods (implementation-only; not part of the StorageAdapter interface)
 storage.transactionSync<T>(fn: () => T): T;
 await storage.getInterruptedRuns(): Promise<WorkflowRunRecord[]>;
 await storage.getLastCompletedStep(runId): Promise<WorkflowRunStepRecord | null>;
@@ -868,7 +867,7 @@ await storage.getStepResults(runId): Promise<StepResult[]>;
 await storage.saveStepResult(result): Promise<void>;
 ```
 
-**Resume support:**
+**Resume support** (implementation-only; not part of the StorageAdapter interface):
 
 ```typescript
 await storage.getInterruptedRuns(): Promise<WorkflowRunRecord[]>;
@@ -1245,6 +1244,7 @@ import {
   StepTimeoutError,
   WorkflowCanceledError,
   WorkflowTimeoutError,
+  WaitForRunTimeoutError,
 } from '@multiplier-labs/stepflow';
 ```
 
@@ -1258,6 +1258,7 @@ import {
 | `StepTimeoutError` | `STEP_TIMEOUT` | `(stepKey, timeoutMs)` |
 | `WorkflowCanceledError` | `WORKFLOW_CANCELED` | `(runId)` |
 | `WorkflowTimeoutError` | `WORKFLOW_TIMEOUT` | `(runId, timeoutMs)` |
+| `WaitForRunTimeoutError` | `WAIT_FOR_RUN_TIMEOUT` | `(runId, timeoutMs)` |
 
 ### WorkflowError Record
 
@@ -1423,7 +1424,7 @@ import {
   WorkflowEngineError, WorkflowNotFoundError,
   WorkflowAlreadyRegisteredError, RunNotFoundError,
   StepError, StepTimeoutError,
-  WorkflowCanceledError, WorkflowTimeoutError,
+  WorkflowCanceledError, WorkflowTimeoutError, WaitForRunTimeoutError,
   ConsoleLogger, SilentLogger, createScopedLogger,
   generateId, sleep, withRetry, calculateRetryDelay,
   DEFAULT_RETRY_OPTIONS,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@multiplier-labs/stepflow",
-  "version": "0.3.1",
+  "version": "0.3.4",
   "description": "A durable, type-safe workflow orchestration engine for Node.js with SQLite persistence, scheduling, and real-time events",
   "type": "module",
   "license": "MIT",
@@ -9,7 +9,8 @@
     "url": "https://github.com/Multiplier-Labs/stepflow"
   },
   "publishConfig": {
-    "registry": "https://npm.pkg.github.com"
+    "registry": "https://registry.npmjs.org",
+    "access": "public"
   },
   "keywords": [
     "workflow",
@@ -52,11 +53,11 @@
   },
   "dependencies": {
     "cron-parser": "^5.5.0",
-    "re2": "^1.23.3"
+    "re2": "^1.24.0"
   },
   "peerDependencies": {
     "better-sqlite3": ">=11.0.0",
-    "kysely": ">=0.28.14",
+    "kysely": ">=0.28.16",
     "pg": ">=8.13.0"
   },
   "peerDependenciesMeta": {
@@ -74,12 +75,15 @@
     "@types/better-sqlite3": "^7.6.11",
     "@types/node": "^22.0.0",
     "@types/pg": "^8.18.0",
-    "@vitest/coverage-v8": "^4.1.0",
+    "@vitest/coverage-v8": "^4.1.4",
     "better-sqlite3": "^12.8.0",
-    "kysely": "^0.28.12",
+    "kysely": "^0.28.16",
     "pg": "^8.20.0",
-    "tsup": "^8.3.5",
+    "tsup": "^8.5.1",
     "typescript": "^5.7.2",
-    "vitest": "^4.1.0"
+    "vitest": "^4.1.4"
+  },
+  "overrides": {
+    "postcss": ">=8.5.10"
   }
 }

package/src/core/engine/concurrency.ts

@@ -0,0 +1,229 @@
+/**
+ * Concurrency control for the WorkflowEngine.
+ *
+ * Owns:
+ *   - the priority-ordered queue of pending runs (`runQueue`)
+ *   - the set of in-flight run abort controllers (`activeRuns`)
+ *   - the timer handles used to defer launches via setTimeout/setImmediate
+ *
+ * Responsibilities:
+ *   - `hasCapacity()` against the configured `maxConcurrency` ceiling
+ *   - priority-ordered enqueue (higher priority first, FIFO within a priority)
+ *   - dispatch loop that drains the queue while capacity exists, with a
+ *     try/catch around each dispatch so a single bad run cannot tear down
+ *     the loop and strand the rest of the queue (preserves the audit
+ *     2026-04-27 C3 fix in `processQueue`).
+ *   - dispatch-failure telemetry: log + emit `run.failed` + best-effort
+ *     storage update with `QUEUE_DISPATCH_FAILED`.
+ *
+ * NOT responsible for:
+ *   - building the `executeWorkflow` invocation (that lives in WorkflowEngine
+ *     so the orchestrator/spawnChild wiring stays where it is)
+ *   - storage initialization / shutdown sequencing (see `lifecycle.ts`)
+ */
+
+import type { StorageAdapter } from '../../storage/types';
+import type { EventTransport } from '../../events/types';
+import type { Logger, WorkflowDefinition } from '../types';
+
+/**
+ * Queued run waiting to be executed when capacity is available.
+ */
+export interface QueuedRun {
+  runId: string;
+  definition: WorkflowDefinition;
+  input: Record<string, unknown>;
+  metadata?: Record<string, unknown>;
+  priority: number;
+  queuedAt: Date;
+}
+
+type TimerHandle = ReturnType<typeof setTimeout> | ReturnType<typeof setImmediate>;
+
+export interface ConcurrencyManagerDeps {
+  storage: StorageAdapter;
+  events: EventTransport;
+  logger: Logger;
+  /** Maximum concurrent runs; undefined or <=0 means unlimited. */
+  maxConcurrency?: number;
+}
+
+export class ConcurrencyManager {
+  private readonly activeRuns = new Map<string, AbortController>();
+  private readonly runQueue: QueuedRun[] = [];
+  private readonly timerHandles = new Set<TimerHandle>();
+  private readonly storage: StorageAdapter;
+  private readonly events: EventTransport;
+  private readonly logger: Logger;
+  private readonly maxConcurrency: number | undefined;
+
+  constructor(deps: ConcurrencyManagerDeps) {
+    this.storage = deps.storage;
+    this.events = deps.events;
+    this.logger = deps.logger;
+    this.maxConcurrency = deps.maxConcurrency;
+  }
+
+  hasCapacity(): boolean {
+    if (this.maxConcurrency === undefined || this.maxConcurrency <= 0) {
+      return true;
+    }
+    return this.activeRuns.size < this.maxConcurrency;
+  }
+
+  getActiveCount(): number {
+    return this.activeRuns.size;
+  }
+
+  getQueuedCount(): number {
+    return this.runQueue.length;
+  }
+
+  isActive(runId: string): boolean {
+    return this.activeRuns.has(runId);
+  }
+
+  /** Insert a run in priority order (higher priority first, FIFO within same). */
+  enqueue(run: QueuedRun): void {
+    let inserted = false;
+    for (let i = 0; i < this.runQueue.length; i++) {
+      if (run.priority > this.runQueue[i].priority) {
+        this.runQueue.splice(i, 0, run);
+        inserted = true;
+        break;
+      }
+    }
+    if (!inserted) {
+      this.runQueue.push(run);
+    }
+  }
+
+  /** Remove a run from the queue if present. Returns true if it was queued. */
+  removeFromQueue(runId: string): boolean {
+    const idx = this.runQueue.findIndex((q) => q.runId === runId);
+    if (idx === -1) return false;
+    this.runQueue.splice(idx, 1);
+    return true;
+  }
+
+  registerActive(runId: string, controller: AbortController): void {
+    this.activeRuns.set(runId, controller);
+  }
+
+  unregisterActive(runId: string): void {
+    this.activeRuns.delete(runId);
+  }
+
+  abortActive(runId: string): boolean {
+    const controller = this.activeRuns.get(runId);
+    if (!controller) return false;
+    controller.abort();
+    return true;
+  }
+
+  /**
+   * Abort every in-flight run. `onAbort` is called for each run id before its
+   * controller is signaled (so callers can log, emit, etc.).
+   */
+  abortAllActive(onAbort?: (runId: string) => void): void {
+    for (const [runId, controller] of this.activeRuns) {
+      if (onAbort) onAbort(runId);
+      controller.abort();
+    }
+  }
+
+  /** Forget all active-run bookkeeping without aborting (used during shutdown). */
+  clearActive(): void {
+    this.activeRuns.clear();
+  }
+
+  /**
+   * Schedule a launch via setTimeout (when `delay` is truthy) or setImmediate
+   * (otherwise). Tracks the handle so it can be cleared during shutdown.
+   */
+  scheduleLaunch(fn: () => void, delay?: number): void {
+    if (delay) {
+      const handle = setTimeout(fn, delay);
+      this.timerHandles.add(handle);
+    } else {
+      const handle = setImmediate(fn);
+      this.timerHandles.add(handle);
+    }
+  }
+
+  clearTimers(): void {
+    for (const handle of this.timerHandles) {
+      if (typeof handle === 'object' && 'ref' in handle) {
+        clearImmediate(handle as ReturnType<typeof setImmediate>);
+      } else {
+        clearTimeout(handle as ReturnType<typeof setTimeout>);
+      }
+    }
+    this.timerHandles.clear();
+  }
+
+  /**
+   * Drain the queue while capacity is available, dispatching each item via
+   * `dispatch(run)`. Each iteration is wrapped in try/catch so that a failure
+   * dispatching one queued item (e.g., an event-transport that throws) cannot
+   * tear down the loop and leave remaining items stuck in the queue. Failed
+   * dispatches are logged, the run is marked as failed in storage as
+   * best-effort telemetry, and the loop advances to the next item.
+   *
+   * Emits `run.dequeued` for each dispatched run (this is what the C3
+   * resilience test exercises by having the emit throw).
+   */
+  processQueue(dispatch: (run: QueuedRun) => void): void {
+    while (this.hasCapacity() && this.runQueue.length > 0) {
+      const next = this.runQueue.shift()!;
+
+      try {
+        this.logger.debug(`Starting queued run ${next.runId}`);
+
+        this.events.emit({
+          runId: next.runId,
+          kind: next.definition.kind,
+          eventType: 'run.dequeued',
+          timestamp: new Date(),
+        });
+
+        dispatch(next);
+      } catch (err) {
+        this.recordQueueDispatchFailure(next, err);
+      }
+    }
+  }
+
+  /**
+   * Record a queue-dispatch failure: log it, emit a failure event, and
+   * best-effort mark the run as failed in storage. All side-effects are
+   * defensively wrapped so this method itself never throws.
+   */
+  private recordQueueDispatchFailure(queued: QueuedRun, err: unknown): void {
+    const message = err instanceof Error ? err.message : String(err);
+    this.logger.error(`Failed to dispatch queued run ${queued.runId}`, err);
+
+    try {
+      this.events.emit({
+        runId: queued.runId,
+        kind: queued.definition.kind,
+        eventType: 'run.failed',
+        timestamp: new Date(),
+        payload: { error: { code: 'QUEUE_DISPATCH_FAILED', message } },
+      });
+    } catch (emitErr) {
+      this.logger.error(`Failed to emit dispatch-failure event for ${queued.runId}`, emitErr);
+    }
+
+    // Fire-and-forget; we don't want to block the queue loop on storage I/O.
+    this.storage
+      .updateRun(queued.runId, {
+        status: 'failed',
+        finishedAt: new Date(),
+        error: { code: 'QUEUE_DISPATCH_FAILED', message },
+      })
+      .catch((updateErr) => {
+        this.logger.error(`Failed to record dispatch failure for ${queued.runId}`, updateErr);
+      });
+  }
+}

package/src/core/engine/lifecycle.ts

@@ -0,0 +1,176 @@
+/**
+ * Lifecycle and resume orchestration for the WorkflowEngine.
+ *
+ * Owns the cross-cutting "engine itself" lifecycle:
+ *   - `initialize()` — bring up storage adapters that need explicit init
+ *     (e.g., PostgresStorageAdapter)
+ *   - `shutdown()` — drain timers, abort in-flight runs, close transports
+ *   - `resumeRun()` / `getResumableRuns()` / `resumeAllInterrupted()` —
+ *     restart interrupted runs from their persisted checkpoints
+ *
+ * NOT responsible for:
+ *   - per-step execution (that lives in `orchestrator.ts`)
+ *   - queue/active/timer bookkeeping (that lives in `engine/concurrency.ts`;
+ *     this module delegates to a `ConcurrencyManager`)
+ *   - building the `executeWorkflow` invocation (the engine still owns that
+ *     so the orchestrator/spawnChild wiring stays in one place; this module
+ *     calls it via the injected `launchRun` callback)
+ */
+
+import type { StorageAdapter, WorkflowRunRecord } from '../../storage/types';
+import type { EventTransport } from '../../events/types';
+import type { Logger, WorkflowDefinition, WorkflowKind } from '../types';
+import { RunNotFoundError, WorkflowNotFoundError } from '../../utils/errors';
+import type { ConcurrencyManager } from './concurrency';
+
+/**
+ * Callback the engine provides for actually launching a run. Lifecycle uses
+ * it to start a resumed run with its checkpoint; the engine's implementation
+ * registers the abort controller, schedules the launch via the concurrency
+ * manager, and invokes the orchestrator.
+ */
+export type LaunchRunFn = (
+  runId: string,
+  definition: WorkflowDefinition,
+  input: Record<string, unknown>,
+  metadata?: Record<string, unknown>,
+  delay?: number,
+  checkpoint?: { completedStepKeys: Set<string>; results: Record<string, unknown> }
+) => void;
+
+export interface LifecycleManagerDeps {
+  storage: StorageAdapter;
+  events: EventTransport;
+  logger: Logger;
+  registry: Map<WorkflowKind, WorkflowDefinition>;
+  concurrency: ConcurrencyManager;
+  launchRun: LaunchRunFn;
+}
+
+export class LifecycleManager {
+  private readonly storage: StorageAdapter;
+  private readonly events: EventTransport;
+  private readonly logger: Logger;
+  private readonly registry: Map<WorkflowKind, WorkflowDefinition>;
+  private readonly concurrency: ConcurrencyManager;
+  private readonly launchRun: LaunchRunFn;
+
+  constructor(deps: LifecycleManagerDeps) {
+    this.storage = deps.storage;
+    this.events = deps.events;
+    this.logger = deps.logger;
+    this.registry = deps.registry;
+    this.concurrency = deps.concurrency;
+    this.launchRun = deps.launchRun;
+  }
+
+  /**
+   * Initialize the engine's storage adapter if it requires it.
+   * Safe to call when the adapter has no `initialize` hook.
+   */
+  async initialize(): Promise<void> {
+    if (this.storage.initialize) {
+      await this.storage.initialize();
+    }
+  }
+
+  /**
+   * Shutdown the engine gracefully:
+   *   1. clear pending timer handles (so no further launches fire)
+   *   2. abort all in-flight runs
+   *   3. close the event transport
+   *   4. close the storage adapter
+   *   5. drop active-run bookkeeping
+   */
+  async shutdown(): Promise<void> {
+    this.logger.info('Shutting down workflow engine...');
+
+    this.concurrency.clearTimers();
+
+    this.concurrency.abortAllActive((runId) => {
+      this.logger.debug(`Canceling active run: ${runId}`);
+    });
+
+    if (this.events.close) {
+      this.events.close();
+    }
+
+    if (this.storage.close) {
+      await this.storage.close();
+    }
+
+    this.concurrency.clearActive();
+    this.logger.info('Workflow engine shutdown complete');
+  }
+
+  /**
+   * Resume an interrupted workflow run from its checkpoint.
+   * The run must be in 'queued' or 'running' status.
+   */
+  async resumeRun(runId: string): Promise<string> {
+    const run = await this.storage.getRun(runId);
+    if (!run) {
+      throw new RunNotFoundError(runId);
+    }
+
+    if (!['queued', 'running'].includes(run.status)) {
+      throw new Error(`Cannot resume run ${runId}: status is "${run.status}"`);
+    }
+
+    const definition = this.registry.get(run.kind);
+    if (!definition) {
+      throw new WorkflowNotFoundError(run.kind);
+    }
+
+    if (this.concurrency.isActive(runId)) {
+      this.logger.warn(`Run ${runId} is already active, skipping resume`);
+      return runId;
+    }
+
+    this.logger.info(`Resuming run ${runId} from checkpoint`);
+
+    const completedStepKeys = new Set(run.completedSteps ?? []);
+
+    this.launchRun(runId, definition, run.input, run.metadata, undefined, {
+      completedStepKeys,
+      results: run.context,
+    });
+
+    return runId;
+  }
+
+  /**
+   * Get all runs that were interrupted and can be resumed.
+   * Returns runs with status 'queued' or 'running'.
+   */
+  async getResumableRuns(): Promise<WorkflowRunRecord[]> {
+    const result = await this.storage.listRuns({
+      status: ['queued', 'running'],
+    });
+    return result.items;
+  }
+
+  /**
+   * Resume every interrupted run whose workflow kind is currently registered.
+   * Useful for recovering after a server restart.
+   */
+  async resumeAllInterrupted(): Promise<string[]> {
+    const resumableRuns = await this.getResumableRuns();
+    const resumedIds: string[] = [];
+
+    for (const run of resumableRuns) {
+      if (this.registry.has(run.kind)) {
+        try {
+          await this.resumeRun(run.id);
+          resumedIds.push(run.id);
+        } catch (error) {
+          this.logger.error(`Failed to resume run ${run.id}:`, error);
+        }
+      } else {
+        this.logger.warn(`Cannot resume run ${run.id}: workflow "${run.kind}" not registered`);
+      }
+    }
+
+    return resumedIds;
+  }
+}