@multiplier-labs/stepflow 0.3.1 → 0.3.4

package/dist/events/index.d.ts

@@ -1,6 +1,6 @@
-import { a as EventTransport, W as WorkflowEvent, E as EventCallback, U as Unsubscribe, b as WorkflowEventType } from '../types-DzkEqnQJ.js';
-export { B as BuiltInEventType } from '../types-DzkEqnQJ.js';
-import { L as Logger } from '../types-BVShq7zT.js';
+import { a as EventTransport, W as WorkflowEvent, E as EventCallback, U as Unsubscribe, b as WorkflowEventType } from '../types-DZUdWmCc.js';
+export { B as BuiltInEventType } from '../types-DZUdWmCc.js';
+import { L as Logger } from '../types-K5Gjk3H_.js';
 
 /**
  * In-memory event transport using Node.js EventEmitter.
@@ -215,6 +215,23 @@ interface WebhookEventTransportConfig {
      * Requests beyond this limit are queued.
      */
     maxConcurrentRequests?: number;
+    /**
+     * Maximum number of queued webhook deliveries waiting for a concurrency slot
+     * (default: 1000, env override: `STEPFLOW_WEBHOOK_MAX_QUEUE_DEPTH`).
+     *
+     * When the queue is full, incoming `emit()` deliveries are dropped to
+     * protect the process from memory exhaustion under burst traffic. Drops
+     * are logged via `logger.warn` with the endpoint id, current queue depth,
+     * and a recommended `retryAfterMs`. A value of `0` disables the cap (not
+     * recommended in production).
+     */
+    maxQueueDepth?: number;
+    /**
+     * Suggested back-off in milliseconds reported on dropped deliveries
+     * (default: 1000). Surfaced in the warning log so external orchestrators
+     * (sidecars, sweeper jobs) can respect it like an HTTP `Retry-After`.
+     */
+    queueDropRetryAfterMs?: number;
     /** Logger for webhook transport errors (default: console-based) */
     logger?: Logger;
 }
@@ -227,7 +244,7 @@ interface WebhookPayload {
      * The `Date` is converted to a string before delivery so that JSON consumers
      * receive a portable, timezone-aware representation without needing Date parsing.
      */
-    event: Omit<WorkflowEvent, "timestamp"> & {
+    event: Omit<WorkflowEvent, 'timestamp'> & {
         timestamp: string;
     };
     /** ISO 8601 timestamp of when this payload was delivered. */
@@ -267,12 +284,25 @@ declare class WebhookEventTransport implements EventTransport {
     private defaultRetries;
     private retryDelay;
     private fetchFn;
+    /**
+     * Whether `fetchFn` was caller-supplied. When `false` the default fetch is
+     * replaced with `pinnedFetch` so the TCP connection can target the IP that
+     * was just validated (closes the DNS-rebinding TOCTOU window). When `true`
+     * the custom fetch is used as-is so test injection still works.
+     */
+    private hasCustomFetch;
     private allowInsecureUrls;
     private maxPayloadBytes;
     private maxConcurrentRequests;
+    private maxQueueDepth;
+    private queueDropRetryAfterMs;
     private activeRequests;
     private requestQueue;
+    private droppedRequests;
     private logger;
+    private dnsCache;
+    private static readonly DNS_CACHE_TTL_MS;
+    private static readonly DNS_TIMEOUT_MS;
     private runSubscribers;
     private globalSubscribers;
     constructor(config?: WebhookEventTransportConfig);
@@ -298,6 +328,15 @@ declare class WebhookEventTransport implements EventTransport {
      * Emit an event to all matching webhook endpoints.
      */
     emit(event: WorkflowEvent): void;
+    /**
+     * Number of webhook deliveries dropped due to queue overflow since startup.
+     * Useful for surfacing back-pressure to operators (metrics/health checks).
+     */
+    getDroppedRequestCount(): number;
+    /**
+     * Current depth of the pending request queue.
+     */
+    getQueueDepth(): number;
     /**
      * Subscribe to events for a specific run (server-side).
      */
@@ -327,8 +366,18 @@ declare class WebhookEventTransport implements EventTransport {
      */
     private sleep;
     /**
-     * Resolve the hostname from a URL and verify the resolved IP is not private/reserved.
-     * Prevents DNS rebinding attacks where a public hostname resolves to a private IP at send time.
+     * Resolve a webhook URL's hostname exactly once, verify the resolved IP is
+     * not private/reserved, and return the address + family so the caller can
+     * pin the connection to it.
+     *
+     * Returns `null` for raw-IP URLs (already validated by `validateWebhookUrl`).
+     *
+     * Pinning the resolved IP closes the DNS rebinding TOCTOU window (audit M1):
+     * with the original implementation `fetch()` re-resolved the hostname,
+     * giving a hostile DNS server a chance to flip the answer to an internal
+     * address between SSRF validation and connect.
+     *
+     * Uses a short-TTL cache and per-call timeout to avoid blocking the event loop.
      */
     private validateResolvedHost;
     /**
@@ -338,7 +387,11 @@ declare class WebhookEventTransport implements EventTransport {
      */
     private validateWebhookUrl;
     /**
-     * Enqueue a webhook request with concurrency limiting.
+     * Enqueue a webhook request with concurrency limiting and a queue-depth cap.
+     * Returns `true` if the request was accepted (executed or queued), or
+     * `false` if the queue is full and the request was rejected. The caller is
+     * responsible for logging the drop and incrementing any drop metrics, since
+     * it has the endpoint context.
      */
     private enqueueRequest;
 }

package/dist/events/index.js

@@ -2,7 +2,7 @@ import {
   MemoryEventTransport,
   SocketIOEventTransport,
   WebhookEventTransport
-} from "../chunk-XCF4PLN7.js";
+} from "../chunk-U3PCVK6O.js";
 export {
   MemoryEventTransport,
   SocketIOEventTransport,

package/dist/{index-CPfGsmrp.d.ts → index-Cd0IZ40L.d.ts}

@@ -1,6 +1,6 @@
-import { L as Logger, g as WorkflowDefinition, W as WorkflowKind, R as RunStatus } from './types-BVShq7zT.js';
-import { i as StorageAdapter, j as WorkflowRunRecord } from './types-BlwFEGzj.js';
-import { a as EventTransport, E as EventCallback, U as Unsubscribe } from './types-DzkEqnQJ.js';
+import { L as Logger, g as WorkflowDefinition, W as WorkflowKind, R as RunStatus } from './types-K5Gjk3H_.js';
+import { i as StorageAdapter, j as WorkflowRunRecord } from './types-DSEjiLMj.js';
+import { a as EventTransport, E as EventCallback, U as Unsubscribe } from './types-DZUdWmCc.js';
 import { Database } from 'better-sqlite3';
 import { Pool, PoolConfig } from 'pg';
 
@@ -10,6 +10,10 @@ import { Pool, PoolConfig } from 'pg';
  * This is the primary entry point for the workflow engine.
  * It provides methods for registering workflows, starting runs,
  * and managing workflow execution.
+ *
+ * Internal concerns are decomposed into co-located submodules under
+ * `./engine/` (see `engine/concurrency.ts`, `engine/lifecycle.ts`).
+ * WorkflowEngine itself stays the thin public face.
  */
 
 /**
@@ -47,14 +51,40 @@ interface StartRunOptions<TInput = Record<string, unknown>> {
     /** Priority for queue ordering (higher = runs first, default: 0) */
     priority?: number;
 }
+/**
+ * Main workflow engine class.
+ *
+ * @example
+ * ```typescript
+ * const engine = new WorkflowEngine({
+ *   storage: new MemoryStorageAdapter(),
+ * });
+ *
+ * engine.registerWorkflow({
+ *   kind: 'my.workflow',
+ *   name: 'My Workflow',
+ *   steps: [
+ *     {
+ *       key: 'step1',
+ *       name: 'First Step',
+ *       handler: async (ctx) => {
+ *         return { message: 'Hello!' };
+ *       },
+ *     },
+ *   ],
+ * });
+ *
+ * const runId = await engine.startRun({ kind: 'my.workflow' });
+ * ```
+ */
 declare class WorkflowEngine {
     private registry;
     private storage;
     private events;
     private logger;
-    private activeRuns;
     private settings;
-    private runQueue;
+    private concurrency;
+    private lifecycle;
     constructor(config?: WorkflowEngineConfig);
     /**
      * Initialize the engine and its storage/event adapters.
@@ -70,10 +100,6 @@ declare class WorkflowEngine {
      * Get the number of queued runs waiting for capacity.
      */
     getQueuedRunCount(): number;
-    /**
-     * Check if capacity is available for a new run.
-     */
-    private hasCapacity;
     /**
      * Register a workflow definition.
      * Must be called before runs of this type can be started.
@@ -110,10 +136,6 @@ declare class WorkflowEngine {
      * @throws WorkflowNotFoundError if the workflow kind is not registered
      */
     startRun<TInput = Record<string, unknown>>(options: StartRunOptions<TInput>): Promise<string>;
-    /**
-     * Queue a run in priority order.
-     */
-    private queueRun;
     /**
      * Execute a run (internal method).
      */
@@ -124,7 +146,8 @@ declare class WorkflowEngine {
      */
     private launchRun;
     /**
-     * Process the queue and start runs if capacity is available.
+     * Drain the queue, dispatching each ready run via `executeRun`. Resilience
+     * against per-item dispatch failures lives in `ConcurrencyManager.processQueue`.
      */
     private processQueue;
     /**
@@ -226,7 +249,7 @@ declare class WorkflowEngine {
 /**
  * Schedule trigger types.
  */
-type TriggerType = "cron" | "workflow_completed" | "manual";
+type TriggerType = 'cron' | 'workflow_completed' | 'manual';
 /**
  * Schedule definition.
  */
@@ -265,7 +288,7 @@ interface Scheduler {
     /** Stop the scheduler */
     stop(): Promise<void>;
     /** Add a schedule */
-    addSchedule(schedule: Omit<WorkflowSchedule, "id">): Promise<WorkflowSchedule>;
+    addSchedule(schedule: Omit<WorkflowSchedule, 'id'>): Promise<WorkflowSchedule>;
     /** Remove a schedule */
     removeSchedule(scheduleId: string): Promise<void>;
     /** Update a schedule */
@@ -362,7 +385,7 @@ declare class CronScheduler implements Scheduler {
     /**
      * Add a new schedule.
      */
-    addSchedule(scheduleData: Omit<WorkflowSchedule, "id">): Promise<WorkflowSchedule>;
+    addSchedule(scheduleData: Omit<WorkflowSchedule, 'id'>): Promise<WorkflowSchedule>;
     /**
      * Remove a schedule.
      */
@@ -415,6 +438,12 @@ interface SQLiteSchedulePersistenceConfig {
     db: Database;
     /** Table name for schedules (default: workflow_schedules) */
     tableName?: string;
+    /**
+     * Optional structured logger used to surface JSON corruption events.
+     * Without one, parse failures still increment the corruption counter
+     * exposed via `getJsonParseCorruptionCount()` but produce no log output.
+     */
+    logger?: Logger;
 }
 /**
  * SQLite-based persistence for workflow schedules.
@@ -435,6 +464,7 @@ interface SQLiteSchedulePersistenceConfig {
 declare class SQLiteSchedulePersistence implements SchedulePersistence {
     private db;
     private tableName;
+    private logger;
     private stmts;
     constructor(config: SQLiteSchedulePersistenceConfig);
     private initializeDatabase;
@@ -442,6 +472,7 @@ declare class SQLiteSchedulePersistence implements SchedulePersistence {
     saveSchedule(schedule: WorkflowSchedule): Promise<void>;
     updateSchedule(scheduleId: string, updates: Partial<WorkflowSchedule>): Promise<void>;
     deleteSchedule(scheduleId: string): Promise<void>;
+    private safeJsonParse;
     private rowToSchedule;
 }
 
@@ -484,6 +515,12 @@ interface PostgresSchedulePersistenceConfig {
      * @default true
      */
     autoMigrate?: boolean;
+    /**
+     * Optional structured logger used to surface JSON corruption events.
+     * Without one, parse failures still increment the corruption counter
+     * exposed via `getJsonParseCorruptionCount()` but produce no log output.
+     */
+    logger?: Logger;
 }
 /**
  * PostgreSQL-based persistence for workflow schedules.
@@ -523,6 +560,7 @@ declare class PostgresSchedulePersistence implements SchedulePersistence {
     private autoMigrate;
     private initialized;
     private config;
+    private logger;
     constructor(config: PostgresSchedulePersistenceConfig);
     /**
      * Get a schema-scoped query builder.
@@ -561,6 +599,7 @@ declare class PostgresSchedulePersistence implements SchedulePersistence {
      * Get workflow completion triggers for a specific workflow kind.
      */
     getCompletionTriggers(triggerOnWorkflowKind: string): Promise<WorkflowSchedule[]>;
+    private safeParseOptionalField;
     private rowToSchedule;
 }
 

package/dist/index.d.ts

@@ -1,13 +1,17 @@
-import { a as WorkflowError, L as Logger, W as WorkflowKind, b as StepErrorStrategy, c as WorkflowStep } from './types-BVShq7zT.js';
-export { d as RunResult, R as RunStatus, e as SpawnChildOptions, S as StepStatus, f as WorkflowContext, g as WorkflowDefinition, h as WorkflowHooks } from './types-BVShq7zT.js';
-export { C as CronScheduler, a as CronSchedulerConfig, P as PostgresSchedulePersistence, b as PostgresSchedulePersistenceConfig, S as SQLiteSchedulePersistence, c as SQLiteSchedulePersistenceConfig, d as SchedulePersistence, e as Scheduler, f as StartRunOptions, T as TriggerType, W as WorkflowEngine, g as WorkflowEngineConfig, h as WorkflowSchedule } from './index-CPfGsmrp.js';
-export { C as CreateRunInput, E as ExtendedListRunsOptions, a as ExtendedRunStatus, b as ExtendedStepStatus, c as ExtendedWorkflowRunRecord, L as ListEventsOptions, d as ListRunsOptions, P as PaginatedResult, S as StepRecord, e as StepResult, f as StepflowDatabase, g as StepflowRunsTable, h as StepflowStepResultsTable, i as StorageAdapter, U as UpdateRunInput, W as WorkflowEventRecord, j as WorkflowRunRecord, k as WorkflowRunStepRecord, l as WorkflowStorage } from './types-BlwFEGzj.js';
+import { a as WorkflowError, L as Logger, W as WorkflowKind, b as StepErrorStrategy, c as WorkflowStep } from './types-K5Gjk3H_.js';
+export { d as RunResult, R as RunStatus, e as SpawnChildOptions, S as StepStatus, f as WorkflowContext, g as WorkflowDefinition, h as WorkflowHooks } from './types-K5Gjk3H_.js';
+export { C as CronScheduler, a as CronSchedulerConfig, P as PostgresSchedulePersistence, b as PostgresSchedulePersistenceConfig, S as SQLiteSchedulePersistence, c as SQLiteSchedulePersistenceConfig, d as SchedulePersistence, e as Scheduler, f as StartRunOptions, T as TriggerType, W as WorkflowEngine, g as WorkflowEngineConfig, h as WorkflowSchedule } from './index-Cd0IZ40L.js';
+export { C as CreateRunInput, E as ExtendedListRunsOptions, a as ExtendedRunStatus, b as ExtendedStepStatus, c as ExtendedWorkflowRunRecord, L as ListEventsOptions, d as ListRunsOptions, P as PaginatedResult, S as StepRecord, e as StepResult, f as StepflowDatabase, g as StepflowRunsTable, h as StepflowStepResultsTable, i as StorageAdapter, U as UpdateRunInput, W as WorkflowEventRecord, j as WorkflowRunRecord, k as WorkflowRunStepRecord, l as WorkflowStorage } from './types-DSEjiLMj.js';
 export { MemoryStorageAdapter, PostgresStorage, PostgresStorage as PostgresStorageAdapter, PostgresStorageConfig, SQLiteStorageAdapter, SQLiteStorageConfig } from './storage/index.js';
-export { B as BuiltInEventType, E as EventCallback, a as EventTransport, U as Unsubscribe, W as WorkflowEvent, b as WorkflowEventType } from './types-DzkEqnQJ.js';
+export { B as BuiltInEventType, E as EventCallback, a as EventTransport, U as Unsubscribe, W as WorkflowEvent, b as WorkflowEventType } from './types-DZUdWmCc.js';
 export { MemoryEventTransport, SocketIOAuthorizeFn, SocketIOEventTransport, SocketIOEventTransportConfig, SocketIOServer, SocketIOSocket, WebhookEndpoint, WebhookEventTransport, WebhookEventTransportConfig, WebhookPayload } from './events/index.js';
 import 'better-sqlite3';
 import 'pg';
+import 'kysely';
 
+/**
+ * ID generation utilities for the workflow engine.
+ */
 /**
  * Generate a unique ID for database records.
  * Uses a ULID-like format: base36 timestamp + random suffix.
@@ -15,7 +19,6 @@ import 'pg';
  * - Rough time-ordering (useful for debugging)
  * - High collision resistance
  * - URL-safe characters
- * - Cryptographically secure random component
  *
  * @returns A unique identifier string (approx 16 characters)
  */
@@ -103,7 +106,7 @@ declare class WorkflowTimeoutError extends WorkflowEngineError {
  */
 
 /** Log level for ConsoleLogger. Levels are ordered: debug < info < warn < error. */
-type LogLevel = "debug" | "info" | "warn" | "error";
+type LogLevel = 'debug' | 'info' | 'warn' | 'error';
 /**
  * Console-based logger implementation.
  * Uses console.log with prefixes for different levels.
@@ -187,7 +190,7 @@ declare function calculateRetryDelay(attempt: number, baseDelay: number, backoff
 /**
  * Comparison operators for recipe conditions.
  */
-type ConditionOperator = "eq" | "neq" | "gt" | "gte" | "lt" | "lte" | "contains" | "matches" | "exists" | "notExists";
+type ConditionOperator = 'eq' | 'neq' | 'gt' | 'gte' | 'lt' | 'lte' | 'contains' | 'matches' | 'exists' | 'notExists';
 /**
  * A condition that determines when a recipe should be selected.
  * Conditions are evaluated against the workflow input.
@@ -291,7 +294,7 @@ interface Recipe {
 /**
  * Types of modifications that can be made to a plan.
  */
-type PlanModificationType = "add_step" | "remove_step" | "modify_step" | "reorder_steps" | "set_default";
+type PlanModificationType = 'add_step' | 'remove_step' | 'modify_step' | 'reorder_steps' | 'set_default';
 /**
  * A modification to apply to a recipe when generating a plan.
  */
@@ -386,7 +389,7 @@ interface Plan {
 /**
  * Priority modes for planning.
  */
-type PlanningPriority = "speed" | "quality" | "cost" | "balanced";
+type PlanningPriority = 'speed' | 'quality' | 'cost' | 'balanced';
 /**
  * Constraints that limit planning and execution.
  */
@@ -486,7 +489,7 @@ interface RegisteredStepHandler<TInput = Record<string, unknown>> {
     /** Human-readable description */
     description?: string;
     /** The handler function */
-    handler: WorkflowStep<TInput>["handler"];
+    handler: WorkflowStep<TInput>['handler'];
     /** Tags for categorization */
     tags?: string[];
 }
@@ -600,7 +603,7 @@ declare class MemoryStepHandlerRegistry implements StepHandlerRegistry {
      * Resolve a handler reference to a WorkflowStep handler function.
      * Returns undefined if the handler is not found.
      */
-    resolve(handlerRef: string): WorkflowStep["handler"] | undefined;
+    resolve(handlerRef: string): WorkflowStep['handler'] | undefined;
     /**
      * Clear all registered handlers (useful for testing).
      */
@@ -645,19 +648,8 @@ declare class MemoryRecipeRegistry implements RecipeRegistry {
     /**
      * Evaluate recipe conditions against an input.
      * Returns true if all conditions match.
-     *
-     * Note: This intentionally mirrors the condition evaluation in planner.ts
-     * to keep registry queries independent of the planner implementation.
      */
     private evaluateConditions;
-    /**
-     * Get a nested value from an object using dot notation.
-     */
-    private getNestedValue;
-    /**
-     * Evaluate a single condition.
-     */
-    private evaluateCondition;
 }
 /**
  * Create a combined registry that holds both recipes and step handlers.