@multiplier-labs/stepflow 0.3.4 → 0.3.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/README.md +9 -0
  2. package/dist/{chunk-U3PCVK6O.js → chunk-D5HW7VEG.js} +4 -2
  3. package/dist/chunk-D5HW7VEG.js.map +1 -0
  4. package/dist/{chunk-5XJBAPXF.js → chunk-JUPZEBR3.js} +3 -1
  5. package/dist/{chunk-5XJBAPXF.js.map → chunk-JUPZEBR3.js.map} +1 -1
  6. package/dist/events/index.d.ts +2 -0
  7. package/dist/events/index.js +1 -1
  8. package/dist/index.js +2 -2
  9. package/dist/storage/index.js +1 -1
  10. package/package.json +23 -12
  11. package/.codekin/reports/code-review/2026-04-11_code-review-daily.md +0 -146
  12. package/.codekin/reports/code-review/2026-04-13_code-review-daily.md +0 -74
  13. package/.codekin/reports/comments/2026-04-04_comment-assessment.md +0 -179
  14. package/.codekin/reports/comments/2026-04-11_comment-assessment.md +0 -161
  15. package/.codekin/reports/comments/2026-04-14_comment-assessment.md +0 -34
  16. package/.codekin/reports/complexity/2026-04-03_complexity-report.md +0 -152
  17. package/.codekin/reports/complexity/2026-04-11_complexity-report.md +0 -152
  18. package/.codekin/reports/complexity/2026-04-14_complexity-report.md +0 -34
  19. package/.codekin/reports/dependencies/2026-04-04_dependency-health.md +0 -94
  20. package/.codekin/reports/dependencies/2026-04-11_dependency-health.md +0 -93
  21. package/.codekin/reports/docs-audit/2026-04-04_docs-audit.md +0 -10
  22. package/.codekin/reports/docs-audit/2026-04-11_docs-audit.md +0 -138
  23. package/.codekin/reports/docs-audit/2026-04-13_docs-audit.md +0 -121
  24. package/.codekin/reports/security/2026-04-03_security-audit.md +0 -208
  25. package/.codekin/reports/security/2026-04-11_security-audit.md +0 -297
  26. package/.github/CODEOWNERS +0 -15
  27. package/.github/dependabot.yml +0 -23
  28. package/.github/workflows/ci.yml +0 -38
  29. package/.github/workflows/publish.yml +0 -42
  30. package/.github/workflows/security.yml +0 -29
  31. package/CHANGELOG.md +0 -34
  32. package/SECURITY.md +0 -40
  33. package/dist/chunk-U3PCVK6O.js.map +0 -1
  34. package/docs/github-packages-setup.md +0 -97
  35. package/docs/stepflow-api-reference.md +0 -1663
  36. package/src/core/engine/concurrency.ts +0 -229
  37. package/src/core/engine/lifecycle.ts +0 -176
  38. package/src/core/engine.test.ts +0 -1435
  39. package/src/core/engine.ts +0 -598
  40. package/src/core/index.ts +0 -7
  41. package/src/core/orchestrator/persistence.ts +0 -165
  42. package/src/core/orchestrator/plan-resolution.ts +0 -140
  43. package/src/core/orchestrator/state-transitions.ts +0 -109
  44. package/src/core/orchestrator/step-runner.ts +0 -427
  45. package/src/core/orchestrator.ts +0 -252
  46. package/src/core/types.ts +0 -220
  47. package/src/events/index.ts +0 -8
  48. package/src/events/memory.test.ts +0 -107
  49. package/src/events/memory.ts +0 -92
  50. package/src/events/socketio.test.ts +0 -378
  51. package/src/events/socketio.ts +0 -262
  52. package/src/events/types.ts +0 -118
  53. package/src/events/webhook.test.ts +0 -949
  54. package/src/events/webhook.ts +0 -807
  55. package/src/index.ts +0 -270
  56. package/src/planning/evaluate-condition.test.ts +0 -153
  57. package/src/planning/evaluate-condition.ts +0 -105
  58. package/src/planning/index.ts +0 -55
  59. package/src/planning/planner.ts +0 -494
  60. package/src/planning/planning.test.ts +0 -727
  61. package/src/planning/registry.test.ts +0 -398
  62. package/src/planning/registry.ts +0 -283
  63. package/src/planning/types.ts +0 -566
  64. package/src/scheduler/cron.test.ts +0 -479
  65. package/src/scheduler/cron.ts +0 -438
  66. package/src/scheduler/index.ts +0 -8
  67. package/src/scheduler/postgres-persistence.ts +0 -530
  68. package/src/scheduler/sqlite-persistence.test.ts +0 -235
  69. package/src/scheduler/sqlite-persistence.ts +0 -286
  70. package/src/scheduler/types.ts +0 -73
  71. package/src/storage/index.ts +0 -27
  72. package/src/storage/memory.test.ts +0 -349
  73. package/src/storage/memory.ts +0 -227
  74. package/src/storage/postgres-core.ts +0 -550
  75. package/src/storage/postgres-migration.test.ts +0 -69
  76. package/src/storage/postgres-transaction-pagination.test.ts +0 -107
  77. package/src/storage/postgres.ts +0 -835
  78. package/src/storage/sqlite-core.ts +0 -151
  79. package/src/storage/sqlite.test.ts +0 -642
  80. package/src/storage/sqlite.ts +0 -717
  81. package/src/storage/types.ts +0 -343
  82. package/src/utils/errors.test.ts +0 -150
  83. package/src/utils/errors.ts +0 -161
  84. package/src/utils/id.ts +0 -21
  85. package/src/utils/index.ts +0 -10
  86. package/src/utils/logger.test.ts +0 -121
  87. package/src/utils/logger.ts +0 -96
  88. package/src/utils/pinned-fetch.test.ts +0 -84
  89. package/src/utils/pinned-fetch.ts +0 -118
  90. package/src/utils/postgres-deps.ts +0 -79
  91. package/src/utils/retry.test.ts +0 -177
  92. package/src/utils/retry.ts +0 -121
  93. package/src/utils/safe-json.test.ts +0 -166
  94. package/src/utils/safe-json.ts +0 -107
  95. package/src/utils/webhook-signature.test.ts +0 -111
  96. package/src/utils/webhook-signature.ts +0 -74
  97. package/tsconfig.json +0 -21
  98. package/tsup.config.ts +0 -14
  99. package/vitest.config.ts +0 -9
@@ -1,1663 +0,0 @@
1
- # Stepflow API Reference
2
-
3
- A durable, type-safe workflow orchestration engine for Node.js with pluggable storage, scheduling, and real-time events.
4
-
5
- ## Table of Contents
6
-
7
- - [Overview](#overview)
8
- - [Installation](#installation)
9
- - [Quick Start](#quick-start)
10
- - [WorkflowEngine](#workflowengine)
11
- - [Workflow Definitions](#workflow-definitions)
12
- - [Step Handlers](#step-handlers)
13
- - [Event System](#event-system)
14
- - [Scheduler](#scheduler)
15
- - [Storage Adapters](#storage-adapters)
16
- - [PostgreSQL Storage Adapter](#postgresql-storage-adapter)
17
- - [Planning System](#planning-system)
18
- - [Error Handling](#error-handling)
19
- - [Utilities](#utilities)
20
- - [TypeScript Types Reference](#typescript-types-reference)
21
- - [Downstream Integration Patterns](#downstream-integration-patterns)
22
-
23
- ---
24
-
25
- ## Overview
26
-
27
- Stepflow provides:
28
-
29
- - **Workflow Definition & Registration** - Define multi-step workflows with typed inputs
30
- - **Async Execution Engine** - Non-blocking, fire-and-forget run execution
31
- - **State Management** - Run/step status tracking with checkpointing for crash recovery
32
- - **Event System** - Real-time events via in-memory, Socket.IO, or webhooks
33
- - **Pluggable Storage** - Memory, SQLite, or PostgreSQL backends
34
- - **Retry & Error Handling** - Configurable per-step error strategies (fail, retry, skip)
35
- - **Scheduling** - Cron-based and workflow-completion triggers
36
- - **Planning** - Rule-based recipe selection and plan generation
37
-
38
- ---
39
-
40
- ## Installation
41
-
42
- ```bash
43
- npm install @multiplier-labs/stepflow
44
- ```
45
-
46
- ### Optional Peer Dependencies
47
-
48
- Install based on your storage backend:
49
-
50
- ```bash
51
- # For SQLite storage
52
- npm install better-sqlite3
53
-
54
- # For PostgreSQL storage
55
- npm install pg kysely
56
- ```
57
-
58
- These are loaded dynamically at runtime, so users of other backends are unaffected.
59
-
60
- ---
61
-
62
- ## Quick Start
63
-
64
- ```typescript
65
- import { WorkflowEngine, MemoryStorageAdapter } from '@multiplier-labs/stepflow';
66
-
67
- const engine = new WorkflowEngine({
68
- storage: new MemoryStorageAdapter(),
69
- });
70
-
71
- engine.registerWorkflow({
72
- kind: 'email.send_campaign',
73
- name: 'Send Email Campaign',
74
- steps: [
75
- {
76
- key: 'load_recipients',
77
- name: 'Load Recipients',
78
- handler: async (ctx) => {
79
- const recipients = await loadRecipients(ctx.input.campaignId);
80
- return { recipients };
81
- },
82
- },
83
- {
84
- key: 'send_emails',
85
- name: 'Send Emails',
86
- handler: async (ctx) => {
87
- const { recipients } = ctx.results.load_recipients;
88
- return { sent: recipients.length };
89
- },
90
- onError: 'retry',
91
- maxRetries: 3,
92
- },
93
- ],
94
- });
95
-
96
- const runId = await engine.startRun({
97
- kind: 'email.send_campaign',
98
- input: { campaignId: '123' },
99
- });
100
-
101
- engine.subscribeToRun(runId, (event) => {
102
- console.log(`${event.eventType}: ${event.stepKey ?? 'run'}`);
103
- });
104
- ```
105
-
106
- ---
107
-
108
- ## WorkflowEngine
109
-
110
- The primary entry point for the library.
111
-
112
- ### Constructor
113
-
114
- ```typescript
115
- import { WorkflowEngine } from '@multiplier-labs/stepflow';
116
-
117
- const engine = new WorkflowEngine(config?: WorkflowEngineConfig);
118
- ```
119
-
120
- ### Configuration
121
-
122
- ```typescript
123
- interface WorkflowEngineConfig {
124
- storage?: StorageAdapter; // default: MemoryStorageAdapter
125
- events?: EventTransport; // default: MemoryEventTransport
126
- logger?: Logger; // default: ConsoleLogger
127
- settings?: {
128
- defaultTimeout?: number; // workflow timeout in ms
129
- maxConcurrency?: number; // max concurrent workflows
130
- };
131
- }
132
- ```
133
-
134
- ### Methods
135
-
136
- #### Lifecycle
137
-
138
- ```typescript
139
- // Initialize storage and event adapters (required before use)
140
- await engine.initialize(): Promise<void>;
141
-
142
- // Graceful shutdown: cancels active runs, closes transports and storage
143
- await engine.shutdown(): Promise<void>;
144
- ```
145
-
146
- #### Workflow Registration
147
-
148
- ```typescript
149
- // Register a workflow definition
150
- engine.registerWorkflow<TInput>(definition: WorkflowDefinition<TInput>): void;
151
-
152
- // Unregister a workflow by kind
153
- engine.unregisterWorkflow(kind: string): boolean;
154
-
155
- // Get a registered workflow definition
156
- engine.getWorkflow(kind: string): WorkflowDefinition | undefined;
157
-
158
- // List all registered workflow kinds
159
- engine.getRegisteredWorkflows(): string[];
160
- ```
161
-
162
- #### Running Workflows
163
-
164
- ```typescript
165
- // Start a new run (non-blocking, returns immediately)
166
- const runId = await engine.startRun<TInput>(options: StartRunOptions<TInput>): Promise<string>;
167
-
168
- // Cancel a running workflow
169
- await engine.cancelRun(runId: string): Promise<void>;
170
-
171
- // Get current run status
172
- const run = await engine.getRunStatus(runId: string): Promise<WorkflowRunRecord | null>;
173
-
174
- // Poll until a run completes (for testing or synchronous use cases)
175
- const run = await engine.waitForRun(runId: string, options?: {
176
- pollInterval?: number; // default: 100ms
177
- timeout?: number; // default: 30000ms
178
- }): Promise<WorkflowRunRecord>;
179
- ```
180
-
181
- #### Resume & Recovery
182
-
183
- ```typescript
184
- // Resume a run from its last checkpoint
185
- const newRunId = await engine.resumeRun(runId: string): Promise<string>;
186
-
187
- // Get all resumable runs (status 'queued' or 'running')
188
- const runs = await engine.getResumableRuns(): Promise<WorkflowRunRecord[]>;
189
-
190
- // Resume all interrupted runs
191
- const runIds = await engine.resumeAllInterrupted(): Promise<string[]>;
192
- ```
193
-
194
- #### Event Subscriptions
195
-
196
- ```typescript
197
- // Subscribe to events for a specific run
198
- const unsubscribe = engine.subscribeToRun(runId: string, callback: EventCallback): Unsubscribe;
199
-
200
- // Subscribe to all workflow events
201
- const unsubscribe = engine.subscribeToAll(callback: EventCallback): Unsubscribe;
202
- ```
203
-
204
- #### Introspection
205
-
206
- ```typescript
207
- engine.getActiveRunCount(): number;
208
- engine.getQueuedRunCount(): number;
209
- engine.getStorage(): StorageAdapter;
210
- engine.getEvents(): EventTransport;
211
- ```
212
-
213
- ### StartRunOptions
214
-
215
- ```typescript
216
- interface StartRunOptions<TInput = Record<string, unknown>> {
217
- kind: string; // workflow type to run
218
- input?: TInput; // input parameters
219
- metadata?: Record<string, unknown>; // optional metadata (userId, etc.)
220
- parentRunId?: string; // parent run ID (for child workflows)
221
- delay?: number; // delay before starting (ms)
222
- priority?: number; // queue priority (higher = runs first, default: 0)
223
- }
224
- ```
225
-
226
- ---
227
-
228
- ## Workflow Definitions
229
-
230
- ### WorkflowDefinition
231
-
232
- ```typescript
233
- interface WorkflowDefinition<TInput = Record<string, unknown>> {
234
- kind: string; // unique workflow type identifier
235
- name: string; // human-readable name
236
- description?: string;
237
- steps: WorkflowStep<TInput>[]; // ordered list of steps
238
- defaultOnError?: StepErrorStrategy; // default: 'fail'
239
- timeout?: number; // workflow timeout in ms
240
- hooks?: WorkflowHooks<TInput>; // lifecycle hooks
241
- }
242
- ```
243
-
244
- ### WorkflowStep
245
-
246
- ```typescript
247
- interface WorkflowStep<TInput = Record<string, unknown>> {
248
- key: string; // unique step identifier within the workflow
249
- name: string; // human-readable name
250
- handler: (ctx: WorkflowContext<TInput>) => Promise<unknown>;
251
- onError?: StepErrorStrategy; // 'fail' | 'retry' | 'skip' (default: inherited)
252
- maxRetries?: number; // default: 3
253
- retryDelay?: number; // default: 1000ms
254
- retryBackoff?: number; // default: 2 (exponential)
255
- timeout?: number; // step timeout in ms
256
- skipIf?: (ctx: WorkflowContext<TInput>) => boolean | Promise<boolean>;
257
- }
258
- ```
259
-
260
- ### WorkflowHooks
261
-
262
- ```typescript
263
- interface WorkflowHooks<TInput = Record<string, unknown>> {
264
- beforeRun?: (ctx: WorkflowContext<TInput>) => Promise<void>;
265
- afterRun?: (ctx: WorkflowContext<TInput>, result: RunResult) => Promise<void>;
266
- beforeStep?: (ctx: WorkflowContext<TInput>, step: WorkflowStep<TInput>) => Promise<void>;
267
- afterStep?: (ctx: WorkflowContext<TInput>, step: WorkflowStep<TInput>, result: unknown) => Promise<void>;
268
- onStepError?: (ctx: WorkflowContext<TInput>, step: WorkflowStep<TInput>, error: Error) => Promise<void>;
269
- }
270
- ```
271
-
272
- ---
273
-
274
- ## Step Handlers
275
-
276
- ### WorkflowContext
277
-
278
- Every step handler receives a `WorkflowContext`:
279
-
280
- ```typescript
281
- interface WorkflowContext<TInput = Record<string, unknown>> {
282
- runId: string; // unique run ID
283
- stepId: string; // unique step ID (for granular tracking)
284
- kind: string; // workflow type
285
- input: TInput; // run input parameters
286
- results: Record<string, unknown>; // accumulated results from prior steps (keyed by step key)
287
- metadata: Record<string, unknown>;// optional metadata
288
- currentStep?: string; // current step key
289
- logger: Logger; // scoped logger
290
- signal: AbortSignal; // for cancellation
291
- spawnChild: (options: SpawnChildOptions) => Promise<string>;
292
- emit: (eventType: string, payload?: unknown) => void;
293
- }
294
- ```
295
-
296
- ### Spawning Child Workflows
297
-
298
- ```typescript
299
- interface SpawnChildOptions {
300
- kind: string;
301
- input?: Record<string, unknown>;
302
- metadata?: Record<string, unknown>;
303
- }
304
-
305
- // Usage inside a step handler:
306
- const childRunId = await ctx.spawnChild({
307
- kind: 'child.workflow',
308
- input: { parentData: ctx.results.step1 },
309
- });
310
- ```
311
-
312
- ### Emitting Custom Events
313
-
314
- ```typescript
315
- // Inside a step handler:
316
- ctx.emit('progress.update', { percent: 50 });
317
- ```
318
-
319
- ### Execution Flow
320
-
321
- 1. Run status set to `'running'`
322
- 2. `run.started` event emitted
323
- 3. `beforeRun` hook called
324
- 4. For each step (in order):
325
- - Cancellation signal checked
326
- - Skipped if in checkpoint (resume) or `skipIf` returns true
327
- - `beforeStep` hook called
328
- - Handler executed with timeout
329
- - Result stored; `afterStep` hook called; `step.completed` event emitted
330
- - On error: `step.failed` event, `onStepError` hook, then fail/retry/skip per strategy
331
- - Checkpoint saved after each successful step
332
- 5. `afterRun` hook called
333
- 6. Run marked `'succeeded'` or `'failed'`
334
-
335
- ---
336
-
337
- ## Event System
338
-
339
- ### Built-in Event Types
340
-
341
- ```typescript
342
- type BuiltInEventType =
343
- | 'run.created' | 'run.queued' | 'run.dequeued'
344
- | 'run.started' | 'run.resumed' | 'run.completed'
345
- | 'run.failed' | 'run.canceled' | 'run.timeout'
346
- | 'step.started' | 'step.completed'
347
- | 'step.failed' | 'step.skipped' | 'step.retry';
348
- ```
349
-
350
- ### WorkflowEvent
351
-
352
- ```typescript
353
- interface WorkflowEvent {
354
- runId: string;
355
- kind: string;
356
- eventType: string; // BuiltInEventType or custom string
357
- stepKey?: string;
358
- timestamp: Date;
359
- payload?: unknown;
360
- }
361
- ```
362
-
363
- ### EventTransport Interface
364
-
365
- ```typescript
366
- interface EventTransport {
367
- emit(event: WorkflowEvent): void;
368
- subscribe(runId: string, callback: EventCallback): Unsubscribe;
369
- subscribeAll(callback: EventCallback): Unsubscribe;
370
- subscribeToType?(eventType: string, callback: EventCallback): Unsubscribe;
371
- persist?(event: WorkflowEvent): Promise<void>;
372
- close?(): void;
373
- }
374
-
375
- type EventCallback = (event: WorkflowEvent) => void;
376
- type Unsubscribe = () => void;
377
- ```
378
-
379
- ### MemoryEventTransport
380
-
381
- Default in-memory transport. Supports `subscribeToType()`.
382
-
383
- ```typescript
384
- import { MemoryEventTransport } from '@multiplier-labs/stepflow';
385
-
386
- const events = new MemoryEventTransport();
387
- events.getListenerCount(channel?: string): number;
388
- ```
389
-
390
- ### SocketIOEventTransport
391
-
392
- Real-time events via Socket.IO.
393
-
394
- ```typescript
395
- import { SocketIOEventTransport } from '@multiplier-labs/stepflow';
396
-
397
- const events = new SocketIOEventTransport({
398
- io: socketIOServer,
399
- eventName: 'workflow:event', // default
400
- roomPrefix: 'run:', // default
401
- broadcastGlobal: true, // default
402
- globalRoom: 'workflow:all', // default
403
- });
404
-
405
- // Set up client handlers for a socket connection
406
- events.setupClientHandlers(socket, async (runId, socket) => {
407
- return /* authorization check */;
408
- });
409
- ```
410
-
411
- **Client events:** `workflow:subscribe`, `workflow:unsubscribe`, `workflow:subscribe:all`, `workflow:unsubscribe:all`
412
-
413
- ### WebhookEventTransport
414
-
415
- Delivers events to HTTP endpoints with HMAC-SHA256 signing.
416
-
417
- ```typescript
418
- import { WebhookEventTransport } from '@multiplier-labs/stepflow';
419
-
420
- const events = new WebhookEventTransport({
421
- endpoints: [
422
- {
423
- id: 'my-webhook',
424
- url: 'https://example.com/webhook',
425
- secret: 'my-secret', // for HMAC-SHA256 signing
426
- eventTypes: ['run.completed'], // filter (optional)
427
- workflowKinds: ['my.workflow'], // filter (optional)
428
- headers: { 'X-Custom': 'val' },
429
- enabled: true, // default: true
430
- timeout: 5000, // default: 5000ms
431
- retries: 3, // default: 3
432
- },
433
- ],
434
- defaultTimeout: 5000, // default
435
- defaultRetries: 3, // default
436
- retryDelay: 1000, // default
437
- fetchFn: fetch, // custom fetch (optional)
438
- });
439
-
440
- events.addEndpoint(endpoint);
441
- events.removeEndpoint(id: string): boolean;
442
- events.getEndpoints(): WebhookEndpoint[];
443
- events.setEndpointEnabled(id: string, enabled: boolean): void;
444
- ```
445
-
446
- **Webhook payload:**
447
-
448
- ```typescript
449
- interface WebhookPayload {
450
- event: { runId, kind, eventType, stepKey?, payload?, timestamp: string };
451
- deliveredAt: string;
452
- webhookId: string;
453
- }
454
- ```
455
-
456
- Signed with `X-Webhook-Signature` header (HMAC-SHA256).
457
-
458
- ---
459
-
460
- ## Scheduler
461
-
462
- ### CronScheduler
463
-
464
- ```typescript
465
- import { CronScheduler } from '@multiplier-labs/stepflow';
466
-
467
- const scheduler = new CronScheduler({
468
- engine: workflowEngine,
469
- logger: myLogger, // optional
470
- pollInterval: 1000, // default: 1000ms
471
- persistence: myPersistence, // optional SchedulePersistence
472
- });
473
-
474
- await scheduler.start();
475
-
476
- // Add a cron schedule
477
- const schedule = await scheduler.addSchedule({
478
- workflowKind: 'reports.generate',
479
- triggerType: 'cron',
480
- cronExpression: '0 6 * * *', // daily at 6 AM
481
- timezone: 'UTC',
482
- input: { format: 'pdf' },
483
- enabled: true,
484
- });
485
-
486
- // Add a workflow-completion trigger
487
- await scheduler.addSchedule({
488
- workflowKind: 'cleanup.run',
489
- triggerType: 'workflow_completed',
490
- triggerOnWorkflowKind: 'data.import',
491
- triggerOnStatus: ['succeeded'],
492
- enabled: true,
493
- });
494
-
495
- // Manage schedules
496
- await scheduler.removeSchedule(scheduleId);
497
- await scheduler.updateSchedule(scheduleId, { enabled: false });
498
- const schedules = await scheduler.getSchedules();
499
- const runId = await scheduler.triggerNow(scheduleId);
500
-
501
- await scheduler.stop();
502
- ```
503
-
504
- ### WorkflowSchedule
505
-
506
- ```typescript
507
- interface WorkflowSchedule {
508
- id: string;
509
- workflowKind: string;
510
- triggerType: 'cron' | 'workflow_completed' | 'manual';
511
- cronExpression?: string;
512
- timezone?: string;
513
- triggerOnWorkflowKind?: string;
514
- triggerOnStatus?: RunStatus[];
515
- input?: Record<string, unknown>;
516
- metadata?: Record<string, unknown>;
517
- enabled: boolean;
518
- lastRunAt?: Date;
519
- lastRunId?: string;
520
- nextRunAt?: Date;
521
- }
522
- ```
523
-
524
- ### SchedulePersistence Interface
525
-
526
- ```typescript
527
- interface SchedulePersistence {
528
- loadSchedules(): Promise<WorkflowSchedule[]>;
529
- saveSchedule(schedule: WorkflowSchedule): Promise<void>;
530
- updateSchedule(scheduleId: string, updates: Partial<WorkflowSchedule>): Promise<void>;
531
- deleteSchedule(scheduleId: string): Promise<void>;
532
- }
533
- ```
534
-
535
- ### SQLiteSchedulePersistence
536
-
537
- ```typescript
538
- import { SQLiteSchedulePersistence } from '@multiplier-labs/stepflow';
539
-
540
- const persistence = new SQLiteSchedulePersistence({
541
- db: sqliteDatabase,
542
- tableName: 'workflow_schedules', // default
543
- });
544
- ```
545
-
546
- ### PostgresSchedulePersistence
547
-
548
- ```typescript
549
- import { PostgresSchedulePersistence } from '@multiplier-labs/stepflow';
550
-
551
- const persistence = new PostgresSchedulePersistence({
552
- connectionString: process.env.DATABASE_URL,
553
- // OR pool: existingPgPool,
554
- // OR poolConfig: { ... },
555
- schema: 'public', // default
556
- tableName: 'workflow_schedules', // default
557
- autoMigrate: true, // default
558
- });
559
-
560
- await persistence.initialize();
561
-
562
- // Additional query methods:
563
- await persistence.getSchedule(scheduleId): Promise<WorkflowSchedule | null>;
564
- await persistence.getDueSchedules(): Promise<WorkflowSchedule[]>;
565
- await persistence.getSchedulesByWorkflowKind(kind): Promise<WorkflowSchedule[]>;
566
- await persistence.getCompletionTriggers(triggerOnKind): Promise<WorkflowSchedule[]>;
567
-
568
- await persistence.close();
569
- ```
570
-
571
- ---
572
-
573
- ## Storage Adapters
574
-
575
- All storage adapters implement the `StorageAdapter` interface.
576
-
577
- ### StorageAdapter Interface
578
-
579
- ```typescript
580
- interface StorageAdapter {
581
- // Run operations
582
- createRun(run: Omit<WorkflowRunRecord, 'id' | 'createdAt'>): Promise<WorkflowRunRecord>;
583
- getRun(runId: string): Promise<WorkflowRunRecord | null>;
584
- updateRun(runId: string, updates: Partial<WorkflowRunRecord>): Promise<void>;
585
- listRuns(options?: ListRunsOptions): Promise<PaginatedResult<WorkflowRunRecord>>;
586
-
587
- // Step operations
588
- createStep(step: Omit<WorkflowRunStepRecord, 'id'>): Promise<WorkflowRunStepRecord>;
589
- getStep(stepId: string): Promise<WorkflowRunStepRecord | null>;
590
- updateStep(stepId: string, updates: Partial<WorkflowRunStepRecord>): Promise<void>;
591
- getStepsForRun(runId: string): Promise<WorkflowRunStepRecord[]>;
592
-
593
- // Event operations
594
- saveEvent(event: Omit<WorkflowEventRecord, 'id'>): Promise<void>;
595
- getEventsForRun(runId: string, options?: ListEventsOptions): Promise<WorkflowEventRecord[]>;
596
-
597
- // Optional
598
- transaction?<T>(fn: (tx: StorageAdapter) => Promise<T>): Promise<T>;
599
- deleteOldRuns?(olderThan: Date): Promise<number>;
600
- initialize?(): Promise<void>;
601
- close?(): void | Promise<void>;
602
- }
603
- ```
604
-
605
- ### WorkflowStorage Interface
606
-
607
- A newer storage interface with additional methods for queue management and step results. The `PostgresStorageAdapter` implements both `StorageAdapter` and `WorkflowStorage`.
608
-
609
- ```typescript
610
- interface WorkflowStorage {
611
- // Run operations
612
- createRun(run: CreateRunInput): Promise<ExtendedWorkflowRunRecord>;
613
- getRun(id: string): Promise<ExtendedWorkflowRunRecord | null>;
614
- updateRun(id: string, updates: UpdateRunInput): Promise<void>;
615
- listRuns(options?: ExtendedListRunsOptions): Promise<PaginatedResult<ExtendedWorkflowRunRecord>>;
616
- deleteRun(id: string): Promise<void>;
617
-
618
- // Atomic dequeue
619
- dequeueRun(workflowKinds: string[]): Promise<ExtendedWorkflowRunRecord | null>;
620
-
621
- // Stale run management
622
- cleanupStaleRuns(defaultTimeoutMs?: number): Promise<number>;
623
- markRunsAsFailed(runIds: string[], reason: string): Promise<void>;
624
-
625
- // Step results (uses stepflow_step_results table)
626
- getStepResult(runId: string, stepName: string): Promise<StepResult | undefined>;
627
- getStepResults(runId: string): Promise<StepResult[]>;
628
- getStepsForRun(runId: string): Promise<StepRecord[]>;
629
- saveStepResult(result: Omit<StepResult, 'id'> & { id?: string }): Promise<void>;
630
-
631
- // Lifecycle
632
- initialize(): Promise<void>;
633
- close(): Promise<void>;
634
- }
635
- ```
636
-
637
- ### Core Record Types
638
-
639
- ```typescript
640
- interface WorkflowRunRecord {
641
- id: string;
642
- kind: string;
643
- status: RunStatus; // 'queued' | 'running' | 'succeeded' | 'failed' | 'canceled'
644
- parentRunId?: string;
645
- input: Record<string, unknown>;
646
- context: Record<string, unknown>; // checkpoint: accumulated step results
647
- output?: Record<string, unknown>;
648
- error?: { code: string; message: string };
649
- metadata?: Record<string, unknown>;
650
- priority?: number;
651
- timeoutMs?: number;
652
- createdAt: Date;
653
- startedAt?: Date;
654
- finishedAt?: Date;
655
- }
656
-
657
- interface WorkflowRunStepRecord {
658
- id: string;
659
- runId: string;
660
- stepKey: string;
661
- stepName: string;
662
- status: StepStatus; // 'pending' | 'running' | 'succeeded' | 'failed' | 'skipped' | 'canceled'
663
- attempt: number;
664
- result?: unknown;
665
- error?: WorkflowError;
666
- startedAt?: Date;
667
- finishedAt?: Date;
668
- }
669
-
670
- interface WorkflowEventRecord {
671
- id: string;
672
- runId: string;
673
- stepKey?: string;
674
- eventType: string;
675
- level: 'info' | 'warn' | 'error';
676
- payload?: unknown;
677
- timestamp: Date;
678
- }
679
- ```
680
-
681
- ### Extended Types
682
-
683
- Used by the `WorkflowStorage` interface and `PostgresStorageAdapter`:
684
-
685
- ```typescript
686
- // Adds 'pending' and 'timeout' to core RunStatus
687
- type ExtendedRunStatus = 'pending' | 'queued' | 'running' | 'succeeded' | 'failed' | 'canceled' | 'timeout';
688
-
689
- // Uses 'completed' instead of 'succeeded'
690
- type ExtendedStepStatus = 'pending' | 'running' | 'completed' | 'failed' | 'skipped';
691
-
692
- interface StepResult {
693
- id: string;
694
- runId: string;
695
- stepName: string;
696
- status: ExtendedStepStatus;
697
- output?: Record<string, unknown>;
698
- error?: Record<string, unknown>;
699
- attempt: number;
700
- startedAt?: Date;
701
- completedAt?: Date;
702
- }
703
-
704
- interface StepRecord {
705
- stepKey: string;
706
- stepName: string;
707
- status: ExtendedStepStatus;
708
- result?: Record<string, unknown>;
709
- error?: Record<string, unknown>;
710
- attempt: number;
711
- startedAt?: Date;
712
- finishedAt?: Date;
713
- }
714
- ```
715
-
716
- ### Query Options
717
-
718
- ```typescript
719
- interface ListRunsOptions {
720
- kind?: string;
721
- status?: RunStatus | RunStatus[];
722
- parentRunId?: string;
723
- limit?: number; // default: 50
724
- offset?: number; // default: 0
725
- orderBy?: 'createdAt' | 'startedAt' | 'finishedAt';
726
- orderDirection?: 'asc' | 'desc'; // default: 'desc'
727
- }
728
-
729
- interface ExtendedListRunsOptions {
730
- kind?: string;
731
- status?: ExtendedRunStatus | ExtendedRunStatus[];
732
- limit?: number;
733
- offset?: number;
734
- orderBy?: 'createdAt' | 'startedAt' | 'finishedAt';
735
- orderDir?: 'asc' | 'desc';
736
- }
737
-
738
- interface ListEventsOptions {
739
- stepKey?: string;
740
- level?: 'info' | 'warn' | 'error';
741
- limit?: number; // default: 1000
742
- offset?: number;
743
- }
744
-
745
- interface PaginatedResult<T> {
746
- items: T[];
747
- total: number;
748
- limit?: number;
749
- offset?: number;
750
- }
751
- ```
752
-
753
- ### MemoryStorageAdapter
754
-
755
- In-memory storage. Data is lost on process exit.
756
-
757
- ```typescript
758
- import { MemoryStorageAdapter } from '@multiplier-labs/stepflow';
759
-
760
- const storage = new MemoryStorageAdapter();
761
-
762
- // Testing utilities
763
- storage.clear(): void;
764
- storage.getStats(): { runs: number; steps: number; events: number };
765
- ```
766
-
767
- ### SQLiteStorageAdapter
768
-
769
- File-based or in-memory SQLite storage.
770
-
771
- ```typescript
772
- import { SQLiteStorageAdapter } from '@multiplier-labs/stepflow';
773
- import Database from 'better-sqlite3';
774
-
775
- const db = new Database('./workflows.db');
776
- const storage = new SQLiteStorageAdapter({
777
- db,
778
- autoCreateTables: true, // default: true
779
- tablePrefix: 'workflow', // default: 'workflow'
780
- });
781
-
782
- // Additional methods (implementation-only; not part of the StorageAdapter interface)
783
- storage.transactionSync<T>(fn: () => T): T;
784
- await storage.getInterruptedRuns(): Promise<WorkflowRunRecord[]>;
785
- await storage.getLastCompletedStep(runId): Promise<WorkflowRunStepRecord | null>;
786
- storage.close(): void;
787
- storage.getStats(): { runs: number; steps: number; events: number };
788
- ```
789
-
790
- **Tables created:** `workflow_runs`, `workflow_run_steps`, `workflow_events`
791
-
792
- ---
793
-
794
- ## PostgreSQL Storage Adapter
795
-
796
- Production-ready PostgreSQL backend with connection pooling, distributed worker support, and atomic dequeue.
797
-
798
- ### Configuration
799
-
800
- ```typescript
801
- import { PostgresStorageAdapter } from '@multiplier-labs/stepflow';
802
-
803
- interface PostgresStorageConfig {
804
- connectionString?: string; // e.g., "postgresql://user:pass@localhost:5432/dbname"
805
- pool?: pg.Pool; // existing pool for connection sharing
806
- poolConfig?: pg.PoolConfig; // pool configuration options
807
- schema?: string; // schema name (default: 'public')
808
- autoMigrate?: boolean; // auto-create tables (default: true)
809
- }
810
- ```
811
-
812
- ### Usage
813
-
814
- ```typescript
815
- // Option 1: Connection string
816
- const storage = new PostgresStorageAdapter({
817
- connectionString: process.env.DATABASE_URL,
818
- schema: 'stepflow',
819
- });
820
- await storage.initialize();
821
-
822
- // Option 2: Existing pool (for connection sharing)
823
- import pg from 'pg';
824
- const pool = new pg.Pool({ connectionString: process.env.DATABASE_URL, max: 20 });
825
- const storage = new PostgresStorageAdapter({ pool, schema: 'stepflow' });
826
- await storage.initialize();
827
-
828
- // Use with engine
829
- const engine = new WorkflowEngine({ storage });
830
- await engine.initialize();
831
- ```
832
-
833
- The adapter only closes the pool on `close()` if it created it. Externally-provided pools are left open.
834
-
835
- ### All Methods
836
-
837
- The adapter implements both `StorageAdapter` and `WorkflowStorage`, providing:
838
-
839
- **StorageAdapter methods (legacy, used by WorkflowEngine):**
840
-
841
- ```typescript
842
- await storage.createRun(run): Promise<WorkflowRunRecord>;
843
- await storage.getRun(runId): Promise<WorkflowRunRecord | null>;
844
- await storage.updateRun(runId, updates): Promise<void>;
845
- await storage.listRuns(options?): Promise<PaginatedResult<WorkflowRunRecord>>;
846
- await storage.createStep(step): Promise<WorkflowRunStepRecord>;
847
- await storage.getStep(stepId): Promise<WorkflowRunStepRecord | null>;
848
- await storage.updateStep(stepId, updates): Promise<void>;
849
- await storage.getStepsForRun(runId): Promise<WorkflowRunStepRecord[]>;
850
- await storage.saveEvent(event): Promise<void>;
851
- await storage.getEventsForRun(runId, options?): Promise<WorkflowEventRecord[]>;
852
- await storage.transaction<T>(fn): Promise<T>;
853
- await storage.deleteOldRuns(olderThan): Promise<number>;
854
- await storage.initialize(): Promise<void>;
855
- await storage.close(): Promise<void>;
856
- ```
857
-
858
- **WorkflowStorage methods (used for queue operations and step results):**
859
-
860
- ```typescript
861
- await storage.deleteRun(id): Promise<void>;
862
- await storage.dequeueRun(workflowKinds?): Promise<WorkflowRunRecord | null>;
863
- await storage.cleanupStaleRuns(defaultTimeoutMs?): Promise<number>;
864
- await storage.markRunsAsFailed(runIds, reason): Promise<void>;
865
- await storage.getStepResult(runId, stepName): Promise<StepResult | undefined>;
866
- await storage.getStepResults(runId): Promise<StepResult[]>;
867
- await storage.saveStepResult(result): Promise<void>;
868
- ```
869
-
870
- **Resume support** (implementation-only; not part of the StorageAdapter interface):
871
-
872
- ```typescript
873
- await storage.getInterruptedRuns(): Promise<WorkflowRunRecord[]>;
874
- await storage.getLastCompletedStep(runId): Promise<WorkflowRunStepRecord | null>;
875
- ```
876
-
877
- **Diagnostics:**
878
-
879
- ```typescript
880
- await storage.getStats(): Promise<{ runs: number; steps: number; events: number }>;
881
- ```
882
-
883
- ### Atomic Dequeue
884
-
885
- Safe concurrent processing by multiple workers using `FOR UPDATE SKIP LOCKED`:
886
-
887
- ```typescript
888
- const run = await storage.dequeueRun(['sync.xero', 'sync.directo']);
889
- if (run) {
890
- // run.status is now 'running', startedAt is set
891
- console.log(`Processing ${run.kind}: ${run.id}`);
892
- }
893
- ```
894
-
895
- The dequeue query:
896
-
897
- ```sql
898
- UPDATE {schema}.runs
899
- SET status = 'running', started_at = NOW()
900
- WHERE id = (
901
- SELECT id FROM {schema}.runs
902
- WHERE status = 'queued' AND kind = ANY($1::text[])
903
- ORDER BY priority DESC, created_at ASC
904
- LIMIT 1
905
- FOR UPDATE SKIP LOCKED
906
- )
907
- RETURNING *;
908
- ```
909
-
910
- This ensures: one worker per run, priority ordering, FIFO within same priority, no blocking between workers.
911
-
912
- ### Stale Run Cleanup
913
-
914
- ```typescript
915
- // Mark runs as timed out if they exceed their timeout_ms (or a default of 10 minutes)
916
- const count = await storage.cleanupStaleRuns(600000);
917
-
918
- // Mark specific runs as failed (e.g., during graceful shutdown)
919
- await storage.markRunsAsFailed([runId1, runId2], 'Worker process was terminated');
920
- ```
921
-
922
- ### Step Results (WorkflowStorage Interface)
923
-
924
- ```typescript
925
- // Save or upsert a step result (uses ON CONFLICT DO UPDATE on run_id + step_name)
926
- await storage.saveStepResult({
927
- runId: run.id,
928
- stepName: 'xero.accounts',
929
- status: 'running',
930
- attempt: 1,
931
- startedAt: new Date(),
932
- });
933
-
934
- // Get a single step result
935
- const step = await storage.getStepResult(runId, 'xero.accounts');
936
-
937
- // Get all step results for a run
938
- const steps = await storage.getStepResults(runId);
939
- ```
940
-
941
- ### Database Schema
942
-
943
- When `autoMigrate` is enabled (default), the adapter creates the following tables. All table names are prefixed with the configured `schema` (default: `'public'`).
944
-
945
- #### Table: `{schema}.runs`
946
-
947
- ```sql
948
- CREATE TABLE IF NOT EXISTS {schema}.runs (
949
- id TEXT PRIMARY KEY,
950
- kind TEXT NOT NULL,
951
- status TEXT NOT NULL,
952
- parent_run_id TEXT,
953
- input_json JSONB NOT NULL DEFAULT '{}',
954
- metadata_json JSONB NOT NULL DEFAULT '{}',
955
- context_json JSONB NOT NULL DEFAULT '{}',
956
- output_json JSONB,
957
- error_json JSONB,
958
- priority INTEGER NOT NULL DEFAULT 0,
959
- timeout_ms INTEGER,
960
- created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
961
- started_at TIMESTAMPTZ,
962
- finished_at TIMESTAMPTZ,
963
- CONSTRAINT runs_status_check CHECK (
964
- status IN ('pending', 'queued', 'running', 'succeeded', 'failed', 'canceled', 'timeout')
965
- )
966
- );
967
-
968
- CREATE INDEX idx_runs_kind_status ON {schema}.runs (kind, status);
969
- CREATE INDEX idx_runs_parent ON {schema}.runs (parent_run_id);
970
- CREATE INDEX idx_runs_created ON {schema}.runs (created_at DESC);
971
- CREATE INDEX idx_runs_status ON {schema}.runs (status);
972
- CREATE INDEX idx_runs_priority ON {schema}.runs (priority DESC, created_at ASC);
973
- ```
974
-
975
- #### Table: `{schema}.workflow_run_steps`
976
-
977
- Used by the `StorageAdapter` interface (legacy step tracking by the engine).
978
-
979
- ```sql
980
- CREATE TABLE IF NOT EXISTS {schema}.workflow_run_steps (
981
- id TEXT PRIMARY KEY,
982
- run_id TEXT NOT NULL REFERENCES {schema}.runs (id) ON DELETE CASCADE,
983
- step_key TEXT NOT NULL,
984
- step_name TEXT,
985
- status TEXT NOT NULL,
986
- attempt INTEGER NOT NULL DEFAULT 1,
987
- result_json JSONB,
988
- error_json JSONB,
989
- started_at TIMESTAMPTZ,
990
- finished_at TIMESTAMPTZ,
991
- CONSTRAINT workflow_run_steps_status_check CHECK (
992
- status IN ('pending', 'running', 'succeeded', 'failed', 'skipped', 'completed')
993
- )
994
- );
995
-
996
- CREATE INDEX idx_workflow_run_steps_run ON {schema}.workflow_run_steps (run_id);
997
- CREATE INDEX idx_workflow_run_steps_run_key ON {schema}.workflow_run_steps (run_id, step_key);
998
- ```
999
-
1000
- #### Table: `{schema}.stepflow_step_results`
1001
-
1002
- Used by the `WorkflowStorage` interface (step result tracking for external consumers).
1003
-
1004
- ```sql
1005
- CREATE TABLE IF NOT EXISTS {schema}.stepflow_step_results (
1006
- id TEXT PRIMARY KEY DEFAULT gen_random_uuid()::text,
1007
- run_id TEXT NOT NULL REFERENCES {schema}.runs (id) ON DELETE CASCADE,
1008
- step_name TEXT NOT NULL,
1009
- status TEXT NOT NULL,
1010
- output_json JSONB,
1011
- error_json JSONB,
1012
- attempt INTEGER NOT NULL DEFAULT 1,
1013
- started_at TIMESTAMPTZ,
1014
- completed_at TIMESTAMPTZ,
1015
- CONSTRAINT stepflow_step_results_status_check CHECK (
1016
- status IN ('pending', 'running', 'completed', 'failed', 'skipped')
1017
- ),
1018
- UNIQUE (run_id, step_name)
1019
- );
1020
-
1021
- CREATE INDEX idx_stepflow_step_results_run ON {schema}.stepflow_step_results (run_id);
1022
- CREATE INDEX idx_stepflow_step_results_run_name ON {schema}.stepflow_step_results (run_id, step_name);
1023
- ```
1024
-
1025
- #### Table: `{schema}.workflow_events`
1026
-
1027
- ```sql
1028
- CREATE TABLE IF NOT EXISTS {schema}.workflow_events (
1029
- id TEXT PRIMARY KEY,
1030
- run_id TEXT NOT NULL REFERENCES {schema}.runs (id) ON DELETE CASCADE,
1031
- step_key TEXT,
1032
- event_type TEXT NOT NULL,
1033
- level TEXT NOT NULL,
1034
- payload_json JSONB,
1035
- timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW(),
1036
- CONSTRAINT workflow_events_level_check CHECK (
1037
- level IN ('info', 'warn', 'error')
1038
- )
1039
- );
1040
-
1041
- CREATE INDEX idx_workflow_events_run ON {schema}.workflow_events (run_id);
1042
- CREATE INDEX idx_workflow_events_run_ts ON {schema}.workflow_events (run_id, timestamp);
1043
- ```
1044
-
1045
- ### Implementation Notes
1046
-
1047
- - **IDs are TEXT, not UUID.** The adapter uses `generateId()` which produces ULID-like base36 strings (~16 characters) for time-ordering and collision resistance.
1048
- - **JSONB columns use `_json` suffix** (e.g., `input_json`, `output_json`, `error_json`) to distinguish from the camelCase TypeScript field names.
1049
- - **`finished_at` (not `completed_at`)** is used on the `runs` table. The `stepflow_step_results` table uses `completed_at`.
1050
- - **Lazy dependency loading.** `pg` and `kysely` are imported dynamically in `initialize()`, so they're only needed if you actually use the PostgreSQL adapter.
1051
- - **Schema isolation.** All queries use `db.withSchema(schema)` to support custom schemas and multi-tenant deployments.
1052
- - **Transaction support.** `storage.transaction(fn)` wraps operations in a PostgreSQL transaction via Kysely.
1053
-
1054
- ### SQL Queries for Debugging
1055
-
1056
- ```sql
1057
- -- Recent workflow runs
1058
- SELECT id, kind, status, created_at, started_at, finished_at,
1059
- error_json->>'message' as error_message
1060
- FROM {schema}.runs
1061
- ORDER BY created_at DESC
1062
- LIMIT 20;
1063
-
1064
- -- Step results for a run
1065
- SELECT step_name, status, output_json, error_json, started_at, completed_at
1066
- FROM {schema}.stepflow_step_results
1067
- WHERE run_id = 'your-run-id'
1068
- ORDER BY started_at;
1069
-
1070
- -- Count by status
1071
- SELECT status, COUNT(*) FROM {schema}.runs GROUP BY status;
1072
-
1073
- -- Find stale running workflows
1074
- SELECT id, kind, started_at, timeout_ms
1075
- FROM {schema}.runs
1076
- WHERE status = 'running'
1077
- AND started_at < NOW() - INTERVAL '10 minutes';
1078
-
1079
- -- Queue depth
1080
- SELECT kind, COUNT(*) FROM {schema}.runs
1081
- WHERE status = 'queued' GROUP BY kind;
1082
- ```
1083
-
1084
- ---
1085
-
1086
- ## Planning System
1087
-
1088
- The planning system enables dynamic workflow construction from recipes and step handler registries.
1089
-
1090
- ### Recipes
1091
-
1092
- A recipe is a template for building a workflow plan:
1093
-
1094
- ```typescript
1095
- interface Recipe {
1096
- id: string;
1097
- name: string;
1098
- description?: string;
1099
- workflowKind: string;
1100
- variant: string; // e.g., 'default', 'fast', 'thorough'
1101
- steps: RecipeStep[];
1102
- defaults?: RecipeDefaults;
1103
- conditions?: RecipeCondition[]; // conditions for recipe selection
1104
- priority?: number; // higher = preferred
1105
- tags?: string[];
1106
- }
1107
-
1108
- interface RecipeStep {
1109
- key: string;
1110
- name: string;
1111
- handlerRef: string; // reference to a registered step handler
1112
- config?: Record<string, unknown>;
1113
- onError?: StepErrorStrategy;
1114
- maxRetries?: number;
1115
- retryDelay?: number;
1116
- retryBackoff?: number;
1117
- timeout?: number;
1118
- skipCondition?: string; // field reference (dot notation)
1119
- }
1120
-
1121
- interface RecipeDefaults {
1122
- timeout?: number;
1123
- maxRetries?: number;
1124
- retryDelay?: number;
1125
- retryBackoff?: number;
1126
- onError?: StepErrorStrategy;
1127
- }
1128
- ```
1129
-
1130
- ### Conditions
1131
-
1132
- ```typescript
1133
- type ConditionOperator =
1134
- | 'eq' | 'neq' | 'gt' | 'gte' | 'lt' | 'lte'
1135
- | 'contains' | 'matches' | 'exists' | 'notExists';
1136
-
1137
- interface RecipeCondition {
1138
- field: string; // supports dot notation
1139
- operator: ConditionOperator;
1140
- value?: unknown;
1141
- }
1142
- ```
1143
-
1144
- ### Registries
1145
-
1146
- ```typescript
1147
- import { MemoryRecipeRegistry, MemoryStepHandlerRegistry, createRegistry } from '@multiplier-labs/stepflow';
1148
-
1149
- // Create both registries at once
1150
- const { recipes, handlers } = createRegistry();
1151
-
1152
- // Register step handlers
1153
- handlers.register({
1154
- id: 'fetch-data',
1155
- name: 'Fetch Data',
1156
- description: 'Fetches data from API',
1157
- tags: ['io', 'api'],
1158
- handler: async (ctx) => { /* ... */ },
1159
- });
1160
-
1161
- // Register recipes
1162
- recipes.register({
1163
- id: 'data-pipeline-default',
1164
- name: 'Data Pipeline',
1165
- workflowKind: 'data.pipeline',
1166
- variant: 'default',
1167
- steps: [
1168
- { key: 'fetch', name: 'Fetch', handlerRef: 'fetch-data' },
1169
- ],
1170
- });
1171
-
1172
- // Query recipes
1173
- recipes.getByKind('data.pipeline'): Recipe[];
1174
- recipes.getVariant('data.pipeline', 'fast'): Recipe | undefined;
1175
- recipes.getDefault('data.pipeline'): Recipe | undefined;
1176
- recipes.listVariants('data.pipeline'): string[];
1177
- recipes.query({ workflowKind: 'data.pipeline', tags: ['fast'] }): Recipe[];
1178
- ```
1179
-
1180
- ### RuleBasedPlanner
1181
-
1182
- Selects recipes based on conditions and generates execution plans:
1183
-
1184
- ```typescript
1185
- import { RuleBasedPlanner } from '@multiplier-labs/stepflow';
1186
-
1187
- const planner = new RuleBasedPlanner({
1188
- recipeRegistry: recipes,
1189
- handlerRegistry: handlers, // optional
1190
- validateHandlers: true, // validate handler refs exist
1191
- });
1192
-
1193
- // Select best recipe for given input
1194
- const selection = await planner.selectRecipe('data.pipeline', { size: 'large' });
1195
-
1196
- // Generate a full plan
1197
- const plan = await planner.plan('data.pipeline', { size: 'large' }, {
1198
- constraints: { maxDuration: 60000, priority: 'speed' },
1199
- hints: { preferredVariant: 'fast', skipSteps: ['optional-step'] },
1200
- });
1201
-
1202
- // Validate and estimate
1203
- const validation = planner.validatePlan(plan);
1204
- const estimate = planner.estimateResources(plan);
1205
- ```
1206
-
1207
- ### Plan Output
1208
-
1209
- ```typescript
1210
- interface Plan {
1211
- id: string;
1212
- recipeId: string;
1213
- variant: string;
1214
- modifications: PlanModification[];
1215
- steps: PlannedStep[];
1216
- childWorkflows?: ChildWorkflowPlan[];
1217
- defaults: RecipeDefaults;
1218
- reasoning?: string;
1219
- resourceEstimate?: ResourceEstimate;
1220
- createdAt: Date;
1221
- }
1222
-
1223
- interface ResourceEstimate {
1224
- apiCalls?: number;
1225
- tokens?: number;
1226
- duration?: number;
1227
- memory?: number;
1228
- }
1229
- ```
1230
-
1231
- ---
1232
-
1233
- ## Error Handling
1234
-
1235
- ### Error Classes
1236
-
1237
- ```typescript
1238
- import {
1239
- WorkflowEngineError,
1240
- WorkflowNotFoundError,
1241
- WorkflowAlreadyRegisteredError,
1242
- RunNotFoundError,
1243
- StepError,
1244
- StepTimeoutError,
1245
- WorkflowCanceledError,
1246
- WorkflowTimeoutError,
1247
- WaitForRunTimeoutError,
1248
- } from '@multiplier-labs/stepflow';
1249
- ```
1250
-
1251
- | Class | Code | Constructor |
1252
- |-------|------|-------------|
1253
- | `WorkflowEngineError` | (custom) | `(code, message, details?)` |
1254
- | `WorkflowNotFoundError` | `WORKFLOW_NOT_FOUND` | `(kind)` |
1255
- | `WorkflowAlreadyRegisteredError` | `WORKFLOW_ALREADY_REGISTERED` | `(kind)` |
1256
- | `RunNotFoundError` | `RUN_NOT_FOUND` | `(runId)` |
1257
- | `StepError` | `STEP_ERROR` | `(stepKey, message, attempt, cause?)` |
1258
- | `StepTimeoutError` | `STEP_TIMEOUT` | `(stepKey, timeoutMs)` |
1259
- | `WorkflowCanceledError` | `WORKFLOW_CANCELED` | `(runId)` |
1260
- | `WorkflowTimeoutError` | `WORKFLOW_TIMEOUT` | `(runId, timeoutMs)` |
1261
- | `WaitForRunTimeoutError` | `WAIT_FOR_RUN_TIMEOUT` | `(runId, timeoutMs)` |
1262
-
1263
- ### WorkflowError Record
1264
-
1265
- ```typescript
1266
- interface WorkflowError {
1267
- code: string;
1268
- message: string;
1269
- stack?: string;
1270
- details?: Record<string, unknown>;
1271
- }
1272
-
1273
- // Convert error instances
1274
- const record = engineError.toRecord();
1275
- const record = WorkflowEngineError.fromError(unknownError, 'DEFAULT_CODE');
1276
- ```
1277
-
1278
- ### Step Error Strategies
1279
-
1280
- | Strategy | Behavior |
1281
- |----------|----------|
1282
- | `'fail'` | Stop the workflow immediately (default) |
1283
- | `'retry'` | Retry up to `maxRetries` times with exponential backoff |
1284
- | `'skip'` | Mark step as skipped and continue to next step |
1285
-
1286
- ### RunResult
1287
-
1288
- ```typescript
1289
- interface RunResult {
1290
- status: 'succeeded' | 'failed' | 'canceled';
1291
- results: Record<string, unknown>;
1292
- error?: WorkflowError;
1293
- duration: number; // ms
1294
- }
1295
- ```
1296
-
1297
- ---
1298
-
1299
- ## Utilities
1300
-
1301
- ### ID Generation
1302
-
1303
- ```typescript
1304
- import { generateId } from '@multiplier-labs/stepflow';
1305
-
1306
- const id = generateId();
1307
- // ULID-like format: base36 timestamp + random suffix (~16 characters)
1308
- // Time-ordered, collision-resistant, URL-safe
1309
- ```
1310
-
1311
- ### Logger
1312
-
1313
- ```typescript
1314
- import { ConsoleLogger, SilentLogger, createScopedLogger } from '@multiplier-labs/stepflow';
1315
-
1316
- // Logs to console with prefix
1317
- const logger = new ConsoleLogger('[my-app]'); // default prefix: '[workflow]'
1318
-
1319
- // No-op logger
1320
- const silent = new SilentLogger();
1321
-
1322
- // Create a scoped logger (prefixes messages with runId/stepKey)
1323
- const scoped = createScopedLogger(logger, runId, stepKey?);
1324
-
1325
- // Logger interface (compatible with pino, winston, etc.)
1326
- interface Logger {
1327
- debug(message: string, ...args: unknown[]): void;
1328
- info(message: string, ...args: unknown[]): void;
1329
- warn(message: string, ...args: unknown[]): void;
1330
- error(message: string, ...args: unknown[]): void;
1331
- }
1332
- ```
1333
-
1334
- ### Retry Utilities
1335
-
1336
- ```typescript
1337
- import { withRetry, sleep, calculateRetryDelay, DEFAULT_RETRY_OPTIONS } from '@multiplier-labs/stepflow';
1338
-
1339
- // Retry a function with exponential backoff
1340
- const result = await withRetry(
1341
- () => fetchFromApi(),
1342
- {
1343
- maxRetries: 3, // default: 3
1344
- delay: 1000, // default: 1000ms
1345
- backoff: 2, // default: 2 (exponential)
1346
- signal: abortSignal, // optional
1347
- onRetry: (attempt, error, nextDelay) => {
1348
- console.log(`Retry ${attempt}, next delay: ${nextDelay}ms`);
1349
- },
1350
- }
1351
- );
1352
-
1353
- // Sleep with abort support
1354
- await sleep(1000, abortSignal?);
1355
-
1356
- // Calculate delay for a given attempt
1357
- const delay = calculateRetryDelay(attempt, baseDelay, backoff);
1358
- ```
1359
-
1360
- ---
1361
-
1362
- ## TypeScript Types Reference
1363
-
1364
- All types are exported from the main entry point:
1365
-
1366
- ```typescript
1367
- import type {
1368
- // Core
1369
- WorkflowKind, RunStatus, StepStatus, StepErrorStrategy,
1370
- WorkflowError, RunResult, Logger, SpawnChildOptions,
1371
- WorkflowContext, WorkflowStep, WorkflowHooks, WorkflowDefinition,
1372
-
1373
- // Engine
1374
- WorkflowEngineConfig, StartRunOptions,
1375
-
1376
- // Storage (legacy)
1377
- StorageAdapter, WorkflowRunRecord, WorkflowRunStepRecord,
1378
- WorkflowEventRecord, ListRunsOptions, ListEventsOptions,
1379
- PaginatedResult,
1380
-
1381
- // Storage (extended)
1382
- WorkflowStorage, ExtendedWorkflowRunRecord, ExtendedRunStatus,
1383
- ExtendedStepStatus, ExtendedListRunsOptions,
1384
- CreateRunInput, UpdateRunInput, StepResult, StepRecord,
1385
- StepflowRunsTable, StepflowStepResultsTable, StepflowDatabase,
1386
-
1387
- // Events
1388
- EventTransport, WorkflowEvent, WorkflowEventType,
1389
- BuiltInEventType, EventCallback, Unsubscribe,
1390
- SocketIOEventTransportConfig, WebhookEventTransportConfig,
1391
- WebhookEndpoint, WebhookPayload,
1392
-
1393
- // Scheduler
1394
- TriggerType, WorkflowSchedule, Scheduler,
1395
- CronSchedulerConfig, SchedulePersistence,
1396
- PostgresSchedulePersistenceConfig,
1397
-
1398
- // Planning
1399
- Recipe, RecipeStep, RecipeCondition, RecipeDefaults,
1400
- Plan, PlannedStep, PlanModification, PlanModificationType,
1401
- Planner, PlanningContext, PlanningConstraints, PlanningHints,
1402
- ResourceEstimate, RecipeSelectionResult, PlanValidationResult,
1403
- StepHandlerRegistry, RecipeRegistry, RegisteredStepHandler,
1404
- RecipeQueryOptions, ConditionOperator, StepHandlerRef,
1405
- ChildWorkflowPlan, PlanningPriority,
1406
-
1407
- // Config
1408
- PostgresStorageConfig, SQLiteStorageConfig,
1409
- SQLiteSchedulePersistenceConfig,
1410
-
1411
- // Retry
1412
- RetryOptions,
1413
- } from '@multiplier-labs/stepflow';
1414
-
1415
- // Classes and values
1416
- import {
1417
- WorkflowEngine,
1418
- MemoryStorageAdapter, SQLiteStorageAdapter,
1419
- PostgresStorageAdapter, PostgresStorageAdapter as PostgresStorage,
1420
- MemoryEventTransport, SocketIOEventTransport, WebhookEventTransport,
1421
- CronScheduler, SQLiteSchedulePersistence, PostgresSchedulePersistence,
1422
- MemoryStepHandlerRegistry, MemoryRecipeRegistry,
1423
- createRegistry, RuleBasedPlanner,
1424
- WorkflowEngineError, WorkflowNotFoundError,
1425
- WorkflowAlreadyRegisteredError, RunNotFoundError,
1426
- StepError, StepTimeoutError,
1427
- WorkflowCanceledError, WorkflowTimeoutError, WaitForRunTimeoutError,
1428
- ConsoleLogger, SilentLogger, createScopedLogger,
1429
- generateId, sleep, withRetry, calculateRetryDelay,
1430
- DEFAULT_RETRY_OPTIONS,
1431
- } from '@multiplier-labs/stepflow';
1432
- ```
1433
-
1434
- Subpath exports are also available:
1435
-
1436
- ```typescript
1437
- import { ... } from '@multiplier-labs/stepflow/storage';
1438
- import { ... } from '@multiplier-labs/stepflow/events';
1439
- import { ... } from '@multiplier-labs/stepflow/scheduler';
1440
- ```
1441
-
1442
- ---
1443
-
1444
- ## Downstream Integration Patterns
1445
-
1446
- This section documents patterns for integrating Stepflow into application services. The examples below are based on a data synchronization use case but apply to any domain.
1447
-
1448
- ### Fastify Plugin Pattern
1449
-
1450
- Wrap Stepflow storage as a Fastify plugin for API-level workflow management:
1451
-
1452
- ```typescript
1453
- // plugins/stepflow.ts
1454
- import fp from 'fastify-plugin';
1455
- import { PostgresStorageAdapter, type WorkflowRunRecord, type PaginatedResult } from '@multiplier-labs/stepflow';
1456
-
1457
- export interface StepflowService {
1458
- storage: PostgresStorageAdapter;
1459
-
1460
- queueWorkflow(
1461
- kind: string,
1462
- input: Record<string, unknown>,
1463
- options?: { priority?: number; timeoutMs?: number }
1464
- ): Promise<WorkflowRunRecord>;
1465
-
1466
- getRun(runId: string): Promise<WorkflowRunRecord | undefined>;
1467
-
1468
- listRuns(options?: {
1469
- kind?: string;
1470
- status?: string | string[];
1471
- limit?: number;
1472
- offset?: number;
1473
- }): Promise<PaginatedResult<WorkflowRunRecord>>;
1474
-
1475
- cancelRun(runId: string): Promise<void>;
1476
- }
1477
-
1478
- export default fp(async (fastify) => {
1479
- const storage = new PostgresStorageAdapter({
1480
- pool: fastify.pg.pool, // share the app's pool
1481
- schema: 'stepflow',
1482
- });
1483
- await storage.initialize();
1484
-
1485
- const service: StepflowService = {
1486
- storage,
1487
-
1488
- async queueWorkflow(kind, input, options = {}) {
1489
- return storage.createRun({
1490
- kind,
1491
- status: 'queued',
1492
- input,
1493
- context: {},
1494
- priority: options.priority ?? 0,
1495
- timeoutMs: options.timeoutMs,
1496
- });
1497
- },
1498
-
1499
- async getRun(runId) {
1500
- return (await storage.getRun(runId)) ?? undefined;
1501
- },
1502
-
1503
- async listRuns(options = {}) {
1504
- return storage.listRuns(options);
1505
- },
1506
-
1507
- async cancelRun(runId) {
1508
- await storage.updateRun(runId, {
1509
- status: 'canceled',
1510
- finishedAt: new Date(),
1511
- });
1512
- },
1513
- };
1514
-
1515
- fastify.decorate('stepflow', service);
1516
-
1517
- fastify.addHook('onClose', async () => {
1518
- await storage.close();
1519
- });
1520
- });
1521
- ```
1522
-
1523
- ### Worker Service Pattern
1524
-
1525
- A background worker that polls for queued workflows:
1526
-
1527
- ```typescript
1528
- import { PostgresStorageAdapter } from '@multiplier-labs/stepflow';
1529
-
1530
- const storage = new PostgresStorageAdapter({
1531
- connectionString: process.env.DATABASE_URL,
1532
- schema: 'stepflow',
1533
- });
1534
- await storage.initialize();
1535
-
1536
- let currentRunId: string | undefined;
1537
- let running = true;
1538
-
1539
- // Poll loop
1540
- async function poll() {
1541
- while (running) {
1542
- const run = await storage.dequeueRun(['sync.xero', 'sync.directo']);
1543
-
1544
- if (run) {
1545
- currentRunId = run.id;
1546
- try {
1547
- await processRun(run);
1548
- await storage.updateRun(run.id, {
1549
- status: 'succeeded',
1550
- output: { /* results */ },
1551
- finishedAt: new Date(),
1552
- });
1553
- } catch (err) {
1554
- await storage.updateRun(run.id, {
1555
- status: 'failed',
1556
- error: { code: 'WORKFLOW_ERROR', message: err.message },
1557
- finishedAt: new Date(),
1558
- });
1559
- } finally {
1560
- currentRunId = undefined;
1561
- }
1562
- }
1563
-
1564
- await new Promise(r => setTimeout(r, 5000));
1565
-
1566
- // Periodic stale run cleanup
1567
- await storage.cleanupStaleRuns(600000);
1568
- }
1569
- }
1570
-
1571
- // Graceful shutdown
1572
- process.on('SIGTERM', async () => {
1573
- running = false;
1574
- if (currentRunId) {
1575
- await storage.markRunsAsFailed(
1576
- [currentRunId],
1577
- 'Worker process was terminated during workflow execution'
1578
- );
1579
- }
1580
- await storage.close();
1581
- process.exit(0);
1582
- });
1583
- ```
1584
-
1585
- ### Step Tracking with saveStepResult
1586
-
1587
- Track granular step progress within a workflow run:
1588
-
1589
- ```typescript
1590
- async function processRun(run: WorkflowRunRecord) {
1591
- const steps = ['fetch_accounts', 'fetch_contacts', 'fetch_invoices'];
1592
-
1593
- for (const stepName of steps) {
1594
- const startedAt = new Date();
1595
-
1596
- await storage.saveStepResult({
1597
- runId: run.id,
1598
- stepName,
1599
- status: 'running',
1600
- attempt: 1,
1601
- startedAt,
1602
- });
1603
-
1604
- try {
1605
- const result = await executeStep(stepName, run.input);
1606
-
1607
- await storage.saveStepResult({
1608
- runId: run.id,
1609
- stepName,
1610
- status: 'completed',
1611
- output: { rowsRead: result.read, rowsWritten: result.written },
1612
- attempt: 1,
1613
- startedAt,
1614
- completedAt: new Date(),
1615
- });
1616
- } catch (err) {
1617
- await storage.saveStepResult({
1618
- runId: run.id,
1619
- stepName,
1620
- status: 'failed',
1621
- error: { code: 'STEP_FAILED', message: err.message },
1622
- attempt: 1,
1623
- startedAt,
1624
- completedAt: new Date(),
1625
- });
1626
- throw err;
1627
- }
1628
- }
1629
- }
1630
- ```
1631
-
1632
- ### API Routes Pattern
1633
-
1634
- ```typescript
1635
- // List runs
1636
- fastify.get('/api/workflows', async (request) => {
1637
- const { items, total } = await fastify.stepflow.listRuns({ limit: 100 });
1638
- return { data: items, total };
1639
- });
1640
-
1641
- // Get run with steps
1642
- fastify.get('/api/workflows/:runId', async (request) => {
1643
- const { runId } = request.params;
1644
- const run = await fastify.stepflow.getRun(runId);
1645
- if (!run) return reply.code(404).send({ error: 'Not found' });
1646
-
1647
- const steps = await fastify.stepflow.storage.getStepResults(runId);
1648
- return { data: { ...run, steps } };
1649
- });
1650
-
1651
- // Queue a workflow
1652
- fastify.post('/api/workflows/trigger', async (request) => {
1653
- const { kind, input, priority } = request.body;
1654
- const run = await fastify.stepflow.queueWorkflow(kind, input, { priority });
1655
- return { data: { runId: run.id, status: run.status } };
1656
- });
1657
-
1658
- // Cancel a workflow
1659
- fastify.post('/api/workflows/:runId/cancel', async (request) => {
1660
- await fastify.stepflow.cancelRun(request.params.runId);
1661
- return { data: { message: 'Workflow canceled' } };
1662
- });
1663
- ```