@multiplier-labs/stepflow 0.3.4 → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/dist/{chunk-U3PCVK6O.js → chunk-D5HW7VEG.js} +4 -2
- package/dist/chunk-D5HW7VEG.js.map +1 -0
- package/dist/{chunk-5XJBAPXF.js → chunk-JUPZEBR3.js} +3 -1
- package/dist/{chunk-5XJBAPXF.js.map → chunk-JUPZEBR3.js.map} +1 -1
- package/dist/events/index.d.ts +2 -0
- package/dist/events/index.js +1 -1
- package/dist/index.js +2 -2
- package/dist/storage/index.js +1 -1
- package/package.json +23 -12
- package/.codekin/reports/code-review/2026-04-11_code-review-daily.md +0 -146
- package/.codekin/reports/code-review/2026-04-13_code-review-daily.md +0 -74
- package/.codekin/reports/comments/2026-04-04_comment-assessment.md +0 -179
- package/.codekin/reports/comments/2026-04-11_comment-assessment.md +0 -161
- package/.codekin/reports/comments/2026-04-14_comment-assessment.md +0 -34
- package/.codekin/reports/complexity/2026-04-03_complexity-report.md +0 -152
- package/.codekin/reports/complexity/2026-04-11_complexity-report.md +0 -152
- package/.codekin/reports/complexity/2026-04-14_complexity-report.md +0 -34
- package/.codekin/reports/dependencies/2026-04-04_dependency-health.md +0 -94
- package/.codekin/reports/dependencies/2026-04-11_dependency-health.md +0 -93
- package/.codekin/reports/docs-audit/2026-04-04_docs-audit.md +0 -10
- package/.codekin/reports/docs-audit/2026-04-11_docs-audit.md +0 -138
- package/.codekin/reports/docs-audit/2026-04-13_docs-audit.md +0 -121
- package/.codekin/reports/security/2026-04-03_security-audit.md +0 -208
- package/.codekin/reports/security/2026-04-11_security-audit.md +0 -297
- package/.github/CODEOWNERS +0 -15
- package/.github/dependabot.yml +0 -23
- package/.github/workflows/ci.yml +0 -38
- package/.github/workflows/publish.yml +0 -42
- package/.github/workflows/security.yml +0 -29
- package/CHANGELOG.md +0 -34
- package/SECURITY.md +0 -40
- package/dist/chunk-U3PCVK6O.js.map +0 -1
- package/docs/github-packages-setup.md +0 -97
- package/docs/stepflow-api-reference.md +0 -1663
- package/src/core/engine/concurrency.ts +0 -229
- package/src/core/engine/lifecycle.ts +0 -176
- package/src/core/engine.test.ts +0 -1435
- package/src/core/engine.ts +0 -598
- package/src/core/index.ts +0 -7
- package/src/core/orchestrator/persistence.ts +0 -165
- package/src/core/orchestrator/plan-resolution.ts +0 -140
- package/src/core/orchestrator/state-transitions.ts +0 -109
- package/src/core/orchestrator/step-runner.ts +0 -427
- package/src/core/orchestrator.ts +0 -252
- package/src/core/types.ts +0 -220
- package/src/events/index.ts +0 -8
- package/src/events/memory.test.ts +0 -107
- package/src/events/memory.ts +0 -92
- package/src/events/socketio.test.ts +0 -378
- package/src/events/socketio.ts +0 -262
- package/src/events/types.ts +0 -118
- package/src/events/webhook.test.ts +0 -949
- package/src/events/webhook.ts +0 -807
- package/src/index.ts +0 -270
- package/src/planning/evaluate-condition.test.ts +0 -153
- package/src/planning/evaluate-condition.ts +0 -105
- package/src/planning/index.ts +0 -55
- package/src/planning/planner.ts +0 -494
- package/src/planning/planning.test.ts +0 -727
- package/src/planning/registry.test.ts +0 -398
- package/src/planning/registry.ts +0 -283
- package/src/planning/types.ts +0 -566
- package/src/scheduler/cron.test.ts +0 -479
- package/src/scheduler/cron.ts +0 -438
- package/src/scheduler/index.ts +0 -8
- package/src/scheduler/postgres-persistence.ts +0 -530
- package/src/scheduler/sqlite-persistence.test.ts +0 -235
- package/src/scheduler/sqlite-persistence.ts +0 -286
- package/src/scheduler/types.ts +0 -73
- package/src/storage/index.ts +0 -27
- package/src/storage/memory.test.ts +0 -349
- package/src/storage/memory.ts +0 -227
- package/src/storage/postgres-core.ts +0 -550
- package/src/storage/postgres-migration.test.ts +0 -69
- package/src/storage/postgres-transaction-pagination.test.ts +0 -107
- package/src/storage/postgres.ts +0 -835
- package/src/storage/sqlite-core.ts +0 -151
- package/src/storage/sqlite.test.ts +0 -642
- package/src/storage/sqlite.ts +0 -717
- package/src/storage/types.ts +0 -343
- package/src/utils/errors.test.ts +0 -150
- package/src/utils/errors.ts +0 -161
- package/src/utils/id.ts +0 -21
- package/src/utils/index.ts +0 -10
- package/src/utils/logger.test.ts +0 -121
- package/src/utils/logger.ts +0 -96
- package/src/utils/pinned-fetch.test.ts +0 -84
- package/src/utils/pinned-fetch.ts +0 -118
- package/src/utils/postgres-deps.ts +0 -79
- package/src/utils/retry.test.ts +0 -177
- package/src/utils/retry.ts +0 -121
- package/src/utils/safe-json.test.ts +0 -166
- package/src/utils/safe-json.ts +0 -107
- package/src/utils/webhook-signature.test.ts +0 -111
- package/src/utils/webhook-signature.ts +0 -74
- package/tsconfig.json +0 -21
- package/tsup.config.ts +0 -14
- package/vitest.config.ts +0 -9
|
@@ -1,1663 +0,0 @@
|
|
|
1
|
-
# Stepflow API Reference
|
|
2
|
-
|
|
3
|
-
A durable, type-safe workflow orchestration engine for Node.js with pluggable storage, scheduling, and real-time events.
|
|
4
|
-
|
|
5
|
-
## Table of Contents
|
|
6
|
-
|
|
7
|
-
- [Overview](#overview)
|
|
8
|
-
- [Installation](#installation)
|
|
9
|
-
- [Quick Start](#quick-start)
|
|
10
|
-
- [WorkflowEngine](#workflowengine)
|
|
11
|
-
- [Workflow Definitions](#workflow-definitions)
|
|
12
|
-
- [Step Handlers](#step-handlers)
|
|
13
|
-
- [Event System](#event-system)
|
|
14
|
-
- [Scheduler](#scheduler)
|
|
15
|
-
- [Storage Adapters](#storage-adapters)
|
|
16
|
-
- [PostgreSQL Storage Adapter](#postgresql-storage-adapter)
|
|
17
|
-
- [Planning System](#planning-system)
|
|
18
|
-
- [Error Handling](#error-handling)
|
|
19
|
-
- [Utilities](#utilities)
|
|
20
|
-
- [TypeScript Types Reference](#typescript-types-reference)
|
|
21
|
-
- [Downstream Integration Patterns](#downstream-integration-patterns)
|
|
22
|
-
|
|
23
|
-
---
|
|
24
|
-
|
|
25
|
-
## Overview
|
|
26
|
-
|
|
27
|
-
Stepflow provides:
|
|
28
|
-
|
|
29
|
-
- **Workflow Definition & Registration** - Define multi-step workflows with typed inputs
|
|
30
|
-
- **Async Execution Engine** - Non-blocking, fire-and-forget run execution
|
|
31
|
-
- **State Management** - Run/step status tracking with checkpointing for crash recovery
|
|
32
|
-
- **Event System** - Real-time events via in-memory, Socket.IO, or webhooks
|
|
33
|
-
- **Pluggable Storage** - Memory, SQLite, or PostgreSQL backends
|
|
34
|
-
- **Retry & Error Handling** - Configurable per-step error strategies (fail, retry, skip)
|
|
35
|
-
- **Scheduling** - Cron-based and workflow-completion triggers
|
|
36
|
-
- **Planning** - Rule-based recipe selection and plan generation
|
|
37
|
-
|
|
38
|
-
---
|
|
39
|
-
|
|
40
|
-
## Installation
|
|
41
|
-
|
|
42
|
-
```bash
|
|
43
|
-
npm install @multiplier-labs/stepflow
|
|
44
|
-
```
|
|
45
|
-
|
|
46
|
-
### Optional Peer Dependencies
|
|
47
|
-
|
|
48
|
-
Install based on your storage backend:
|
|
49
|
-
|
|
50
|
-
```bash
|
|
51
|
-
# For SQLite storage
|
|
52
|
-
npm install better-sqlite3
|
|
53
|
-
|
|
54
|
-
# For PostgreSQL storage
|
|
55
|
-
npm install pg kysely
|
|
56
|
-
```
|
|
57
|
-
|
|
58
|
-
These are loaded dynamically at runtime, so users of other backends are unaffected.
|
|
59
|
-
|
|
60
|
-
---
|
|
61
|
-
|
|
62
|
-
## Quick Start
|
|
63
|
-
|
|
64
|
-
```typescript
|
|
65
|
-
import { WorkflowEngine, MemoryStorageAdapter } from '@multiplier-labs/stepflow';
|
|
66
|
-
|
|
67
|
-
const engine = new WorkflowEngine({
|
|
68
|
-
storage: new MemoryStorageAdapter(),
|
|
69
|
-
});
|
|
70
|
-
|
|
71
|
-
engine.registerWorkflow({
|
|
72
|
-
kind: 'email.send_campaign',
|
|
73
|
-
name: 'Send Email Campaign',
|
|
74
|
-
steps: [
|
|
75
|
-
{
|
|
76
|
-
key: 'load_recipients',
|
|
77
|
-
name: 'Load Recipients',
|
|
78
|
-
handler: async (ctx) => {
|
|
79
|
-
const recipients = await loadRecipients(ctx.input.campaignId);
|
|
80
|
-
return { recipients };
|
|
81
|
-
},
|
|
82
|
-
},
|
|
83
|
-
{
|
|
84
|
-
key: 'send_emails',
|
|
85
|
-
name: 'Send Emails',
|
|
86
|
-
handler: async (ctx) => {
|
|
87
|
-
const { recipients } = ctx.results.load_recipients;
|
|
88
|
-
return { sent: recipients.length };
|
|
89
|
-
},
|
|
90
|
-
onError: 'retry',
|
|
91
|
-
maxRetries: 3,
|
|
92
|
-
},
|
|
93
|
-
],
|
|
94
|
-
});
|
|
95
|
-
|
|
96
|
-
const runId = await engine.startRun({
|
|
97
|
-
kind: 'email.send_campaign',
|
|
98
|
-
input: { campaignId: '123' },
|
|
99
|
-
});
|
|
100
|
-
|
|
101
|
-
engine.subscribeToRun(runId, (event) => {
|
|
102
|
-
console.log(`${event.eventType}: ${event.stepKey ?? 'run'}`);
|
|
103
|
-
});
|
|
104
|
-
```
|
|
105
|
-
|
|
106
|
-
---
|
|
107
|
-
|
|
108
|
-
## WorkflowEngine
|
|
109
|
-
|
|
110
|
-
The primary entry point for the library.
|
|
111
|
-
|
|
112
|
-
### Constructor
|
|
113
|
-
|
|
114
|
-
```typescript
|
|
115
|
-
import { WorkflowEngine } from '@multiplier-labs/stepflow';
|
|
116
|
-
|
|
117
|
-
const engine = new WorkflowEngine(config?: WorkflowEngineConfig);
|
|
118
|
-
```
|
|
119
|
-
|
|
120
|
-
### Configuration
|
|
121
|
-
|
|
122
|
-
```typescript
|
|
123
|
-
interface WorkflowEngineConfig {
|
|
124
|
-
storage?: StorageAdapter; // default: MemoryStorageAdapter
|
|
125
|
-
events?: EventTransport; // default: MemoryEventTransport
|
|
126
|
-
logger?: Logger; // default: ConsoleLogger
|
|
127
|
-
settings?: {
|
|
128
|
-
defaultTimeout?: number; // workflow timeout in ms
|
|
129
|
-
maxConcurrency?: number; // max concurrent workflows
|
|
130
|
-
};
|
|
131
|
-
}
|
|
132
|
-
```
|
|
133
|
-
|
|
134
|
-
### Methods
|
|
135
|
-
|
|
136
|
-
#### Lifecycle
|
|
137
|
-
|
|
138
|
-
```typescript
|
|
139
|
-
// Initialize storage and event adapters (required before use)
|
|
140
|
-
await engine.initialize(): Promise<void>;
|
|
141
|
-
|
|
142
|
-
// Graceful shutdown: cancels active runs, closes transports and storage
|
|
143
|
-
await engine.shutdown(): Promise<void>;
|
|
144
|
-
```
|
|
145
|
-
|
|
146
|
-
#### Workflow Registration
|
|
147
|
-
|
|
148
|
-
```typescript
|
|
149
|
-
// Register a workflow definition
|
|
150
|
-
engine.registerWorkflow<TInput>(definition: WorkflowDefinition<TInput>): void;
|
|
151
|
-
|
|
152
|
-
// Unregister a workflow by kind
|
|
153
|
-
engine.unregisterWorkflow(kind: string): boolean;
|
|
154
|
-
|
|
155
|
-
// Get a registered workflow definition
|
|
156
|
-
engine.getWorkflow(kind: string): WorkflowDefinition | undefined;
|
|
157
|
-
|
|
158
|
-
// List all registered workflow kinds
|
|
159
|
-
engine.getRegisteredWorkflows(): string[];
|
|
160
|
-
```
|
|
161
|
-
|
|
162
|
-
#### Running Workflows
|
|
163
|
-
|
|
164
|
-
```typescript
|
|
165
|
-
// Start a new run (non-blocking, returns immediately)
|
|
166
|
-
const runId = await engine.startRun<TInput>(options: StartRunOptions<TInput>): Promise<string>;
|
|
167
|
-
|
|
168
|
-
// Cancel a running workflow
|
|
169
|
-
await engine.cancelRun(runId: string): Promise<void>;
|
|
170
|
-
|
|
171
|
-
// Get current run status
|
|
172
|
-
const run = await engine.getRunStatus(runId: string): Promise<WorkflowRunRecord | null>;
|
|
173
|
-
|
|
174
|
-
// Poll until a run completes (for testing or synchronous use cases)
|
|
175
|
-
const run = await engine.waitForRun(runId: string, options?: {
|
|
176
|
-
pollInterval?: number; // default: 100ms
|
|
177
|
-
timeout?: number; // default: 30000ms
|
|
178
|
-
}): Promise<WorkflowRunRecord>;
|
|
179
|
-
```
|
|
180
|
-
|
|
181
|
-
#### Resume & Recovery
|
|
182
|
-
|
|
183
|
-
```typescript
|
|
184
|
-
// Resume a run from its last checkpoint
|
|
185
|
-
const newRunId = await engine.resumeRun(runId: string): Promise<string>;
|
|
186
|
-
|
|
187
|
-
// Get all resumable runs (status 'queued' or 'running')
|
|
188
|
-
const runs = await engine.getResumableRuns(): Promise<WorkflowRunRecord[]>;
|
|
189
|
-
|
|
190
|
-
// Resume all interrupted runs
|
|
191
|
-
const runIds = await engine.resumeAllInterrupted(): Promise<string[]>;
|
|
192
|
-
```
|
|
193
|
-
|
|
194
|
-
#### Event Subscriptions
|
|
195
|
-
|
|
196
|
-
```typescript
|
|
197
|
-
// Subscribe to events for a specific run
|
|
198
|
-
const unsubscribe = engine.subscribeToRun(runId: string, callback: EventCallback): Unsubscribe;
|
|
199
|
-
|
|
200
|
-
// Subscribe to all workflow events
|
|
201
|
-
const unsubscribe = engine.subscribeToAll(callback: EventCallback): Unsubscribe;
|
|
202
|
-
```
|
|
203
|
-
|
|
204
|
-
#### Introspection
|
|
205
|
-
|
|
206
|
-
```typescript
|
|
207
|
-
engine.getActiveRunCount(): number;
|
|
208
|
-
engine.getQueuedRunCount(): number;
|
|
209
|
-
engine.getStorage(): StorageAdapter;
|
|
210
|
-
engine.getEvents(): EventTransport;
|
|
211
|
-
```
|
|
212
|
-
|
|
213
|
-
### StartRunOptions
|
|
214
|
-
|
|
215
|
-
```typescript
|
|
216
|
-
interface StartRunOptions<TInput = Record<string, unknown>> {
|
|
217
|
-
kind: string; // workflow type to run
|
|
218
|
-
input?: TInput; // input parameters
|
|
219
|
-
metadata?: Record<string, unknown>; // optional metadata (userId, etc.)
|
|
220
|
-
parentRunId?: string; // parent run ID (for child workflows)
|
|
221
|
-
delay?: number; // delay before starting (ms)
|
|
222
|
-
priority?: number; // queue priority (higher = runs first, default: 0)
|
|
223
|
-
}
|
|
224
|
-
```
|
|
225
|
-
|
|
226
|
-
---
|
|
227
|
-
|
|
228
|
-
## Workflow Definitions
|
|
229
|
-
|
|
230
|
-
### WorkflowDefinition
|
|
231
|
-
|
|
232
|
-
```typescript
|
|
233
|
-
interface WorkflowDefinition<TInput = Record<string, unknown>> {
|
|
234
|
-
kind: string; // unique workflow type identifier
|
|
235
|
-
name: string; // human-readable name
|
|
236
|
-
description?: string;
|
|
237
|
-
steps: WorkflowStep<TInput>[]; // ordered list of steps
|
|
238
|
-
defaultOnError?: StepErrorStrategy; // default: 'fail'
|
|
239
|
-
timeout?: number; // workflow timeout in ms
|
|
240
|
-
hooks?: WorkflowHooks<TInput>; // lifecycle hooks
|
|
241
|
-
}
|
|
242
|
-
```
|
|
243
|
-
|
|
244
|
-
### WorkflowStep
|
|
245
|
-
|
|
246
|
-
```typescript
|
|
247
|
-
interface WorkflowStep<TInput = Record<string, unknown>> {
|
|
248
|
-
key: string; // unique step identifier within the workflow
|
|
249
|
-
name: string; // human-readable name
|
|
250
|
-
handler: (ctx: WorkflowContext<TInput>) => Promise<unknown>;
|
|
251
|
-
onError?: StepErrorStrategy; // 'fail' | 'retry' | 'skip' (default: inherited)
|
|
252
|
-
maxRetries?: number; // default: 3
|
|
253
|
-
retryDelay?: number; // default: 1000ms
|
|
254
|
-
retryBackoff?: number; // default: 2 (exponential)
|
|
255
|
-
timeout?: number; // step timeout in ms
|
|
256
|
-
skipIf?: (ctx: WorkflowContext<TInput>) => boolean | Promise<boolean>;
|
|
257
|
-
}
|
|
258
|
-
```
|
|
259
|
-
|
|
260
|
-
### WorkflowHooks
|
|
261
|
-
|
|
262
|
-
```typescript
|
|
263
|
-
interface WorkflowHooks<TInput = Record<string, unknown>> {
|
|
264
|
-
beforeRun?: (ctx: WorkflowContext<TInput>) => Promise<void>;
|
|
265
|
-
afterRun?: (ctx: WorkflowContext<TInput>, result: RunResult) => Promise<void>;
|
|
266
|
-
beforeStep?: (ctx: WorkflowContext<TInput>, step: WorkflowStep<TInput>) => Promise<void>;
|
|
267
|
-
afterStep?: (ctx: WorkflowContext<TInput>, step: WorkflowStep<TInput>, result: unknown) => Promise<void>;
|
|
268
|
-
onStepError?: (ctx: WorkflowContext<TInput>, step: WorkflowStep<TInput>, error: Error) => Promise<void>;
|
|
269
|
-
}
|
|
270
|
-
```
|
|
271
|
-
|
|
272
|
-
---
|
|
273
|
-
|
|
274
|
-
## Step Handlers
|
|
275
|
-
|
|
276
|
-
### WorkflowContext
|
|
277
|
-
|
|
278
|
-
Every step handler receives a `WorkflowContext`:
|
|
279
|
-
|
|
280
|
-
```typescript
|
|
281
|
-
interface WorkflowContext<TInput = Record<string, unknown>> {
|
|
282
|
-
runId: string; // unique run ID
|
|
283
|
-
stepId: string; // unique step ID (for granular tracking)
|
|
284
|
-
kind: string; // workflow type
|
|
285
|
-
input: TInput; // run input parameters
|
|
286
|
-
results: Record<string, unknown>; // accumulated results from prior steps (keyed by step key)
|
|
287
|
-
metadata: Record<string, unknown>;// optional metadata
|
|
288
|
-
currentStep?: string; // current step key
|
|
289
|
-
logger: Logger; // scoped logger
|
|
290
|
-
signal: AbortSignal; // for cancellation
|
|
291
|
-
spawnChild: (options: SpawnChildOptions) => Promise<string>;
|
|
292
|
-
emit: (eventType: string, payload?: unknown) => void;
|
|
293
|
-
}
|
|
294
|
-
```
|
|
295
|
-
|
|
296
|
-
### Spawning Child Workflows
|
|
297
|
-
|
|
298
|
-
```typescript
|
|
299
|
-
interface SpawnChildOptions {
|
|
300
|
-
kind: string;
|
|
301
|
-
input?: Record<string, unknown>;
|
|
302
|
-
metadata?: Record<string, unknown>;
|
|
303
|
-
}
|
|
304
|
-
|
|
305
|
-
// Usage inside a step handler:
|
|
306
|
-
const childRunId = await ctx.spawnChild({
|
|
307
|
-
kind: 'child.workflow',
|
|
308
|
-
input: { parentData: ctx.results.step1 },
|
|
309
|
-
});
|
|
310
|
-
```
|
|
311
|
-
|
|
312
|
-
### Emitting Custom Events
|
|
313
|
-
|
|
314
|
-
```typescript
|
|
315
|
-
// Inside a step handler:
|
|
316
|
-
ctx.emit('progress.update', { percent: 50 });
|
|
317
|
-
```
|
|
318
|
-
|
|
319
|
-
### Execution Flow
|
|
320
|
-
|
|
321
|
-
1. Run status set to `'running'`
|
|
322
|
-
2. `run.started` event emitted
|
|
323
|
-
3. `beforeRun` hook called
|
|
324
|
-
4. For each step (in order):
|
|
325
|
-
- Cancellation signal checked
|
|
326
|
-
- Skipped if in checkpoint (resume) or `skipIf` returns true
|
|
327
|
-
- `beforeStep` hook called
|
|
328
|
-
- Handler executed with timeout
|
|
329
|
-
- Result stored; `afterStep` hook called; `step.completed` event emitted
|
|
330
|
-
- On error: `step.failed` event, `onStepError` hook, then fail/retry/skip per strategy
|
|
331
|
-
- Checkpoint saved after each successful step
|
|
332
|
-
5. `afterRun` hook called
|
|
333
|
-
6. Run marked `'succeeded'` or `'failed'`
|
|
334
|
-
|
|
335
|
-
---
|
|
336
|
-
|
|
337
|
-
## Event System
|
|
338
|
-
|
|
339
|
-
### Built-in Event Types
|
|
340
|
-
|
|
341
|
-
```typescript
|
|
342
|
-
type BuiltInEventType =
|
|
343
|
-
| 'run.created' | 'run.queued' | 'run.dequeued'
|
|
344
|
-
| 'run.started' | 'run.resumed' | 'run.completed'
|
|
345
|
-
| 'run.failed' | 'run.canceled' | 'run.timeout'
|
|
346
|
-
| 'step.started' | 'step.completed'
|
|
347
|
-
| 'step.failed' | 'step.skipped' | 'step.retry';
|
|
348
|
-
```
|
|
349
|
-
|
|
350
|
-
### WorkflowEvent
|
|
351
|
-
|
|
352
|
-
```typescript
|
|
353
|
-
interface WorkflowEvent {
|
|
354
|
-
runId: string;
|
|
355
|
-
kind: string;
|
|
356
|
-
eventType: string; // BuiltInEventType or custom string
|
|
357
|
-
stepKey?: string;
|
|
358
|
-
timestamp: Date;
|
|
359
|
-
payload?: unknown;
|
|
360
|
-
}
|
|
361
|
-
```
|
|
362
|
-
|
|
363
|
-
### EventTransport Interface
|
|
364
|
-
|
|
365
|
-
```typescript
|
|
366
|
-
interface EventTransport {
|
|
367
|
-
emit(event: WorkflowEvent): void;
|
|
368
|
-
subscribe(runId: string, callback: EventCallback): Unsubscribe;
|
|
369
|
-
subscribeAll(callback: EventCallback): Unsubscribe;
|
|
370
|
-
subscribeToType?(eventType: string, callback: EventCallback): Unsubscribe;
|
|
371
|
-
persist?(event: WorkflowEvent): Promise<void>;
|
|
372
|
-
close?(): void;
|
|
373
|
-
}
|
|
374
|
-
|
|
375
|
-
type EventCallback = (event: WorkflowEvent) => void;
|
|
376
|
-
type Unsubscribe = () => void;
|
|
377
|
-
```
|
|
378
|
-
|
|
379
|
-
### MemoryEventTransport
|
|
380
|
-
|
|
381
|
-
Default in-memory transport. Supports `subscribeToType()`.
|
|
382
|
-
|
|
383
|
-
```typescript
|
|
384
|
-
import { MemoryEventTransport } from '@multiplier-labs/stepflow';
|
|
385
|
-
|
|
386
|
-
const events = new MemoryEventTransport();
|
|
387
|
-
events.getListenerCount(channel?: string): number;
|
|
388
|
-
```
|
|
389
|
-
|
|
390
|
-
### SocketIOEventTransport
|
|
391
|
-
|
|
392
|
-
Real-time events via Socket.IO.
|
|
393
|
-
|
|
394
|
-
```typescript
|
|
395
|
-
import { SocketIOEventTransport } from '@multiplier-labs/stepflow';
|
|
396
|
-
|
|
397
|
-
const events = new SocketIOEventTransport({
|
|
398
|
-
io: socketIOServer,
|
|
399
|
-
eventName: 'workflow:event', // default
|
|
400
|
-
roomPrefix: 'run:', // default
|
|
401
|
-
broadcastGlobal: true, // default
|
|
402
|
-
globalRoom: 'workflow:all', // default
|
|
403
|
-
});
|
|
404
|
-
|
|
405
|
-
// Set up client handlers for a socket connection
|
|
406
|
-
events.setupClientHandlers(socket, async (runId, socket) => {
|
|
407
|
-
return /* authorization check */;
|
|
408
|
-
});
|
|
409
|
-
```
|
|
410
|
-
|
|
411
|
-
**Client events:** `workflow:subscribe`, `workflow:unsubscribe`, `workflow:subscribe:all`, `workflow:unsubscribe:all`
|
|
412
|
-
|
|
413
|
-
### WebhookEventTransport
|
|
414
|
-
|
|
415
|
-
Delivers events to HTTP endpoints with HMAC-SHA256 signing.
|
|
416
|
-
|
|
417
|
-
```typescript
|
|
418
|
-
import { WebhookEventTransport } from '@multiplier-labs/stepflow';
|
|
419
|
-
|
|
420
|
-
const events = new WebhookEventTransport({
|
|
421
|
-
endpoints: [
|
|
422
|
-
{
|
|
423
|
-
id: 'my-webhook',
|
|
424
|
-
url: 'https://example.com/webhook',
|
|
425
|
-
secret: 'my-secret', // for HMAC-SHA256 signing
|
|
426
|
-
eventTypes: ['run.completed'], // filter (optional)
|
|
427
|
-
workflowKinds: ['my.workflow'], // filter (optional)
|
|
428
|
-
headers: { 'X-Custom': 'val' },
|
|
429
|
-
enabled: true, // default: true
|
|
430
|
-
timeout: 5000, // default: 5000ms
|
|
431
|
-
retries: 3, // default: 3
|
|
432
|
-
},
|
|
433
|
-
],
|
|
434
|
-
defaultTimeout: 5000, // default
|
|
435
|
-
defaultRetries: 3, // default
|
|
436
|
-
retryDelay: 1000, // default
|
|
437
|
-
fetchFn: fetch, // custom fetch (optional)
|
|
438
|
-
});
|
|
439
|
-
|
|
440
|
-
events.addEndpoint(endpoint);
|
|
441
|
-
events.removeEndpoint(id: string): boolean;
|
|
442
|
-
events.getEndpoints(): WebhookEndpoint[];
|
|
443
|
-
events.setEndpointEnabled(id: string, enabled: boolean): void;
|
|
444
|
-
```
|
|
445
|
-
|
|
446
|
-
**Webhook payload:**
|
|
447
|
-
|
|
448
|
-
```typescript
|
|
449
|
-
interface WebhookPayload {
|
|
450
|
-
event: { runId, kind, eventType, stepKey?, payload?, timestamp: string };
|
|
451
|
-
deliveredAt: string;
|
|
452
|
-
webhookId: string;
|
|
453
|
-
}
|
|
454
|
-
```
|
|
455
|
-
|
|
456
|
-
Signed with `X-Webhook-Signature` header (HMAC-SHA256).
|
|
457
|
-
|
|
458
|
-
---
|
|
459
|
-
|
|
460
|
-
## Scheduler
|
|
461
|
-
|
|
462
|
-
### CronScheduler
|
|
463
|
-
|
|
464
|
-
```typescript
|
|
465
|
-
import { CronScheduler } from '@multiplier-labs/stepflow';
|
|
466
|
-
|
|
467
|
-
const scheduler = new CronScheduler({
|
|
468
|
-
engine: workflowEngine,
|
|
469
|
-
logger: myLogger, // optional
|
|
470
|
-
pollInterval: 1000, // default: 1000ms
|
|
471
|
-
persistence: myPersistence, // optional SchedulePersistence
|
|
472
|
-
});
|
|
473
|
-
|
|
474
|
-
await scheduler.start();
|
|
475
|
-
|
|
476
|
-
// Add a cron schedule
|
|
477
|
-
const schedule = await scheduler.addSchedule({
|
|
478
|
-
workflowKind: 'reports.generate',
|
|
479
|
-
triggerType: 'cron',
|
|
480
|
-
cronExpression: '0 6 * * *', // daily at 6 AM
|
|
481
|
-
timezone: 'UTC',
|
|
482
|
-
input: { format: 'pdf' },
|
|
483
|
-
enabled: true,
|
|
484
|
-
});
|
|
485
|
-
|
|
486
|
-
// Add a workflow-completion trigger
|
|
487
|
-
await scheduler.addSchedule({
|
|
488
|
-
workflowKind: 'cleanup.run',
|
|
489
|
-
triggerType: 'workflow_completed',
|
|
490
|
-
triggerOnWorkflowKind: 'data.import',
|
|
491
|
-
triggerOnStatus: ['succeeded'],
|
|
492
|
-
enabled: true,
|
|
493
|
-
});
|
|
494
|
-
|
|
495
|
-
// Manage schedules
|
|
496
|
-
await scheduler.removeSchedule(scheduleId);
|
|
497
|
-
await scheduler.updateSchedule(scheduleId, { enabled: false });
|
|
498
|
-
const schedules = await scheduler.getSchedules();
|
|
499
|
-
const runId = await scheduler.triggerNow(scheduleId);
|
|
500
|
-
|
|
501
|
-
await scheduler.stop();
|
|
502
|
-
```
|
|
503
|
-
|
|
504
|
-
### WorkflowSchedule
|
|
505
|
-
|
|
506
|
-
```typescript
|
|
507
|
-
interface WorkflowSchedule {
|
|
508
|
-
id: string;
|
|
509
|
-
workflowKind: string;
|
|
510
|
-
triggerType: 'cron' | 'workflow_completed' | 'manual';
|
|
511
|
-
cronExpression?: string;
|
|
512
|
-
timezone?: string;
|
|
513
|
-
triggerOnWorkflowKind?: string;
|
|
514
|
-
triggerOnStatus?: RunStatus[];
|
|
515
|
-
input?: Record<string, unknown>;
|
|
516
|
-
metadata?: Record<string, unknown>;
|
|
517
|
-
enabled: boolean;
|
|
518
|
-
lastRunAt?: Date;
|
|
519
|
-
lastRunId?: string;
|
|
520
|
-
nextRunAt?: Date;
|
|
521
|
-
}
|
|
522
|
-
```
|
|
523
|
-
|
|
524
|
-
### SchedulePersistence Interface
|
|
525
|
-
|
|
526
|
-
```typescript
|
|
527
|
-
interface SchedulePersistence {
|
|
528
|
-
loadSchedules(): Promise<WorkflowSchedule[]>;
|
|
529
|
-
saveSchedule(schedule: WorkflowSchedule): Promise<void>;
|
|
530
|
-
updateSchedule(scheduleId: string, updates: Partial<WorkflowSchedule>): Promise<void>;
|
|
531
|
-
deleteSchedule(scheduleId: string): Promise<void>;
|
|
532
|
-
}
|
|
533
|
-
```
|
|
534
|
-
|
|
535
|
-
### SQLiteSchedulePersistence
|
|
536
|
-
|
|
537
|
-
```typescript
|
|
538
|
-
import { SQLiteSchedulePersistence } from '@multiplier-labs/stepflow';
|
|
539
|
-
|
|
540
|
-
const persistence = new SQLiteSchedulePersistence({
|
|
541
|
-
db: sqliteDatabase,
|
|
542
|
-
tableName: 'workflow_schedules', // default
|
|
543
|
-
});
|
|
544
|
-
```
|
|
545
|
-
|
|
546
|
-
### PostgresSchedulePersistence
|
|
547
|
-
|
|
548
|
-
```typescript
|
|
549
|
-
import { PostgresSchedulePersistence } from '@multiplier-labs/stepflow';
|
|
550
|
-
|
|
551
|
-
const persistence = new PostgresSchedulePersistence({
|
|
552
|
-
connectionString: process.env.DATABASE_URL,
|
|
553
|
-
// OR pool: existingPgPool,
|
|
554
|
-
// OR poolConfig: { ... },
|
|
555
|
-
schema: 'public', // default
|
|
556
|
-
tableName: 'workflow_schedules', // default
|
|
557
|
-
autoMigrate: true, // default
|
|
558
|
-
});
|
|
559
|
-
|
|
560
|
-
await persistence.initialize();
|
|
561
|
-
|
|
562
|
-
// Additional query methods:
|
|
563
|
-
await persistence.getSchedule(scheduleId): Promise<WorkflowSchedule | null>;
|
|
564
|
-
await persistence.getDueSchedules(): Promise<WorkflowSchedule[]>;
|
|
565
|
-
await persistence.getSchedulesByWorkflowKind(kind): Promise<WorkflowSchedule[]>;
|
|
566
|
-
await persistence.getCompletionTriggers(triggerOnKind): Promise<WorkflowSchedule[]>;
|
|
567
|
-
|
|
568
|
-
await persistence.close();
|
|
569
|
-
```
|
|
570
|
-
|
|
571
|
-
---
|
|
572
|
-
|
|
573
|
-
## Storage Adapters
|
|
574
|
-
|
|
575
|
-
All storage adapters implement the `StorageAdapter` interface.
|
|
576
|
-
|
|
577
|
-
### StorageAdapter Interface
|
|
578
|
-
|
|
579
|
-
```typescript
|
|
580
|
-
interface StorageAdapter {
|
|
581
|
-
// Run operations
|
|
582
|
-
createRun(run: Omit<WorkflowRunRecord, 'id' | 'createdAt'>): Promise<WorkflowRunRecord>;
|
|
583
|
-
getRun(runId: string): Promise<WorkflowRunRecord | null>;
|
|
584
|
-
updateRun(runId: string, updates: Partial<WorkflowRunRecord>): Promise<void>;
|
|
585
|
-
listRuns(options?: ListRunsOptions): Promise<PaginatedResult<WorkflowRunRecord>>;
|
|
586
|
-
|
|
587
|
-
// Step operations
|
|
588
|
-
createStep(step: Omit<WorkflowRunStepRecord, 'id'>): Promise<WorkflowRunStepRecord>;
|
|
589
|
-
getStep(stepId: string): Promise<WorkflowRunStepRecord | null>;
|
|
590
|
-
updateStep(stepId: string, updates: Partial<WorkflowRunStepRecord>): Promise<void>;
|
|
591
|
-
getStepsForRun(runId: string): Promise<WorkflowRunStepRecord[]>;
|
|
592
|
-
|
|
593
|
-
// Event operations
|
|
594
|
-
saveEvent(event: Omit<WorkflowEventRecord, 'id'>): Promise<void>;
|
|
595
|
-
getEventsForRun(runId: string, options?: ListEventsOptions): Promise<WorkflowEventRecord[]>;
|
|
596
|
-
|
|
597
|
-
// Optional
|
|
598
|
-
transaction?<T>(fn: (tx: StorageAdapter) => Promise<T>): Promise<T>;
|
|
599
|
-
deleteOldRuns?(olderThan: Date): Promise<number>;
|
|
600
|
-
initialize?(): Promise<void>;
|
|
601
|
-
close?(): void | Promise<void>;
|
|
602
|
-
}
|
|
603
|
-
```
|
|
604
|
-
|
|
605
|
-
### WorkflowStorage Interface
|
|
606
|
-
|
|
607
|
-
A newer storage interface with additional methods for queue management and step results. The `PostgresStorageAdapter` implements both `StorageAdapter` and `WorkflowStorage`.
|
|
608
|
-
|
|
609
|
-
```typescript
|
|
610
|
-
interface WorkflowStorage {
|
|
611
|
-
// Run operations
|
|
612
|
-
createRun(run: CreateRunInput): Promise<ExtendedWorkflowRunRecord>;
|
|
613
|
-
getRun(id: string): Promise<ExtendedWorkflowRunRecord | null>;
|
|
614
|
-
updateRun(id: string, updates: UpdateRunInput): Promise<void>;
|
|
615
|
-
listRuns(options?: ExtendedListRunsOptions): Promise<PaginatedResult<ExtendedWorkflowRunRecord>>;
|
|
616
|
-
deleteRun(id: string): Promise<void>;
|
|
617
|
-
|
|
618
|
-
// Atomic dequeue
|
|
619
|
-
dequeueRun(workflowKinds: string[]): Promise<ExtendedWorkflowRunRecord | null>;
|
|
620
|
-
|
|
621
|
-
// Stale run management
|
|
622
|
-
cleanupStaleRuns(defaultTimeoutMs?: number): Promise<number>;
|
|
623
|
-
markRunsAsFailed(runIds: string[], reason: string): Promise<void>;
|
|
624
|
-
|
|
625
|
-
// Step results (uses stepflow_step_results table)
|
|
626
|
-
getStepResult(runId: string, stepName: string): Promise<StepResult | undefined>;
|
|
627
|
-
getStepResults(runId: string): Promise<StepResult[]>;
|
|
628
|
-
getStepsForRun(runId: string): Promise<StepRecord[]>;
|
|
629
|
-
saveStepResult(result: Omit<StepResult, 'id'> & { id?: string }): Promise<void>;
|
|
630
|
-
|
|
631
|
-
// Lifecycle
|
|
632
|
-
initialize(): Promise<void>;
|
|
633
|
-
close(): Promise<void>;
|
|
634
|
-
}
|
|
635
|
-
```
|
|
636
|
-
|
|
637
|
-
### Core Record Types
|
|
638
|
-
|
|
639
|
-
```typescript
|
|
640
|
-
interface WorkflowRunRecord {
|
|
641
|
-
id: string;
|
|
642
|
-
kind: string;
|
|
643
|
-
status: RunStatus; // 'queued' | 'running' | 'succeeded' | 'failed' | 'canceled'
|
|
644
|
-
parentRunId?: string;
|
|
645
|
-
input: Record<string, unknown>;
|
|
646
|
-
context: Record<string, unknown>; // checkpoint: accumulated step results
|
|
647
|
-
output?: Record<string, unknown>;
|
|
648
|
-
error?: { code: string; message: string };
|
|
649
|
-
metadata?: Record<string, unknown>;
|
|
650
|
-
priority?: number;
|
|
651
|
-
timeoutMs?: number;
|
|
652
|
-
createdAt: Date;
|
|
653
|
-
startedAt?: Date;
|
|
654
|
-
finishedAt?: Date;
|
|
655
|
-
}
|
|
656
|
-
|
|
657
|
-
interface WorkflowRunStepRecord {
|
|
658
|
-
id: string;
|
|
659
|
-
runId: string;
|
|
660
|
-
stepKey: string;
|
|
661
|
-
stepName: string;
|
|
662
|
-
status: StepStatus; // 'pending' | 'running' | 'succeeded' | 'failed' | 'skipped' | 'canceled'
|
|
663
|
-
attempt: number;
|
|
664
|
-
result?: unknown;
|
|
665
|
-
error?: WorkflowError;
|
|
666
|
-
startedAt?: Date;
|
|
667
|
-
finishedAt?: Date;
|
|
668
|
-
}
|
|
669
|
-
|
|
670
|
-
interface WorkflowEventRecord {
|
|
671
|
-
id: string;
|
|
672
|
-
runId: string;
|
|
673
|
-
stepKey?: string;
|
|
674
|
-
eventType: string;
|
|
675
|
-
level: 'info' | 'warn' | 'error';
|
|
676
|
-
payload?: unknown;
|
|
677
|
-
timestamp: Date;
|
|
678
|
-
}
|
|
679
|
-
```
|
|
680
|
-
|
|
681
|
-
### Extended Types
|
|
682
|
-
|
|
683
|
-
Used by the `WorkflowStorage` interface and `PostgresStorageAdapter`:
|
|
684
|
-
|
|
685
|
-
```typescript
|
|
686
|
-
// Adds 'pending' and 'timeout' to core RunStatus
|
|
687
|
-
type ExtendedRunStatus = 'pending' | 'queued' | 'running' | 'succeeded' | 'failed' | 'canceled' | 'timeout';
|
|
688
|
-
|
|
689
|
-
// Uses 'completed' instead of 'succeeded'
|
|
690
|
-
type ExtendedStepStatus = 'pending' | 'running' | 'completed' | 'failed' | 'skipped';
|
|
691
|
-
|
|
692
|
-
interface StepResult {
|
|
693
|
-
id: string;
|
|
694
|
-
runId: string;
|
|
695
|
-
stepName: string;
|
|
696
|
-
status: ExtendedStepStatus;
|
|
697
|
-
output?: Record<string, unknown>;
|
|
698
|
-
error?: Record<string, unknown>;
|
|
699
|
-
attempt: number;
|
|
700
|
-
startedAt?: Date;
|
|
701
|
-
completedAt?: Date;
|
|
702
|
-
}
|
|
703
|
-
|
|
704
|
-
interface StepRecord {
|
|
705
|
-
stepKey: string;
|
|
706
|
-
stepName: string;
|
|
707
|
-
status: ExtendedStepStatus;
|
|
708
|
-
result?: Record<string, unknown>;
|
|
709
|
-
error?: Record<string, unknown>;
|
|
710
|
-
attempt: number;
|
|
711
|
-
startedAt?: Date;
|
|
712
|
-
finishedAt?: Date;
|
|
713
|
-
}
|
|
714
|
-
```
|
|
715
|
-
|
|
716
|
-
### Query Options
|
|
717
|
-
|
|
718
|
-
```typescript
|
|
719
|
-
interface ListRunsOptions {
|
|
720
|
-
kind?: string;
|
|
721
|
-
status?: RunStatus | RunStatus[];
|
|
722
|
-
parentRunId?: string;
|
|
723
|
-
limit?: number; // default: 50
|
|
724
|
-
offset?: number; // default: 0
|
|
725
|
-
orderBy?: 'createdAt' | 'startedAt' | 'finishedAt';
|
|
726
|
-
orderDirection?: 'asc' | 'desc'; // default: 'desc'
|
|
727
|
-
}
|
|
728
|
-
|
|
729
|
-
interface ExtendedListRunsOptions {
|
|
730
|
-
kind?: string;
|
|
731
|
-
status?: ExtendedRunStatus | ExtendedRunStatus[];
|
|
732
|
-
limit?: number;
|
|
733
|
-
offset?: number;
|
|
734
|
-
orderBy?: 'createdAt' | 'startedAt' | 'finishedAt';
|
|
735
|
-
orderDir?: 'asc' | 'desc';
|
|
736
|
-
}
|
|
737
|
-
|
|
738
|
-
interface ListEventsOptions {
|
|
739
|
-
stepKey?: string;
|
|
740
|
-
level?: 'info' | 'warn' | 'error';
|
|
741
|
-
limit?: number; // default: 1000
|
|
742
|
-
offset?: number;
|
|
743
|
-
}
|
|
744
|
-
|
|
745
|
-
interface PaginatedResult<T> {
|
|
746
|
-
items: T[];
|
|
747
|
-
total: number;
|
|
748
|
-
limit?: number;
|
|
749
|
-
offset?: number;
|
|
750
|
-
}
|
|
751
|
-
```
|
|
752
|
-
|
|
753
|
-
### MemoryStorageAdapter
|
|
754
|
-
|
|
755
|
-
In-memory storage. Data is lost on process exit.
|
|
756
|
-
|
|
757
|
-
```typescript
|
|
758
|
-
import { MemoryStorageAdapter } from '@multiplier-labs/stepflow';
|
|
759
|
-
|
|
760
|
-
const storage = new MemoryStorageAdapter();
|
|
761
|
-
|
|
762
|
-
// Testing utilities
|
|
763
|
-
storage.clear(): void;
|
|
764
|
-
storage.getStats(): { runs: number; steps: number; events: number };
|
|
765
|
-
```
|
|
766
|
-
|
|
767
|
-
### SQLiteStorageAdapter
|
|
768
|
-
|
|
769
|
-
File-based or in-memory SQLite storage.
|
|
770
|
-
|
|
771
|
-
```typescript
|
|
772
|
-
import { SQLiteStorageAdapter } from '@multiplier-labs/stepflow';
|
|
773
|
-
import Database from 'better-sqlite3';
|
|
774
|
-
|
|
775
|
-
const db = new Database('./workflows.db');
|
|
776
|
-
const storage = new SQLiteStorageAdapter({
|
|
777
|
-
db,
|
|
778
|
-
autoCreateTables: true, // default: true
|
|
779
|
-
tablePrefix: 'workflow', // default: 'workflow'
|
|
780
|
-
});
|
|
781
|
-
|
|
782
|
-
// Additional methods (implementation-only; not part of the StorageAdapter interface)
|
|
783
|
-
storage.transactionSync<T>(fn: () => T): T;
|
|
784
|
-
await storage.getInterruptedRuns(): Promise<WorkflowRunRecord[]>;
|
|
785
|
-
await storage.getLastCompletedStep(runId): Promise<WorkflowRunStepRecord | null>;
|
|
786
|
-
storage.close(): void;
|
|
787
|
-
storage.getStats(): { runs: number; steps: number; events: number };
|
|
788
|
-
```
|
|
789
|
-
|
|
790
|
-
**Tables created:** `workflow_runs`, `workflow_run_steps`, `workflow_events`
|
|
791
|
-
|
|
792
|
-
---
|
|
793
|
-
|
|
794
|
-
## PostgreSQL Storage Adapter
|
|
795
|
-
|
|
796
|
-
Production-ready PostgreSQL backend with connection pooling, distributed worker support, and atomic dequeue.
|
|
797
|
-
|
|
798
|
-
### Configuration
|
|
799
|
-
|
|
800
|
-
```typescript
|
|
801
|
-
import { PostgresStorageAdapter } from '@multiplier-labs/stepflow';
|
|
802
|
-
|
|
803
|
-
interface PostgresStorageConfig {
|
|
804
|
-
connectionString?: string; // e.g., "postgresql://user:pass@localhost:5432/dbname"
|
|
805
|
-
pool?: pg.Pool; // existing pool for connection sharing
|
|
806
|
-
poolConfig?: pg.PoolConfig; // pool configuration options
|
|
807
|
-
schema?: string; // schema name (default: 'public')
|
|
808
|
-
autoMigrate?: boolean; // auto-create tables (default: true)
|
|
809
|
-
}
|
|
810
|
-
```
|
|
811
|
-
|
|
812
|
-
### Usage
|
|
813
|
-
|
|
814
|
-
```typescript
|
|
815
|
-
// Option 1: Connection string
|
|
816
|
-
const storage = new PostgresStorageAdapter({
|
|
817
|
-
connectionString: process.env.DATABASE_URL,
|
|
818
|
-
schema: 'stepflow',
|
|
819
|
-
});
|
|
820
|
-
await storage.initialize();
|
|
821
|
-
|
|
822
|
-
// Option 2: Existing pool (for connection sharing)
|
|
823
|
-
import pg from 'pg';
|
|
824
|
-
const pool = new pg.Pool({ connectionString: process.env.DATABASE_URL, max: 20 });
|
|
825
|
-
const storage = new PostgresStorageAdapter({ pool, schema: 'stepflow' });
|
|
826
|
-
await storage.initialize();
|
|
827
|
-
|
|
828
|
-
// Use with engine
|
|
829
|
-
const engine = new WorkflowEngine({ storage });
|
|
830
|
-
await engine.initialize();
|
|
831
|
-
```
|
|
832
|
-
|
|
833
|
-
The adapter only closes the pool on `close()` if it created it. Externally-provided pools are left open.
|
|
834
|
-
|
|
835
|
-
### All Methods
|
|
836
|
-
|
|
837
|
-
The adapter implements both `StorageAdapter` and `WorkflowStorage`, providing:
|
|
838
|
-
|
|
839
|
-
**StorageAdapter methods (legacy, used by WorkflowEngine):**
|
|
840
|
-
|
|
841
|
-
```typescript
|
|
842
|
-
await storage.createRun(run): Promise<WorkflowRunRecord>;
|
|
843
|
-
await storage.getRun(runId): Promise<WorkflowRunRecord | null>;
|
|
844
|
-
await storage.updateRun(runId, updates): Promise<void>;
|
|
845
|
-
await storage.listRuns(options?): Promise<PaginatedResult<WorkflowRunRecord>>;
|
|
846
|
-
await storage.createStep(step): Promise<WorkflowRunStepRecord>;
|
|
847
|
-
await storage.getStep(stepId): Promise<WorkflowRunStepRecord | null>;
|
|
848
|
-
await storage.updateStep(stepId, updates): Promise<void>;
|
|
849
|
-
await storage.getStepsForRun(runId): Promise<WorkflowRunStepRecord[]>;
|
|
850
|
-
await storage.saveEvent(event): Promise<void>;
|
|
851
|
-
await storage.getEventsForRun(runId, options?): Promise<WorkflowEventRecord[]>;
|
|
852
|
-
await storage.transaction<T>(fn): Promise<T>;
|
|
853
|
-
await storage.deleteOldRuns(olderThan): Promise<number>;
|
|
854
|
-
await storage.initialize(): Promise<void>;
|
|
855
|
-
await storage.close(): Promise<void>;
|
|
856
|
-
```
|
|
857
|
-
|
|
858
|
-
**WorkflowStorage methods (used for queue operations and step results):**
|
|
859
|
-
|
|
860
|
-
```typescript
|
|
861
|
-
await storage.deleteRun(id): Promise<void>;
|
|
862
|
-
await storage.dequeueRun(workflowKinds?): Promise<WorkflowRunRecord | null>;
|
|
863
|
-
await storage.cleanupStaleRuns(defaultTimeoutMs?): Promise<number>;
|
|
864
|
-
await storage.markRunsAsFailed(runIds, reason): Promise<void>;
|
|
865
|
-
await storage.getStepResult(runId, stepName): Promise<StepResult | undefined>;
|
|
866
|
-
await storage.getStepResults(runId): Promise<StepResult[]>;
|
|
867
|
-
await storage.saveStepResult(result): Promise<void>;
|
|
868
|
-
```
|
|
869
|
-
|
|
870
|
-
**Resume support** (implementation-only; not part of the StorageAdapter interface):
|
|
871
|
-
|
|
872
|
-
```typescript
|
|
873
|
-
await storage.getInterruptedRuns(): Promise<WorkflowRunRecord[]>;
|
|
874
|
-
await storage.getLastCompletedStep(runId): Promise<WorkflowRunStepRecord | null>;
|
|
875
|
-
```
|
|
876
|
-
|
|
877
|
-
**Diagnostics:**
|
|
878
|
-
|
|
879
|
-
```typescript
|
|
880
|
-
await storage.getStats(): Promise<{ runs: number; steps: number; events: number }>;
|
|
881
|
-
```
|
|
882
|
-
|
|
883
|
-
### Atomic Dequeue
|
|
884
|
-
|
|
885
|
-
Safe concurrent processing by multiple workers using `FOR UPDATE SKIP LOCKED`:
|
|
886
|
-
|
|
887
|
-
```typescript
|
|
888
|
-
const run = await storage.dequeueRun(['sync.xero', 'sync.directo']);
|
|
889
|
-
if (run) {
|
|
890
|
-
// run.status is now 'running', startedAt is set
|
|
891
|
-
console.log(`Processing ${run.kind}: ${run.id}`);
|
|
892
|
-
}
|
|
893
|
-
```
|
|
894
|
-
|
|
895
|
-
The dequeue query:
|
|
896
|
-
|
|
897
|
-
```sql
|
|
898
|
-
UPDATE {schema}.runs
|
|
899
|
-
SET status = 'running', started_at = NOW()
|
|
900
|
-
WHERE id = (
|
|
901
|
-
SELECT id FROM {schema}.runs
|
|
902
|
-
WHERE status = 'queued' AND kind = ANY($1::text[])
|
|
903
|
-
ORDER BY priority DESC, created_at ASC
|
|
904
|
-
LIMIT 1
|
|
905
|
-
FOR UPDATE SKIP LOCKED
|
|
906
|
-
)
|
|
907
|
-
RETURNING *;
|
|
908
|
-
```
|
|
909
|
-
|
|
910
|
-
This ensures: one worker per run, priority ordering, FIFO within same priority, no blocking between workers.
|
|
911
|
-
|
|
912
|
-
### Stale Run Cleanup
|
|
913
|
-
|
|
914
|
-
```typescript
|
|
915
|
-
// Mark runs as timed out if they exceed their timeout_ms (or a default of 10 minutes)
|
|
916
|
-
const count = await storage.cleanupStaleRuns(600000);
|
|
917
|
-
|
|
918
|
-
// Mark specific runs as failed (e.g., during graceful shutdown)
|
|
919
|
-
await storage.markRunsAsFailed([runId1, runId2], 'Worker process was terminated');
|
|
920
|
-
```
|
|
921
|
-
|
|
922
|
-
### Step Results (WorkflowStorage Interface)
|
|
923
|
-
|
|
924
|
-
```typescript
|
|
925
|
-
// Save or upsert a step result (uses ON CONFLICT DO UPDATE on run_id + step_name)
|
|
926
|
-
await storage.saveStepResult({
|
|
927
|
-
runId: run.id,
|
|
928
|
-
stepName: 'xero.accounts',
|
|
929
|
-
status: 'running',
|
|
930
|
-
attempt: 1,
|
|
931
|
-
startedAt: new Date(),
|
|
932
|
-
});
|
|
933
|
-
|
|
934
|
-
// Get a single step result
|
|
935
|
-
const step = await storage.getStepResult(runId, 'xero.accounts');
|
|
936
|
-
|
|
937
|
-
// Get all step results for a run
|
|
938
|
-
const steps = await storage.getStepResults(runId);
|
|
939
|
-
```
|
|
940
|
-
|
|
941
|
-
### Database Schema
|
|
942
|
-
|
|
943
|
-
When `autoMigrate` is enabled (default), the adapter creates the following tables. All table names are prefixed with the configured `schema` (default: `'public'`).
|
|
944
|
-
|
|
945
|
-
#### Table: `{schema}.runs`
|
|
946
|
-
|
|
947
|
-
```sql
|
|
948
|
-
CREATE TABLE IF NOT EXISTS {schema}.runs (
|
|
949
|
-
id TEXT PRIMARY KEY,
|
|
950
|
-
kind TEXT NOT NULL,
|
|
951
|
-
status TEXT NOT NULL,
|
|
952
|
-
parent_run_id TEXT,
|
|
953
|
-
input_json JSONB NOT NULL DEFAULT '{}',
|
|
954
|
-
metadata_json JSONB NOT NULL DEFAULT '{}',
|
|
955
|
-
context_json JSONB NOT NULL DEFAULT '{}',
|
|
956
|
-
output_json JSONB,
|
|
957
|
-
error_json JSONB,
|
|
958
|
-
priority INTEGER NOT NULL DEFAULT 0,
|
|
959
|
-
timeout_ms INTEGER,
|
|
960
|
-
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
961
|
-
started_at TIMESTAMPTZ,
|
|
962
|
-
finished_at TIMESTAMPTZ,
|
|
963
|
-
CONSTRAINT runs_status_check CHECK (
|
|
964
|
-
status IN ('pending', 'queued', 'running', 'succeeded', 'failed', 'canceled', 'timeout')
|
|
965
|
-
)
|
|
966
|
-
);
|
|
967
|
-
|
|
968
|
-
CREATE INDEX idx_runs_kind_status ON {schema}.runs (kind, status);
|
|
969
|
-
CREATE INDEX idx_runs_parent ON {schema}.runs (parent_run_id);
|
|
970
|
-
CREATE INDEX idx_runs_created ON {schema}.runs (created_at DESC);
|
|
971
|
-
CREATE INDEX idx_runs_status ON {schema}.runs (status);
|
|
972
|
-
CREATE INDEX idx_runs_priority ON {schema}.runs (priority DESC, created_at ASC);
|
|
973
|
-
```
|
|
974
|
-
|
|
975
|
-
#### Table: `{schema}.workflow_run_steps`
|
|
976
|
-
|
|
977
|
-
Used by the `StorageAdapter` interface (legacy step tracking by the engine).
|
|
978
|
-
|
|
979
|
-
```sql
|
|
980
|
-
CREATE TABLE IF NOT EXISTS {schema}.workflow_run_steps (
|
|
981
|
-
id TEXT PRIMARY KEY,
|
|
982
|
-
run_id TEXT NOT NULL REFERENCES {schema}.runs (id) ON DELETE CASCADE,
|
|
983
|
-
step_key TEXT NOT NULL,
|
|
984
|
-
step_name TEXT,
|
|
985
|
-
status TEXT NOT NULL,
|
|
986
|
-
attempt INTEGER NOT NULL DEFAULT 1,
|
|
987
|
-
result_json JSONB,
|
|
988
|
-
error_json JSONB,
|
|
989
|
-
started_at TIMESTAMPTZ,
|
|
990
|
-
finished_at TIMESTAMPTZ,
|
|
991
|
-
CONSTRAINT workflow_run_steps_status_check CHECK (
|
|
992
|
-
status IN ('pending', 'running', 'succeeded', 'failed', 'skipped', 'completed')
|
|
993
|
-
)
|
|
994
|
-
);
|
|
995
|
-
|
|
996
|
-
CREATE INDEX idx_workflow_run_steps_run ON {schema}.workflow_run_steps (run_id);
|
|
997
|
-
CREATE INDEX idx_workflow_run_steps_run_key ON {schema}.workflow_run_steps (run_id, step_key);
|
|
998
|
-
```
|
|
999
|
-
|
|
1000
|
-
#### Table: `{schema}.stepflow_step_results`
|
|
1001
|
-
|
|
1002
|
-
Used by the `WorkflowStorage` interface (step result tracking for external consumers).
|
|
1003
|
-
|
|
1004
|
-
```sql
|
|
1005
|
-
CREATE TABLE IF NOT EXISTS {schema}.stepflow_step_results (
|
|
1006
|
-
id TEXT PRIMARY KEY DEFAULT gen_random_uuid()::text,
|
|
1007
|
-
run_id TEXT NOT NULL REFERENCES {schema}.runs (id) ON DELETE CASCADE,
|
|
1008
|
-
step_name TEXT NOT NULL,
|
|
1009
|
-
status TEXT NOT NULL,
|
|
1010
|
-
output_json JSONB,
|
|
1011
|
-
error_json JSONB,
|
|
1012
|
-
attempt INTEGER NOT NULL DEFAULT 1,
|
|
1013
|
-
started_at TIMESTAMPTZ,
|
|
1014
|
-
completed_at TIMESTAMPTZ,
|
|
1015
|
-
CONSTRAINT stepflow_step_results_status_check CHECK (
|
|
1016
|
-
status IN ('pending', 'running', 'completed', 'failed', 'skipped')
|
|
1017
|
-
),
|
|
1018
|
-
UNIQUE (run_id, step_name)
|
|
1019
|
-
);
|
|
1020
|
-
|
|
1021
|
-
CREATE INDEX idx_stepflow_step_results_run ON {schema}.stepflow_step_results (run_id);
|
|
1022
|
-
CREATE INDEX idx_stepflow_step_results_run_name ON {schema}.stepflow_step_results (run_id, step_name);
|
|
1023
|
-
```
|
|
1024
|
-
|
|
1025
|
-
#### Table: `{schema}.workflow_events`
|
|
1026
|
-
|
|
1027
|
-
```sql
|
|
1028
|
-
CREATE TABLE IF NOT EXISTS {schema}.workflow_events (
|
|
1029
|
-
id TEXT PRIMARY KEY,
|
|
1030
|
-
run_id TEXT NOT NULL REFERENCES {schema}.runs (id) ON DELETE CASCADE,
|
|
1031
|
-
step_key TEXT,
|
|
1032
|
-
event_type TEXT NOT NULL,
|
|
1033
|
-
level TEXT NOT NULL,
|
|
1034
|
-
payload_json JSONB,
|
|
1035
|
-
timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
1036
|
-
CONSTRAINT workflow_events_level_check CHECK (
|
|
1037
|
-
level IN ('info', 'warn', 'error')
|
|
1038
|
-
)
|
|
1039
|
-
);
|
|
1040
|
-
|
|
1041
|
-
CREATE INDEX idx_workflow_events_run ON {schema}.workflow_events (run_id);
|
|
1042
|
-
CREATE INDEX idx_workflow_events_run_ts ON {schema}.workflow_events (run_id, timestamp);
|
|
1043
|
-
```
|
|
1044
|
-
|
|
1045
|
-
### Implementation Notes
|
|
1046
|
-
|
|
1047
|
-
- **IDs are TEXT, not UUID.** The adapter uses `generateId()` which produces ULID-like base36 strings (~16 characters) for time-ordering and collision resistance.
|
|
1048
|
-
- **JSONB columns use `_json` suffix** (e.g., `input_json`, `output_json`, `error_json`) to distinguish from the camelCase TypeScript field names.
|
|
1049
|
-
- **`finished_at` (not `completed_at`)** is used on the `runs` table. The `stepflow_step_results` table uses `completed_at`.
|
|
1050
|
-
- **Lazy dependency loading.** `pg` and `kysely` are imported dynamically in `initialize()`, so they're only needed if you actually use the PostgreSQL adapter.
|
|
1051
|
-
- **Schema isolation.** All queries use `db.withSchema(schema)` to support custom schemas and multi-tenant deployments.
|
|
1052
|
-
- **Transaction support.** `storage.transaction(fn)` wraps operations in a PostgreSQL transaction via Kysely.
|
|
1053
|
-
|
|
1054
|
-
### SQL Queries for Debugging
|
|
1055
|
-
|
|
1056
|
-
```sql
|
|
1057
|
-
-- Recent workflow runs
|
|
1058
|
-
SELECT id, kind, status, created_at, started_at, finished_at,
|
|
1059
|
-
error_json->>'message' as error_message
|
|
1060
|
-
FROM {schema}.runs
|
|
1061
|
-
ORDER BY created_at DESC
|
|
1062
|
-
LIMIT 20;
|
|
1063
|
-
|
|
1064
|
-
-- Step results for a run
|
|
1065
|
-
SELECT step_name, status, output_json, error_json, started_at, completed_at
|
|
1066
|
-
FROM {schema}.stepflow_step_results
|
|
1067
|
-
WHERE run_id = 'your-run-id'
|
|
1068
|
-
ORDER BY started_at;
|
|
1069
|
-
|
|
1070
|
-
-- Count by status
|
|
1071
|
-
SELECT status, COUNT(*) FROM {schema}.runs GROUP BY status;
|
|
1072
|
-
|
|
1073
|
-
-- Find stale running workflows
|
|
1074
|
-
SELECT id, kind, started_at, timeout_ms
|
|
1075
|
-
FROM {schema}.runs
|
|
1076
|
-
WHERE status = 'running'
|
|
1077
|
-
AND started_at < NOW() - INTERVAL '10 minutes';
|
|
1078
|
-
|
|
1079
|
-
-- Queue depth
|
|
1080
|
-
SELECT kind, COUNT(*) FROM {schema}.runs
|
|
1081
|
-
WHERE status = 'queued' GROUP BY kind;
|
|
1082
|
-
```
|
|
1083
|
-
|
|
1084
|
-
---
|
|
1085
|
-
|
|
1086
|
-
## Planning System
|
|
1087
|
-
|
|
1088
|
-
The planning system enables dynamic workflow construction from recipes and step handler registries.
|
|
1089
|
-
|
|
1090
|
-
### Recipes
|
|
1091
|
-
|
|
1092
|
-
A recipe is a template for building a workflow plan:
|
|
1093
|
-
|
|
1094
|
-
```typescript
|
|
1095
|
-
interface Recipe {
|
|
1096
|
-
id: string;
|
|
1097
|
-
name: string;
|
|
1098
|
-
description?: string;
|
|
1099
|
-
workflowKind: string;
|
|
1100
|
-
variant: string; // e.g., 'default', 'fast', 'thorough'
|
|
1101
|
-
steps: RecipeStep[];
|
|
1102
|
-
defaults?: RecipeDefaults;
|
|
1103
|
-
conditions?: RecipeCondition[]; // conditions for recipe selection
|
|
1104
|
-
priority?: number; // higher = preferred
|
|
1105
|
-
tags?: string[];
|
|
1106
|
-
}
|
|
1107
|
-
|
|
1108
|
-
interface RecipeStep {
|
|
1109
|
-
key: string;
|
|
1110
|
-
name: string;
|
|
1111
|
-
handlerRef: string; // reference to a registered step handler
|
|
1112
|
-
config?: Record<string, unknown>;
|
|
1113
|
-
onError?: StepErrorStrategy;
|
|
1114
|
-
maxRetries?: number;
|
|
1115
|
-
retryDelay?: number;
|
|
1116
|
-
retryBackoff?: number;
|
|
1117
|
-
timeout?: number;
|
|
1118
|
-
skipCondition?: string; // field reference (dot notation)
|
|
1119
|
-
}
|
|
1120
|
-
|
|
1121
|
-
interface RecipeDefaults {
|
|
1122
|
-
timeout?: number;
|
|
1123
|
-
maxRetries?: number;
|
|
1124
|
-
retryDelay?: number;
|
|
1125
|
-
retryBackoff?: number;
|
|
1126
|
-
onError?: StepErrorStrategy;
|
|
1127
|
-
}
|
|
1128
|
-
```
|
|
1129
|
-
|
|
1130
|
-
### Conditions
|
|
1131
|
-
|
|
1132
|
-
```typescript
|
|
1133
|
-
type ConditionOperator =
|
|
1134
|
-
| 'eq' | 'neq' | 'gt' | 'gte' | 'lt' | 'lte'
|
|
1135
|
-
| 'contains' | 'matches' | 'exists' | 'notExists';
|
|
1136
|
-
|
|
1137
|
-
interface RecipeCondition {
|
|
1138
|
-
field: string; // supports dot notation
|
|
1139
|
-
operator: ConditionOperator;
|
|
1140
|
-
value?: unknown;
|
|
1141
|
-
}
|
|
1142
|
-
```
|
|
1143
|
-
|
|
1144
|
-
### Registries
|
|
1145
|
-
|
|
1146
|
-
```typescript
|
|
1147
|
-
import { MemoryRecipeRegistry, MemoryStepHandlerRegistry, createRegistry } from '@multiplier-labs/stepflow';
|
|
1148
|
-
|
|
1149
|
-
// Create both registries at once
|
|
1150
|
-
const { recipes, handlers } = createRegistry();
|
|
1151
|
-
|
|
1152
|
-
// Register step handlers
|
|
1153
|
-
handlers.register({
|
|
1154
|
-
id: 'fetch-data',
|
|
1155
|
-
name: 'Fetch Data',
|
|
1156
|
-
description: 'Fetches data from API',
|
|
1157
|
-
tags: ['io', 'api'],
|
|
1158
|
-
handler: async (ctx) => { /* ... */ },
|
|
1159
|
-
});
|
|
1160
|
-
|
|
1161
|
-
// Register recipes
|
|
1162
|
-
recipes.register({
|
|
1163
|
-
id: 'data-pipeline-default',
|
|
1164
|
-
name: 'Data Pipeline',
|
|
1165
|
-
workflowKind: 'data.pipeline',
|
|
1166
|
-
variant: 'default',
|
|
1167
|
-
steps: [
|
|
1168
|
-
{ key: 'fetch', name: 'Fetch', handlerRef: 'fetch-data' },
|
|
1169
|
-
],
|
|
1170
|
-
});
|
|
1171
|
-
|
|
1172
|
-
// Query recipes
|
|
1173
|
-
recipes.getByKind('data.pipeline'): Recipe[];
|
|
1174
|
-
recipes.getVariant('data.pipeline', 'fast'): Recipe | undefined;
|
|
1175
|
-
recipes.getDefault('data.pipeline'): Recipe | undefined;
|
|
1176
|
-
recipes.listVariants('data.pipeline'): string[];
|
|
1177
|
-
recipes.query({ workflowKind: 'data.pipeline', tags: ['fast'] }): Recipe[];
|
|
1178
|
-
```
|
|
1179
|
-
|
|
1180
|
-
### RuleBasedPlanner
|
|
1181
|
-
|
|
1182
|
-
Selects recipes based on conditions and generates execution plans:
|
|
1183
|
-
|
|
1184
|
-
```typescript
|
|
1185
|
-
import { RuleBasedPlanner } from '@multiplier-labs/stepflow';
|
|
1186
|
-
|
|
1187
|
-
const planner = new RuleBasedPlanner({
|
|
1188
|
-
recipeRegistry: recipes,
|
|
1189
|
-
handlerRegistry: handlers, // optional
|
|
1190
|
-
validateHandlers: true, // validate handler refs exist
|
|
1191
|
-
});
|
|
1192
|
-
|
|
1193
|
-
// Select best recipe for given input
|
|
1194
|
-
const selection = await planner.selectRecipe('data.pipeline', { size: 'large' });
|
|
1195
|
-
|
|
1196
|
-
// Generate a full plan
|
|
1197
|
-
const plan = await planner.plan('data.pipeline', { size: 'large' }, {
|
|
1198
|
-
constraints: { maxDuration: 60000, priority: 'speed' },
|
|
1199
|
-
hints: { preferredVariant: 'fast', skipSteps: ['optional-step'] },
|
|
1200
|
-
});
|
|
1201
|
-
|
|
1202
|
-
// Validate and estimate
|
|
1203
|
-
const validation = planner.validatePlan(plan);
|
|
1204
|
-
const estimate = planner.estimateResources(plan);
|
|
1205
|
-
```
|
|
1206
|
-
|
|
1207
|
-
### Plan Output
|
|
1208
|
-
|
|
1209
|
-
```typescript
|
|
1210
|
-
interface Plan {
|
|
1211
|
-
id: string;
|
|
1212
|
-
recipeId: string;
|
|
1213
|
-
variant: string;
|
|
1214
|
-
modifications: PlanModification[];
|
|
1215
|
-
steps: PlannedStep[];
|
|
1216
|
-
childWorkflows?: ChildWorkflowPlan[];
|
|
1217
|
-
defaults: RecipeDefaults;
|
|
1218
|
-
reasoning?: string;
|
|
1219
|
-
resourceEstimate?: ResourceEstimate;
|
|
1220
|
-
createdAt: Date;
|
|
1221
|
-
}
|
|
1222
|
-
|
|
1223
|
-
interface ResourceEstimate {
|
|
1224
|
-
apiCalls?: number;
|
|
1225
|
-
tokens?: number;
|
|
1226
|
-
duration?: number;
|
|
1227
|
-
memory?: number;
|
|
1228
|
-
}
|
|
1229
|
-
```
|
|
1230
|
-
|
|
1231
|
-
---
|
|
1232
|
-
|
|
1233
|
-
## Error Handling
|
|
1234
|
-
|
|
1235
|
-
### Error Classes
|
|
1236
|
-
|
|
1237
|
-
```typescript
|
|
1238
|
-
import {
|
|
1239
|
-
WorkflowEngineError,
|
|
1240
|
-
WorkflowNotFoundError,
|
|
1241
|
-
WorkflowAlreadyRegisteredError,
|
|
1242
|
-
RunNotFoundError,
|
|
1243
|
-
StepError,
|
|
1244
|
-
StepTimeoutError,
|
|
1245
|
-
WorkflowCanceledError,
|
|
1246
|
-
WorkflowTimeoutError,
|
|
1247
|
-
WaitForRunTimeoutError,
|
|
1248
|
-
} from '@multiplier-labs/stepflow';
|
|
1249
|
-
```
|
|
1250
|
-
|
|
1251
|
-
| Class | Code | Constructor |
|
|
1252
|
-
|-------|------|-------------|
|
|
1253
|
-
| `WorkflowEngineError` | (custom) | `(code, message, details?)` |
|
|
1254
|
-
| `WorkflowNotFoundError` | `WORKFLOW_NOT_FOUND` | `(kind)` |
|
|
1255
|
-
| `WorkflowAlreadyRegisteredError` | `WORKFLOW_ALREADY_REGISTERED` | `(kind)` |
|
|
1256
|
-
| `RunNotFoundError` | `RUN_NOT_FOUND` | `(runId)` |
|
|
1257
|
-
| `StepError` | `STEP_ERROR` | `(stepKey, message, attempt, cause?)` |
|
|
1258
|
-
| `StepTimeoutError` | `STEP_TIMEOUT` | `(stepKey, timeoutMs)` |
|
|
1259
|
-
| `WorkflowCanceledError` | `WORKFLOW_CANCELED` | `(runId)` |
|
|
1260
|
-
| `WorkflowTimeoutError` | `WORKFLOW_TIMEOUT` | `(runId, timeoutMs)` |
|
|
1261
|
-
| `WaitForRunTimeoutError` | `WAIT_FOR_RUN_TIMEOUT` | `(runId, timeoutMs)` |
|
|
1262
|
-
|
|
1263
|
-
### WorkflowError Record
|
|
1264
|
-
|
|
1265
|
-
```typescript
|
|
1266
|
-
interface WorkflowError {
|
|
1267
|
-
code: string;
|
|
1268
|
-
message: string;
|
|
1269
|
-
stack?: string;
|
|
1270
|
-
details?: Record<string, unknown>;
|
|
1271
|
-
}
|
|
1272
|
-
|
|
1273
|
-
// Convert error instances
|
|
1274
|
-
const record = engineError.toRecord();
|
|
1275
|
-
const record = WorkflowEngineError.fromError(unknownError, 'DEFAULT_CODE');
|
|
1276
|
-
```
|
|
1277
|
-
|
|
1278
|
-
### Step Error Strategies
|
|
1279
|
-
|
|
1280
|
-
| Strategy | Behavior |
|
|
1281
|
-
|----------|----------|
|
|
1282
|
-
| `'fail'` | Stop the workflow immediately (default) |
|
|
1283
|
-
| `'retry'` | Retry up to `maxRetries` times with exponential backoff |
|
|
1284
|
-
| `'skip'` | Mark step as skipped and continue to next step |
|
|
1285
|
-
|
|
1286
|
-
### RunResult
|
|
1287
|
-
|
|
1288
|
-
```typescript
|
|
1289
|
-
interface RunResult {
|
|
1290
|
-
status: 'succeeded' | 'failed' | 'canceled';
|
|
1291
|
-
results: Record<string, unknown>;
|
|
1292
|
-
error?: WorkflowError;
|
|
1293
|
-
duration: number; // ms
|
|
1294
|
-
}
|
|
1295
|
-
```
|
|
1296
|
-
|
|
1297
|
-
---
|
|
1298
|
-
|
|
1299
|
-
## Utilities
|
|
1300
|
-
|
|
1301
|
-
### ID Generation
|
|
1302
|
-
|
|
1303
|
-
```typescript
|
|
1304
|
-
import { generateId } from '@multiplier-labs/stepflow';
|
|
1305
|
-
|
|
1306
|
-
const id = generateId();
|
|
1307
|
-
// ULID-like format: base36 timestamp + random suffix (~16 characters)
|
|
1308
|
-
// Time-ordered, collision-resistant, URL-safe
|
|
1309
|
-
```
|
|
1310
|
-
|
|
1311
|
-
### Logger
|
|
1312
|
-
|
|
1313
|
-
```typescript
|
|
1314
|
-
import { ConsoleLogger, SilentLogger, createScopedLogger } from '@multiplier-labs/stepflow';
|
|
1315
|
-
|
|
1316
|
-
// Logs to console with prefix
|
|
1317
|
-
const logger = new ConsoleLogger('[my-app]'); // default prefix: '[workflow]'
|
|
1318
|
-
|
|
1319
|
-
// No-op logger
|
|
1320
|
-
const silent = new SilentLogger();
|
|
1321
|
-
|
|
1322
|
-
// Create a scoped logger (prefixes messages with runId/stepKey)
|
|
1323
|
-
const scoped = createScopedLogger(logger, runId, stepKey?);
|
|
1324
|
-
|
|
1325
|
-
// Logger interface (compatible with pino, winston, etc.)
|
|
1326
|
-
interface Logger {
|
|
1327
|
-
debug(message: string, ...args: unknown[]): void;
|
|
1328
|
-
info(message: string, ...args: unknown[]): void;
|
|
1329
|
-
warn(message: string, ...args: unknown[]): void;
|
|
1330
|
-
error(message: string, ...args: unknown[]): void;
|
|
1331
|
-
}
|
|
1332
|
-
```
|
|
1333
|
-
|
|
1334
|
-
### Retry Utilities
|
|
1335
|
-
|
|
1336
|
-
```typescript
|
|
1337
|
-
import { withRetry, sleep, calculateRetryDelay, DEFAULT_RETRY_OPTIONS } from '@multiplier-labs/stepflow';
|
|
1338
|
-
|
|
1339
|
-
// Retry a function with exponential backoff
|
|
1340
|
-
const result = await withRetry(
|
|
1341
|
-
() => fetchFromApi(),
|
|
1342
|
-
{
|
|
1343
|
-
maxRetries: 3, // default: 3
|
|
1344
|
-
delay: 1000, // default: 1000ms
|
|
1345
|
-
backoff: 2, // default: 2 (exponential)
|
|
1346
|
-
signal: abortSignal, // optional
|
|
1347
|
-
onRetry: (attempt, error, nextDelay) => {
|
|
1348
|
-
console.log(`Retry ${attempt}, next delay: ${nextDelay}ms`);
|
|
1349
|
-
},
|
|
1350
|
-
}
|
|
1351
|
-
);
|
|
1352
|
-
|
|
1353
|
-
// Sleep with abort support
|
|
1354
|
-
await sleep(1000, abortSignal?);
|
|
1355
|
-
|
|
1356
|
-
// Calculate delay for a given attempt
|
|
1357
|
-
const delay = calculateRetryDelay(attempt, baseDelay, backoff);
|
|
1358
|
-
```
|
|
1359
|
-
|
|
1360
|
-
---
|
|
1361
|
-
|
|
1362
|
-
## TypeScript Types Reference
|
|
1363
|
-
|
|
1364
|
-
All types are exported from the main entry point:
|
|
1365
|
-
|
|
1366
|
-
```typescript
|
|
1367
|
-
import type {
|
|
1368
|
-
// Core
|
|
1369
|
-
WorkflowKind, RunStatus, StepStatus, StepErrorStrategy,
|
|
1370
|
-
WorkflowError, RunResult, Logger, SpawnChildOptions,
|
|
1371
|
-
WorkflowContext, WorkflowStep, WorkflowHooks, WorkflowDefinition,
|
|
1372
|
-
|
|
1373
|
-
// Engine
|
|
1374
|
-
WorkflowEngineConfig, StartRunOptions,
|
|
1375
|
-
|
|
1376
|
-
// Storage (legacy)
|
|
1377
|
-
StorageAdapter, WorkflowRunRecord, WorkflowRunStepRecord,
|
|
1378
|
-
WorkflowEventRecord, ListRunsOptions, ListEventsOptions,
|
|
1379
|
-
PaginatedResult,
|
|
1380
|
-
|
|
1381
|
-
// Storage (extended)
|
|
1382
|
-
WorkflowStorage, ExtendedWorkflowRunRecord, ExtendedRunStatus,
|
|
1383
|
-
ExtendedStepStatus, ExtendedListRunsOptions,
|
|
1384
|
-
CreateRunInput, UpdateRunInput, StepResult, StepRecord,
|
|
1385
|
-
StepflowRunsTable, StepflowStepResultsTable, StepflowDatabase,
|
|
1386
|
-
|
|
1387
|
-
// Events
|
|
1388
|
-
EventTransport, WorkflowEvent, WorkflowEventType,
|
|
1389
|
-
BuiltInEventType, EventCallback, Unsubscribe,
|
|
1390
|
-
SocketIOEventTransportConfig, WebhookEventTransportConfig,
|
|
1391
|
-
WebhookEndpoint, WebhookPayload,
|
|
1392
|
-
|
|
1393
|
-
// Scheduler
|
|
1394
|
-
TriggerType, WorkflowSchedule, Scheduler,
|
|
1395
|
-
CronSchedulerConfig, SchedulePersistence,
|
|
1396
|
-
PostgresSchedulePersistenceConfig,
|
|
1397
|
-
|
|
1398
|
-
// Planning
|
|
1399
|
-
Recipe, RecipeStep, RecipeCondition, RecipeDefaults,
|
|
1400
|
-
Plan, PlannedStep, PlanModification, PlanModificationType,
|
|
1401
|
-
Planner, PlanningContext, PlanningConstraints, PlanningHints,
|
|
1402
|
-
ResourceEstimate, RecipeSelectionResult, PlanValidationResult,
|
|
1403
|
-
StepHandlerRegistry, RecipeRegistry, RegisteredStepHandler,
|
|
1404
|
-
RecipeQueryOptions, ConditionOperator, StepHandlerRef,
|
|
1405
|
-
ChildWorkflowPlan, PlanningPriority,
|
|
1406
|
-
|
|
1407
|
-
// Config
|
|
1408
|
-
PostgresStorageConfig, SQLiteStorageConfig,
|
|
1409
|
-
SQLiteSchedulePersistenceConfig,
|
|
1410
|
-
|
|
1411
|
-
// Retry
|
|
1412
|
-
RetryOptions,
|
|
1413
|
-
} from '@multiplier-labs/stepflow';
|
|
1414
|
-
|
|
1415
|
-
// Classes and values
|
|
1416
|
-
import {
|
|
1417
|
-
WorkflowEngine,
|
|
1418
|
-
MemoryStorageAdapter, SQLiteStorageAdapter,
|
|
1419
|
-
PostgresStorageAdapter, PostgresStorageAdapter as PostgresStorage,
|
|
1420
|
-
MemoryEventTransport, SocketIOEventTransport, WebhookEventTransport,
|
|
1421
|
-
CronScheduler, SQLiteSchedulePersistence, PostgresSchedulePersistence,
|
|
1422
|
-
MemoryStepHandlerRegistry, MemoryRecipeRegistry,
|
|
1423
|
-
createRegistry, RuleBasedPlanner,
|
|
1424
|
-
WorkflowEngineError, WorkflowNotFoundError,
|
|
1425
|
-
WorkflowAlreadyRegisteredError, RunNotFoundError,
|
|
1426
|
-
StepError, StepTimeoutError,
|
|
1427
|
-
WorkflowCanceledError, WorkflowTimeoutError, WaitForRunTimeoutError,
|
|
1428
|
-
ConsoleLogger, SilentLogger, createScopedLogger,
|
|
1429
|
-
generateId, sleep, withRetry, calculateRetryDelay,
|
|
1430
|
-
DEFAULT_RETRY_OPTIONS,
|
|
1431
|
-
} from '@multiplier-labs/stepflow';
|
|
1432
|
-
```
|
|
1433
|
-
|
|
1434
|
-
Subpath exports are also available:
|
|
1435
|
-
|
|
1436
|
-
```typescript
|
|
1437
|
-
import { ... } from '@multiplier-labs/stepflow/storage';
|
|
1438
|
-
import { ... } from '@multiplier-labs/stepflow/events';
|
|
1439
|
-
import { ... } from '@multiplier-labs/stepflow/scheduler';
|
|
1440
|
-
```
|
|
1441
|
-
|
|
1442
|
-
---
|
|
1443
|
-
|
|
1444
|
-
## Downstream Integration Patterns
|
|
1445
|
-
|
|
1446
|
-
This section documents patterns for integrating Stepflow into application services. The examples below are based on a data synchronization use case but apply to any domain.
|
|
1447
|
-
|
|
1448
|
-
### Fastify Plugin Pattern
|
|
1449
|
-
|
|
1450
|
-
Wrap Stepflow storage as a Fastify plugin for API-level workflow management:
|
|
1451
|
-
|
|
1452
|
-
```typescript
|
|
1453
|
-
// plugins/stepflow.ts
|
|
1454
|
-
import fp from 'fastify-plugin';
|
|
1455
|
-
import { PostgresStorageAdapter, type WorkflowRunRecord, type PaginatedResult } from '@multiplier-labs/stepflow';
|
|
1456
|
-
|
|
1457
|
-
export interface StepflowService {
|
|
1458
|
-
storage: PostgresStorageAdapter;
|
|
1459
|
-
|
|
1460
|
-
queueWorkflow(
|
|
1461
|
-
kind: string,
|
|
1462
|
-
input: Record<string, unknown>,
|
|
1463
|
-
options?: { priority?: number; timeoutMs?: number }
|
|
1464
|
-
): Promise<WorkflowRunRecord>;
|
|
1465
|
-
|
|
1466
|
-
getRun(runId: string): Promise<WorkflowRunRecord | undefined>;
|
|
1467
|
-
|
|
1468
|
-
listRuns(options?: {
|
|
1469
|
-
kind?: string;
|
|
1470
|
-
status?: string | string[];
|
|
1471
|
-
limit?: number;
|
|
1472
|
-
offset?: number;
|
|
1473
|
-
}): Promise<PaginatedResult<WorkflowRunRecord>>;
|
|
1474
|
-
|
|
1475
|
-
cancelRun(runId: string): Promise<void>;
|
|
1476
|
-
}
|
|
1477
|
-
|
|
1478
|
-
export default fp(async (fastify) => {
|
|
1479
|
-
const storage = new PostgresStorageAdapter({
|
|
1480
|
-
pool: fastify.pg.pool, // share the app's pool
|
|
1481
|
-
schema: 'stepflow',
|
|
1482
|
-
});
|
|
1483
|
-
await storage.initialize();
|
|
1484
|
-
|
|
1485
|
-
const service: StepflowService = {
|
|
1486
|
-
storage,
|
|
1487
|
-
|
|
1488
|
-
async queueWorkflow(kind, input, options = {}) {
|
|
1489
|
-
return storage.createRun({
|
|
1490
|
-
kind,
|
|
1491
|
-
status: 'queued',
|
|
1492
|
-
input,
|
|
1493
|
-
context: {},
|
|
1494
|
-
priority: options.priority ?? 0,
|
|
1495
|
-
timeoutMs: options.timeoutMs,
|
|
1496
|
-
});
|
|
1497
|
-
},
|
|
1498
|
-
|
|
1499
|
-
async getRun(runId) {
|
|
1500
|
-
return (await storage.getRun(runId)) ?? undefined;
|
|
1501
|
-
},
|
|
1502
|
-
|
|
1503
|
-
async listRuns(options = {}) {
|
|
1504
|
-
return storage.listRuns(options);
|
|
1505
|
-
},
|
|
1506
|
-
|
|
1507
|
-
async cancelRun(runId) {
|
|
1508
|
-
await storage.updateRun(runId, {
|
|
1509
|
-
status: 'canceled',
|
|
1510
|
-
finishedAt: new Date(),
|
|
1511
|
-
});
|
|
1512
|
-
},
|
|
1513
|
-
};
|
|
1514
|
-
|
|
1515
|
-
fastify.decorate('stepflow', service);
|
|
1516
|
-
|
|
1517
|
-
fastify.addHook('onClose', async () => {
|
|
1518
|
-
await storage.close();
|
|
1519
|
-
});
|
|
1520
|
-
});
|
|
1521
|
-
```
|
|
1522
|
-
|
|
1523
|
-
### Worker Service Pattern
|
|
1524
|
-
|
|
1525
|
-
A background worker that polls for queued workflows:
|
|
1526
|
-
|
|
1527
|
-
```typescript
|
|
1528
|
-
import { PostgresStorageAdapter } from '@multiplier-labs/stepflow';
|
|
1529
|
-
|
|
1530
|
-
const storage = new PostgresStorageAdapter({
|
|
1531
|
-
connectionString: process.env.DATABASE_URL,
|
|
1532
|
-
schema: 'stepflow',
|
|
1533
|
-
});
|
|
1534
|
-
await storage.initialize();
|
|
1535
|
-
|
|
1536
|
-
let currentRunId: string | undefined;
|
|
1537
|
-
let running = true;
|
|
1538
|
-
|
|
1539
|
-
// Poll loop
|
|
1540
|
-
async function poll() {
|
|
1541
|
-
while (running) {
|
|
1542
|
-
const run = await storage.dequeueRun(['sync.xero', 'sync.directo']);
|
|
1543
|
-
|
|
1544
|
-
if (run) {
|
|
1545
|
-
currentRunId = run.id;
|
|
1546
|
-
try {
|
|
1547
|
-
await processRun(run);
|
|
1548
|
-
await storage.updateRun(run.id, {
|
|
1549
|
-
status: 'succeeded',
|
|
1550
|
-
output: { /* results */ },
|
|
1551
|
-
finishedAt: new Date(),
|
|
1552
|
-
});
|
|
1553
|
-
} catch (err) {
|
|
1554
|
-
await storage.updateRun(run.id, {
|
|
1555
|
-
status: 'failed',
|
|
1556
|
-
error: { code: 'WORKFLOW_ERROR', message: err.message },
|
|
1557
|
-
finishedAt: new Date(),
|
|
1558
|
-
});
|
|
1559
|
-
} finally {
|
|
1560
|
-
currentRunId = undefined;
|
|
1561
|
-
}
|
|
1562
|
-
}
|
|
1563
|
-
|
|
1564
|
-
await new Promise(r => setTimeout(r, 5000));
|
|
1565
|
-
|
|
1566
|
-
// Periodic stale run cleanup
|
|
1567
|
-
await storage.cleanupStaleRuns(600000);
|
|
1568
|
-
}
|
|
1569
|
-
}
|
|
1570
|
-
|
|
1571
|
-
// Graceful shutdown
|
|
1572
|
-
process.on('SIGTERM', async () => {
|
|
1573
|
-
running = false;
|
|
1574
|
-
if (currentRunId) {
|
|
1575
|
-
await storage.markRunsAsFailed(
|
|
1576
|
-
[currentRunId],
|
|
1577
|
-
'Worker process was terminated during workflow execution'
|
|
1578
|
-
);
|
|
1579
|
-
}
|
|
1580
|
-
await storage.close();
|
|
1581
|
-
process.exit(0);
|
|
1582
|
-
});
|
|
1583
|
-
```
|
|
1584
|
-
|
|
1585
|
-
### Step Tracking with saveStepResult
|
|
1586
|
-
|
|
1587
|
-
Track granular step progress within a workflow run:
|
|
1588
|
-
|
|
1589
|
-
```typescript
|
|
1590
|
-
async function processRun(run: WorkflowRunRecord) {
|
|
1591
|
-
const steps = ['fetch_accounts', 'fetch_contacts', 'fetch_invoices'];
|
|
1592
|
-
|
|
1593
|
-
for (const stepName of steps) {
|
|
1594
|
-
const startedAt = new Date();
|
|
1595
|
-
|
|
1596
|
-
await storage.saveStepResult({
|
|
1597
|
-
runId: run.id,
|
|
1598
|
-
stepName,
|
|
1599
|
-
status: 'running',
|
|
1600
|
-
attempt: 1,
|
|
1601
|
-
startedAt,
|
|
1602
|
-
});
|
|
1603
|
-
|
|
1604
|
-
try {
|
|
1605
|
-
const result = await executeStep(stepName, run.input);
|
|
1606
|
-
|
|
1607
|
-
await storage.saveStepResult({
|
|
1608
|
-
runId: run.id,
|
|
1609
|
-
stepName,
|
|
1610
|
-
status: 'completed',
|
|
1611
|
-
output: { rowsRead: result.read, rowsWritten: result.written },
|
|
1612
|
-
attempt: 1,
|
|
1613
|
-
startedAt,
|
|
1614
|
-
completedAt: new Date(),
|
|
1615
|
-
});
|
|
1616
|
-
} catch (err) {
|
|
1617
|
-
await storage.saveStepResult({
|
|
1618
|
-
runId: run.id,
|
|
1619
|
-
stepName,
|
|
1620
|
-
status: 'failed',
|
|
1621
|
-
error: { code: 'STEP_FAILED', message: err.message },
|
|
1622
|
-
attempt: 1,
|
|
1623
|
-
startedAt,
|
|
1624
|
-
completedAt: new Date(),
|
|
1625
|
-
});
|
|
1626
|
-
throw err;
|
|
1627
|
-
}
|
|
1628
|
-
}
|
|
1629
|
-
}
|
|
1630
|
-
```
|
|
1631
|
-
|
|
1632
|
-
### API Routes Pattern
|
|
1633
|
-
|
|
1634
|
-
```typescript
|
|
1635
|
-
// List runs
|
|
1636
|
-
fastify.get('/api/workflows', async (request) => {
|
|
1637
|
-
const { items, total } = await fastify.stepflow.listRuns({ limit: 100 });
|
|
1638
|
-
return { data: items, total };
|
|
1639
|
-
});
|
|
1640
|
-
|
|
1641
|
-
// Get run with steps
|
|
1642
|
-
fastify.get('/api/workflows/:runId', async (request) => {
|
|
1643
|
-
const { runId } = request.params;
|
|
1644
|
-
const run = await fastify.stepflow.getRun(runId);
|
|
1645
|
-
if (!run) return reply.code(404).send({ error: 'Not found' });
|
|
1646
|
-
|
|
1647
|
-
const steps = await fastify.stepflow.storage.getStepResults(runId);
|
|
1648
|
-
return { data: { ...run, steps } };
|
|
1649
|
-
});
|
|
1650
|
-
|
|
1651
|
-
// Queue a workflow
|
|
1652
|
-
fastify.post('/api/workflows/trigger', async (request) => {
|
|
1653
|
-
const { kind, input, priority } = request.body;
|
|
1654
|
-
const run = await fastify.stepflow.queueWorkflow(kind, input, { priority });
|
|
1655
|
-
return { data: { runId: run.id, status: run.status } };
|
|
1656
|
-
});
|
|
1657
|
-
|
|
1658
|
-
// Cancel a workflow
|
|
1659
|
-
fastify.post('/api/workflows/:runId/cancel', async (request) => {
|
|
1660
|
-
await fastify.stepflow.cancelRun(request.params.runId);
|
|
1661
|
-
return { data: { message: 'Workflow canceled' } };
|
|
1662
|
-
});
|
|
1663
|
-
```
|