@multiplier-labs/stepflow 0.3.4 → 0.3.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/README.md +9 -0
  2. package/dist/{chunk-5XJBAPXF.js → chunk-4IDPWWGS.js} +15 -4
  3. package/dist/chunk-4IDPWWGS.js.map +1 -0
  4. package/dist/{chunk-U3PCVK6O.js → chunk-T6GAZBTQ.js} +70 -38
  5. package/dist/chunk-T6GAZBTQ.js.map +1 -0
  6. package/dist/{chunk-FMPJBK2N.js → chunk-TVSADUYU.js} +3 -5
  7. package/dist/{chunk-FMPJBK2N.js.map → chunk-TVSADUYU.js.map} +1 -1
  8. package/dist/events/index.d.ts +43 -3
  9. package/dist/events/index.js +7 -3
  10. package/dist/{index-Cd0IZ40L.d.ts → index-CdXDv30T.d.ts} +10 -2
  11. package/dist/index.d.ts +5 -4
  12. package/dist/index.js +159 -46
  13. package/dist/index.js.map +1 -1
  14. package/dist/scheduler/index.d.ts +2 -2
  15. package/dist/scheduler/index.js +1 -1
  16. package/dist/storage/index.d.ts +6 -4
  17. package/dist/storage/index.js +1 -1
  18. package/dist/{types-DSEjiLMj.d.ts → types-BfWe5yP6.d.ts} +2 -0
  19. package/package.json +26 -14
  20. package/.codekin/reports/code-review/2026-04-11_code-review-daily.md +0 -146
  21. package/.codekin/reports/code-review/2026-04-13_code-review-daily.md +0 -74
  22. package/.codekin/reports/comments/2026-04-04_comment-assessment.md +0 -179
  23. package/.codekin/reports/comments/2026-04-11_comment-assessment.md +0 -161
  24. package/.codekin/reports/comments/2026-04-14_comment-assessment.md +0 -34
  25. package/.codekin/reports/complexity/2026-04-03_complexity-report.md +0 -152
  26. package/.codekin/reports/complexity/2026-04-11_complexity-report.md +0 -152
  27. package/.codekin/reports/complexity/2026-04-14_complexity-report.md +0 -34
  28. package/.codekin/reports/dependencies/2026-04-04_dependency-health.md +0 -94
  29. package/.codekin/reports/dependencies/2026-04-11_dependency-health.md +0 -93
  30. package/.codekin/reports/docs-audit/2026-04-04_docs-audit.md +0 -10
  31. package/.codekin/reports/docs-audit/2026-04-11_docs-audit.md +0 -138
  32. package/.codekin/reports/docs-audit/2026-04-13_docs-audit.md +0 -121
  33. package/.codekin/reports/security/2026-04-03_security-audit.md +0 -208
  34. package/.codekin/reports/security/2026-04-11_security-audit.md +0 -297
  35. package/.github/CODEOWNERS +0 -15
  36. package/.github/dependabot.yml +0 -23
  37. package/.github/workflows/ci.yml +0 -38
  38. package/.github/workflows/publish.yml +0 -42
  39. package/.github/workflows/security.yml +0 -29
  40. package/CHANGELOG.md +0 -34
  41. package/SECURITY.md +0 -40
  42. package/dist/chunk-5XJBAPXF.js.map +0 -1
  43. package/dist/chunk-U3PCVK6O.js.map +0 -1
  44. package/docs/github-packages-setup.md +0 -97
  45. package/docs/stepflow-api-reference.md +0 -1663
  46. package/src/core/engine/concurrency.ts +0 -229
  47. package/src/core/engine/lifecycle.ts +0 -176
  48. package/src/core/engine.test.ts +0 -1435
  49. package/src/core/engine.ts +0 -598
  50. package/src/core/index.ts +0 -7
  51. package/src/core/orchestrator/persistence.ts +0 -165
  52. package/src/core/orchestrator/plan-resolution.ts +0 -140
  53. package/src/core/orchestrator/state-transitions.ts +0 -109
  54. package/src/core/orchestrator/step-runner.ts +0 -427
  55. package/src/core/orchestrator.ts +0 -252
  56. package/src/core/types.ts +0 -220
  57. package/src/events/index.ts +0 -8
  58. package/src/events/memory.test.ts +0 -107
  59. package/src/events/memory.ts +0 -92
  60. package/src/events/socketio.test.ts +0 -378
  61. package/src/events/socketio.ts +0 -262
  62. package/src/events/types.ts +0 -118
  63. package/src/events/webhook.test.ts +0 -949
  64. package/src/events/webhook.ts +0 -807
  65. package/src/index.ts +0 -270
  66. package/src/planning/evaluate-condition.test.ts +0 -153
  67. package/src/planning/evaluate-condition.ts +0 -105
  68. package/src/planning/index.ts +0 -55
  69. package/src/planning/planner.ts +0 -494
  70. package/src/planning/planning.test.ts +0 -727
  71. package/src/planning/registry.test.ts +0 -398
  72. package/src/planning/registry.ts +0 -283
  73. package/src/planning/types.ts +0 -566
  74. package/src/scheduler/cron.test.ts +0 -479
  75. package/src/scheduler/cron.ts +0 -438
  76. package/src/scheduler/index.ts +0 -8
  77. package/src/scheduler/postgres-persistence.ts +0 -530
  78. package/src/scheduler/sqlite-persistence.test.ts +0 -235
  79. package/src/scheduler/sqlite-persistence.ts +0 -286
  80. package/src/scheduler/types.ts +0 -73
  81. package/src/storage/index.ts +0 -27
  82. package/src/storage/memory.test.ts +0 -349
  83. package/src/storage/memory.ts +0 -227
  84. package/src/storage/postgres-core.ts +0 -550
  85. package/src/storage/postgres-migration.test.ts +0 -69
  86. package/src/storage/postgres-transaction-pagination.test.ts +0 -107
  87. package/src/storage/postgres.ts +0 -835
  88. package/src/storage/sqlite-core.ts +0 -151
  89. package/src/storage/sqlite.test.ts +0 -642
  90. package/src/storage/sqlite.ts +0 -717
  91. package/src/storage/types.ts +0 -343
  92. package/src/utils/errors.test.ts +0 -150
  93. package/src/utils/errors.ts +0 -161
  94. package/src/utils/id.ts +0 -21
  95. package/src/utils/index.ts +0 -10
  96. package/src/utils/logger.test.ts +0 -121
  97. package/src/utils/logger.ts +0 -96
  98. package/src/utils/pinned-fetch.test.ts +0 -84
  99. package/src/utils/pinned-fetch.ts +0 -118
  100. package/src/utils/postgres-deps.ts +0 -79
  101. package/src/utils/retry.test.ts +0 -177
  102. package/src/utils/retry.ts +0 -121
  103. package/src/utils/safe-json.test.ts +0 -166
  104. package/src/utils/safe-json.ts +0 -107
  105. package/src/utils/webhook-signature.test.ts +0 -111
  106. package/src/utils/webhook-signature.ts +0 -74
  107. package/tsconfig.json +0 -21
  108. package/tsup.config.ts +0 -14
  109. package/vitest.config.ts +0 -9
@@ -1,835 +0,0 @@
1
- /**
2
- * PostgreSQL storage adapter using Kysely.
3
- *
4
- * Provides durable persistence for workflow runs, steps, and events with
5
- * support for distributed deployments and connection pooling.
6
- *
7
- * ## Module layout
8
- *
9
- * - `PostgresStorageAdapter` (this file) owns the `pg.Pool` / Kysely
10
- * instance, runs schema migrations on `initialize()`, and implements all
11
- * non-CRUD methods: connection lifecycle (`close`, `transaction`),
12
- * workflow-level operations (`dequeueRun`, `cleanupStaleRuns`,
13
- * `markRunsAsFailed`, `getStats`, `deleteRun`, `deleteOldRuns`,
14
- * `getInterruptedRuns`, `getLastCompletedStep`), and the
15
- * `stepflow_step_results` accessors.
16
- *
17
- * - `PostgresTransactionAdapter` (this file) is a thin shell used by
18
- * `PostgresStorageAdapter.transaction()`. It binds a transaction-scoped
19
- * Kysely query builder; everything else comes from the shared core.
20
- * Exported only for regression tests.
21
- *
22
- * - `PostgresStorageCore` (./postgres-core.ts) holds the shared CRUD
23
- * methods (`createRun`, `getRun`, `updateRun`, `listRuns`, `createStep`,
24
- * `getStep`, `updateStep`, `getStepsForRun`, `saveEvent`,
25
- * `getEventsForRun`), the row mappers, and the JSON-parse helpers. Both
26
- * classes here extend it.
27
- */
28
-
29
- import type { Kysely as KyselyType } from 'kysely';
30
- import type { Pool, PoolConfig } from 'pg';
31
- import type { Logger } from '../core/types.js';
32
- import { loadPostgresDeps } from '../utils/postgres-deps.js';
33
-
34
- // Lazy-loaded dependencies - populated by loadPostgresDeps() during initialize()
35
- let Kysely: any;
36
- let PostgresDialect: any;
37
- let sql: any;
38
- let pgModule: any;
39
- import { generateId } from '../utils/id.js';
40
- import type {
41
- StorageAdapter,
42
- WorkflowRunRecord,
43
- WorkflowRunStepRecord,
44
- StepResult,
45
- ExtendedStepStatus,
46
- } from './types.js';
47
- import {
48
- PostgresStorageCore,
49
- mapRunRow,
50
- mapStepRow,
51
- safeParseOptionalField,
52
- type CoreSchemaQueryBuilder,
53
- type PostgresCoreDatabase,
54
- type SchemaQueryBuilder,
55
- type WorkflowRunsTable,
56
- type WorkflowRunStepsTable,
57
- } from './postgres-core.js';
58
-
59
- /** Strip `stack` from a generic error record before persistence. */
60
- function stripStack<T extends Record<string, unknown>>(error: T): Omit<T, 'stack'> {
61
- const { stack: _stack, ...rest } = error;
62
- return rest as Omit<T, 'stack'>;
63
- }
64
-
65
- // ============================================================================
66
- // Database Types (Kysely schema)
67
- // ============================================================================
68
- //
69
- // The `runs`, `workflow_run_steps`, and `workflow_events` table shapes live
70
- // in `./postgres-core.ts` so the shared CRUD core can target them without
71
- // re-declaration. The full `StepflowDatabase` schema below adds the
72
- // adapter-only `stepflow_step_results` table (extended storage).
73
-
74
- /** Kysely row type for the stepflow_step_results table (extended storage). */
75
- interface StepflowStepResultsTable {
76
- id: string;
77
- /** Foreign key to runs.id. */
78
- run_id: string;
79
- /** Step name identifier. */
80
- step_name: string;
81
- /** Current step status (maps to ExtendedStepStatus). */
82
- status: string;
83
- /** JSON-serialized step output, null until step completes. */
84
- output_json: string | null;
85
- /** JSON-serialized error object, null unless step failed. */
86
- error_json: string | null;
87
- /** Current retry attempt number (1-based). */
88
- attempt: number;
89
- started_at: Date | null;
90
- completed_at: Date | null;
91
- }
92
-
93
- interface StepflowDatabase extends PostgresCoreDatabase {
94
- stepflow_step_results: StepflowStepResultsTable;
95
- }
96
-
97
- // ============================================================================
98
- // Configuration
99
- // ============================================================================
100
-
101
- /**
102
- * Configuration options for the PostgreSQL storage adapter.
103
- */
104
- export interface PostgresStorageConfig {
105
- /**
106
- * PostgreSQL connection string.
107
- * Example: "postgresql://user:pass@localhost:5432/dbname"
108
- */
109
- connectionString?: string;
110
-
111
- /**
112
- * Existing pg.Pool instance for connection sharing with application.
113
- * If provided, the adapter will not close this pool on close().
114
- */
115
- pool?: Pool;
116
-
117
- /**
118
- * Pool configuration options (if not providing pool or connectionString).
119
- */
120
- poolConfig?: PoolConfig;
121
-
122
- /**
123
- * Schema name for Stepflow tables.
124
- * @default 'public'
125
- */
126
- schema?: string;
127
-
128
- /**
129
- * Automatically create tables on initialize().
130
- * @default true
131
- */
132
- autoMigrate?: boolean;
133
-
134
- /**
135
- * Optional structured logger used to surface JSON corruption events.
136
- * Without one, parse failures still increment the corruption counter
137
- * exposed via `getJsonParseCorruptionCount()` but produce no log output.
138
- */
139
- logger?: Logger;
140
- }
141
-
142
- // ============================================================================
143
- // PostgreSQL Storage Adapter
144
- // ============================================================================
145
-
146
- /**
147
- * PostgreSQL implementation of StorageAdapter using Kysely.
148
- *
149
- * Features:
150
- * - Connection pooling (shared or dedicated)
151
- * - Automatic table creation
152
- * - Transaction support
153
- * - Atomic dequeue operations for distributed workers
154
- * - JSONB storage for flexible data
155
- *
156
- * @example
157
- * ```typescript
158
- * import { PostgresStorageAdapter } from 'stepflow/storage';
159
- *
160
- * const storage = new PostgresStorageAdapter({
161
- * connectionString: process.env.DATABASE_URL,
162
- * });
163
- *
164
- * await storage.initialize();
165
- *
166
- * const engine = new WorkflowEngine({ storage });
167
- * ```
168
- *
169
- * @example Sharing connection pool
170
- * ```typescript
171
- * import pg from 'pg';
172
- * import { PostgresStorageAdapter } from 'stepflow/storage';
173
- *
174
- * // Application's existing pool
175
- * const pool = new pg.Pool({
176
- * connectionString: process.env.DATABASE_URL,
177
- * max: 20,
178
- * });
179
- *
180
- * // Share with Stepflow
181
- * const storage = new PostgresStorageAdapter({ pool });
182
- * ```
183
- */
184
- export class PostgresStorageAdapter extends PostgresStorageCore {
185
- private db!: KyselyType<StepflowDatabase>;
186
- private pool!: Pool;
187
- private ownsPool = false;
188
- private autoMigrate: boolean;
189
- private initialized = false;
190
- private config: PostgresStorageConfig;
191
-
192
- constructor(config: PostgresStorageConfig) {
193
- const schema = config.schema ?? 'public';
194
- if (!/^[a-zA-Z_][a-zA-Z0-9_]{0,62}$/.test(schema)) {
195
- throw new Error(
196
- `Invalid schema name "${schema}". Schema must start with a letter or underscore, ` +
197
- `contain only alphanumeric characters and underscores, and be at most 63 characters.`
198
- );
199
- }
200
- super(schema, config.logger);
201
- this.autoMigrate = config.autoMigrate !== false;
202
- this.config = config;
203
- }
204
-
205
- /**
206
- * Core-tables query builder consumed by `PostgresStorageCore`. Returns the
207
- * schema-scoped Kysely view typed against the three core tables, which is
208
- * what the shared CRUD methods need.
209
- */
210
- protected override get qb(): CoreSchemaQueryBuilder {
211
- return this.db.withSchema(this.schema) as unknown as CoreSchemaQueryBuilder;
212
- }
213
-
214
- /**
215
- * Extended query builder for adapter-only tables (currently
216
- * `stepflow_step_results`). Kept separate from `qb` so the base class stays
217
- * typed against the core schema.
218
- */
219
- private get extQb(): SchemaQueryBuilder<StepflowDatabase> {
220
- return this.db.withSchema(this.schema);
221
- }
222
-
223
- /** Tag used by shared row mappers in `console.warn` messages. */
224
- protected override get warnLabel(): string {
225
- return 'PostgresStorageAdapter';
226
- }
227
-
228
- /** Override of `PostgresStorageCore.ensureReady()` — enforces `initialize()`. */
229
- protected override ensureReady(): void {
230
- this.ensureInitialized();
231
- }
232
-
233
- private ensureInitialized(): void {
234
- if (!this.initialized) {
235
- throw new Error(
236
- 'PostgresStorageAdapter is not initialized. Call initialize() before using the adapter.'
237
- );
238
- }
239
- }
240
-
241
- /**
242
- * Initialize the storage adapter.
243
- * Creates tables if autoMigrate is enabled.
244
- */
245
- async initialize(): Promise<void> {
246
- if (this.initialized) {
247
- return;
248
- }
249
-
250
- const deps = await loadPostgresDeps();
251
- Kysely = deps.Kysely;
252
- PostgresDialect = deps.PostgresDialect;
253
- sql = deps.sql;
254
- pgModule = deps.pgModule;
255
-
256
- if (this.config.pool) {
257
- this.pool = this.config.pool;
258
- this.ownsPool = false;
259
- } else if (this.config.connectionString) {
260
- this.pool = new pgModule.Pool({ connectionString: this.config.connectionString });
261
- this.ownsPool = true;
262
- } else if (this.config.poolConfig) {
263
- this.pool = new pgModule.Pool(this.config.poolConfig);
264
- this.ownsPool = true;
265
- } else {
266
- throw new Error(
267
- 'PostgresStorageConfig must include either pool, connectionString, or poolConfig'
268
- );
269
- }
270
-
271
- this.db = new Kysely({
272
- dialect: new PostgresDialect({
273
- pool: this.pool,
274
- }),
275
- });
276
-
277
- if (this.autoMigrate) {
278
- await this.createTables();
279
- }
280
-
281
- this.initialized = true;
282
- }
283
-
284
- /**
285
- * Close the database connection.
286
- * Only closes the pool if it was created by this adapter.
287
- */
288
- async close(): Promise<void> {
289
- await this.db.destroy();
290
- // Kysely's destroy() already calls pool.end() via PostgresDialect,
291
- // so we must NOT call pool.end() again.
292
- }
293
-
294
- // ============================================================================
295
- // Schema Creation
296
- // ============================================================================
297
-
298
- /**
299
- * Run a single migration step. Tolerates only known-benign Postgres errors
300
- * (duplicate object codes that map to "already applied"), and rethrows
301
- * everything else with a clear message so partial-state upgrades are not
302
- * silently swallowed.
303
- */
304
- protected async runMigration(label: string, fn: () => Promise<unknown>): Promise<void> {
305
- try {
306
- await fn();
307
- } catch (err) {
308
- if (PostgresStorageAdapter.isBenignMigrationError(err)) {
309
- return;
310
- }
311
- const message = err instanceof Error ? err.message : String(err);
312
- throw new Error(`Schema migration failed (${label}): ${message}`, { cause: err });
313
- }
314
- }
315
-
316
- /**
317
- * Postgres SQLSTATE codes that indicate a migration step has already been
318
- * applied (object already exists). These are safe to ignore on idempotent
319
- * re-runs; everything else must propagate.
320
- *
321
- * - 42701: duplicate_column
322
- * - 42P07: duplicate_table
323
- * - 42P06: duplicate_schema
324
- * - 42710: duplicate_object (e.g., index, constraint)
325
- */
326
- private static readonly BENIGN_MIGRATION_CODES = new Set([
327
- '42701',
328
- '42P07',
329
- '42P06',
330
- '42710',
331
- ]);
332
-
333
- private static isBenignMigrationError(err: unknown): boolean {
334
- if (!err || typeof err !== 'object') return false;
335
- const code = (err as { code?: unknown }).code;
336
- return typeof code === 'string' && PostgresStorageAdapter.BENIGN_MIGRATION_CODES.has(code);
337
- }
338
-
339
- /**
340
- * Create the workflow tables if they don't exist.
341
- */
342
- private async createTables(): Promise<void> {
343
- // Create schema if not public
344
- if (this.schema !== 'public') {
345
- await sql`CREATE SCHEMA IF NOT EXISTS ${sql.ref(this.schema)}`.execute(this.db);
346
- }
347
-
348
- // Create runs table with new columns
349
- await sql`
350
- CREATE TABLE IF NOT EXISTS ${sql.table(`${this.schema}.runs`)} (
351
- id TEXT PRIMARY KEY,
352
- kind TEXT NOT NULL,
353
- status TEXT NOT NULL,
354
- parent_run_id TEXT,
355
- input_json JSONB NOT NULL DEFAULT '{}',
356
- metadata_json JSONB NOT NULL DEFAULT '{}',
357
- context_json JSONB NOT NULL DEFAULT '{}',
358
- output_json JSONB,
359
- error_json JSONB,
360
- priority INTEGER NOT NULL DEFAULT 0,
361
- timeout_ms INTEGER,
362
- created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
363
- started_at TIMESTAMPTZ,
364
- finished_at TIMESTAMPTZ,
365
- CONSTRAINT runs_status_check CHECK (
366
- status IN ('pending', 'queued', 'running', 'succeeded', 'failed', 'canceled', 'timeout')
367
- )
368
- )
369
- `.execute(this.db);
370
-
371
- // Add new columns if they don't exist (for migrations)
372
- await this.runMigration(
373
- `add columns to ${this.schema}.runs`,
374
- () => sql`
375
- ALTER TABLE ${sql.table(`${this.schema}.runs`)}
376
- ADD COLUMN IF NOT EXISTS output_json JSONB,
377
- ADD COLUMN IF NOT EXISTS priority INTEGER NOT NULL DEFAULT 0,
378
- ADD COLUMN IF NOT EXISTS timeout_ms INTEGER
379
- `.execute(this.db)
380
- );
381
-
382
- // Create indexes for runs
383
- await sql`
384
- CREATE INDEX IF NOT EXISTS idx_runs_kind_status
385
- ON ${sql.table(`${this.schema}.runs`)} (kind, status)
386
- `.execute(this.db);
387
-
388
- await sql`
389
- CREATE INDEX IF NOT EXISTS idx_runs_parent
390
- ON ${sql.table(`${this.schema}.runs`)} (parent_run_id)
391
- `.execute(this.db);
392
-
393
- await sql`
394
- CREATE INDEX IF NOT EXISTS idx_runs_created
395
- ON ${sql.table(`${this.schema}.runs`)} (created_at DESC)
396
- `.execute(this.db);
397
-
398
- await sql`
399
- CREATE INDEX IF NOT EXISTS idx_runs_status
400
- ON ${sql.table(`${this.schema}.runs`)} (status)
401
- `.execute(this.db);
402
-
403
- await sql`
404
- CREATE INDEX IF NOT EXISTS idx_runs_priority
405
- ON ${sql.table(`${this.schema}.runs`)} (priority DESC, created_at ASC)
406
- `.execute(this.db);
407
-
408
- // Create workflow_run_steps table (legacy)
409
- await sql`
410
- CREATE TABLE IF NOT EXISTS ${sql.table(`${this.schema}.workflow_run_steps`)} (
411
- id TEXT PRIMARY KEY,
412
- run_id TEXT NOT NULL REFERENCES ${sql.table(`${this.schema}.runs`)} (id) ON DELETE CASCADE,
413
- step_key TEXT NOT NULL,
414
- step_name TEXT,
415
- status TEXT NOT NULL,
416
- attempt INTEGER NOT NULL DEFAULT 1,
417
- result_json JSONB,
418
- error_json JSONB,
419
- started_at TIMESTAMPTZ,
420
- finished_at TIMESTAMPTZ,
421
- CONSTRAINT workflow_run_steps_status_check CHECK (
422
- status IN ('pending', 'running', 'succeeded', 'failed', 'skipped', 'completed')
423
- )
424
- )
425
- `.execute(this.db);
426
-
427
- // Create indexes for workflow_run_steps
428
- await sql`
429
- CREATE INDEX IF NOT EXISTS idx_workflow_run_steps_run
430
- ON ${sql.table(`${this.schema}.workflow_run_steps`)} (run_id)
431
- `.execute(this.db);
432
-
433
- await sql`
434
- CREATE INDEX IF NOT EXISTS idx_workflow_run_steps_run_key
435
- ON ${sql.table(`${this.schema}.workflow_run_steps`)} (run_id, step_key)
436
- `.execute(this.db);
437
-
438
- // Create stepflow_step_results table (new)
439
- await sql`
440
- CREATE TABLE IF NOT EXISTS ${sql.table(`${this.schema}.stepflow_step_results`)} (
441
- id TEXT PRIMARY KEY DEFAULT gen_random_uuid()::text,
442
- run_id TEXT NOT NULL REFERENCES ${sql.table(`${this.schema}.runs`)} (id) ON DELETE CASCADE,
443
- step_name TEXT NOT NULL,
444
- status TEXT NOT NULL,
445
- output_json JSONB,
446
- error_json JSONB,
447
- attempt INTEGER NOT NULL DEFAULT 1,
448
- started_at TIMESTAMPTZ,
449
- completed_at TIMESTAMPTZ,
450
- CONSTRAINT stepflow_step_results_status_check CHECK (
451
- status IN ('pending', 'running', 'completed', 'failed', 'skipped')
452
- ),
453
- UNIQUE (run_id, step_name)
454
- )
455
- `.execute(this.db);
456
-
457
- // Create indexes for stepflow_step_results
458
- await sql`
459
- CREATE INDEX IF NOT EXISTS idx_stepflow_step_results_run
460
- ON ${sql.table(`${this.schema}.stepflow_step_results`)} (run_id)
461
- `.execute(this.db);
462
-
463
- await sql`
464
- CREATE INDEX IF NOT EXISTS idx_stepflow_step_results_run_name
465
- ON ${sql.table(`${this.schema}.stepflow_step_results`)} (run_id, step_name)
466
- `.execute(this.db);
467
-
468
- // Create workflow_events table
469
- await sql`
470
- CREATE TABLE IF NOT EXISTS ${sql.table(`${this.schema}.workflow_events`)} (
471
- id TEXT PRIMARY KEY,
472
- run_id TEXT NOT NULL REFERENCES ${sql.table(`${this.schema}.runs`)} (id) ON DELETE CASCADE,
473
- step_key TEXT,
474
- event_type TEXT NOT NULL,
475
- level TEXT NOT NULL,
476
- payload_json JSONB,
477
- timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW(),
478
- CONSTRAINT workflow_events_level_check CHECK (
479
- level IN ('info', 'warn', 'error')
480
- )
481
- )
482
- `.execute(this.db);
483
-
484
- // Create indexes for workflow_events
485
- await sql`
486
- CREATE INDEX IF NOT EXISTS idx_workflow_events_run
487
- ON ${sql.table(`${this.schema}.workflow_events`)} (run_id)
488
- `.execute(this.db);
489
-
490
- await sql`
491
- CREATE INDEX IF NOT EXISTS idx_workflow_events_run_ts
492
- ON ${sql.table(`${this.schema}.workflow_events`)} (run_id, timestamp)
493
- `.execute(this.db);
494
- }
495
-
496
- // ============================================================================
497
- // Run / Step / Event CRUD
498
- // ============================================================================
499
- //
500
- // The CRUD methods (`createRun`, `getRun`, `updateRun`, `listRuns`,
501
- // `createStep`, `getStep`, `updateStep`, `getStepsForRun`, `saveEvent`,
502
- // `getEventsForRun`) are inherited from `PostgresStorageCore`. Each call
503
- // routes through `this.qb` (overridden above) and `this.ensureReady()`
504
- // which calls `this.ensureInitialized()`.
505
-
506
- // ============================================================================
507
- // Transaction Support
508
- // ============================================================================
509
-
510
- /**
511
- * Execute a function within a database transaction.
512
- */
513
- async transaction<T>(fn: (tx: StorageAdapter) => Promise<T>): Promise<T> {
514
- this.ensureInitialized();
515
- return await this.db.transaction().execute(async (trx) => {
516
- // Create a transactional adapter wrapper
517
- const txAdapter = new PostgresTransactionAdapter(trx, this.schema, this.logger);
518
- return await fn(txAdapter);
519
- });
520
- }
521
-
522
- // ============================================================================
523
- // Cleanup Operations
524
- // ============================================================================
525
-
526
- /**
527
- * Delete runs older than the specified date.
528
- * Also deletes associated steps and events (via CASCADE).
529
- */
530
- async deleteOldRuns(olderThan: Date): Promise<number> {
531
- this.ensureInitialized();
532
- const result = await this.qb
533
- .deleteFrom('runs')
534
- .where('created_at', '<', olderThan)
535
- .executeTakeFirst();
536
-
537
- return Number(result.numDeletedRows);
538
- }
539
-
540
- // ============================================================================
541
- // Resume Support
542
- // ============================================================================
543
-
544
- /**
545
- * Get all runs that were interrupted (status is 'running' or 'queued').
546
- * These runs can potentially be resumed.
547
- */
548
- async getInterruptedRuns(): Promise<WorkflowRunRecord[]> {
549
- this.ensureInitialized();
550
- const rows = await this.qb
551
- .selectFrom('runs')
552
- .selectAll()
553
- .where('status', 'in', ['queued', 'running'])
554
- .orderBy('created_at', 'asc')
555
- .execute();
556
-
557
- return rows.map(row => mapRunRow(row as WorkflowRunsTable, this.mapperCtx));
558
- }
559
-
560
- /**
561
- * Get the last completed step for a run.
562
- * Useful for resuming from a checkpoint.
563
- */
564
- async getLastCompletedStep(runId: string): Promise<WorkflowRunStepRecord | null> {
565
- this.ensureInitialized();
566
- const row = await this.qb
567
- .selectFrom('workflow_run_steps')
568
- .selectAll()
569
- .where('run_id', '=', runId)
570
- .where('status', '=', 'succeeded')
571
- .orderBy('finished_at', 'desc')
572
- .limit(1)
573
- .executeTakeFirst();
574
-
575
- return row ? mapStepRow(row as WorkflowRunStepsTable, this.mapperCtx) : null;
576
- }
577
-
578
- // ============================================================================
579
- // Atomic Dequeue (for distributed workers)
580
- // ============================================================================
581
-
582
- /**
583
- * Atomically dequeue a run for processing.
584
- * Uses FOR UPDATE SKIP LOCKED for safe concurrent access.
585
- *
586
- * @param workflowKinds - Optional list of workflow kinds to filter by
587
- * @returns The dequeued run, or null if no runs are available
588
- */
589
- async dequeueRun(workflowKinds?: string[]): Promise<WorkflowRunRecord | null> {
590
- this.ensureInitialized();
591
- // Use raw SQL for FOR UPDATE SKIP LOCKED
592
- const kindFilter = workflowKinds && workflowKinds.length > 0
593
- ? sql`AND kind = ANY(${workflowKinds}::text[])`
594
- : sql``;
595
-
596
- const result = await sql<WorkflowRunsTable>`
597
- UPDATE ${sql.table(`${this.schema}.runs`)}
598
- SET status = 'running', started_at = NOW()
599
- WHERE id = (
600
- SELECT id FROM ${sql.table(`${this.schema}.runs`)}
601
- WHERE status = 'queued' ${kindFilter}
602
- ORDER BY priority DESC, created_at ASC
603
- LIMIT 1
604
- FOR UPDATE SKIP LOCKED
605
- )
606
- RETURNING *
607
- `.execute(this.db);
608
-
609
- const row = result.rows[0];
610
- return row ? mapRunRow(row, this.mapperCtx) : null;
611
- }
612
-
613
- // ============================================================================
614
- // Row Mapping (adapter-only tables)
615
- // ============================================================================
616
- //
617
- // Mappers and JSON-parse helpers for the three core tables live in
618
- // `./postgres-core.ts`. The single mapper kept here covers
619
- // `stepflow_step_results`, the only table outside the core schema.
620
-
621
- private mapStepResultRow(row: StepflowStepResultsTable): StepResult {
622
- const ctx = this.mapperCtx;
623
- return {
624
- id: row.id,
625
- runId: row.run_id,
626
- stepName: row.step_name,
627
- status: row.status as ExtendedStepStatus,
628
- output: safeParseOptionalField(row.output_json, {
629
- ...ctx,
630
- rowId: row.id,
631
- column: 'output_json',
632
- }) as Record<string, unknown> | undefined,
633
- error: safeParseOptionalField(row.error_json, {
634
- ...ctx,
635
- rowId: row.id,
636
- column: 'error_json',
637
- }) as Record<string, unknown> | undefined,
638
- attempt: row.attempt,
639
- startedAt: row.started_at ? new Date(row.started_at) : undefined,
640
- completedAt: row.completed_at ? new Date(row.completed_at) : undefined,
641
- };
642
- }
643
-
644
- // ============================================================================
645
- // New WorkflowStorage Methods
646
- // ============================================================================
647
-
648
- /**
649
- * Delete a workflow run by ID.
650
- * Also deletes associated steps and events (via CASCADE).
651
- */
652
- async deleteRun(id: string): Promise<void> {
653
- this.ensureInitialized();
654
- await this.qb
655
- .deleteFrom('runs')
656
- .where('id', '=', id)
657
- .execute();
658
- }
659
-
660
- /**
661
- * Cleanup stale runs that have exceeded their timeout.
662
- * Marks them as 'timeout' status with an appropriate error.
663
- *
664
- * @param defaultTimeoutMs - Default timeout in ms for runs without explicit timeout (default: 600000 = 10 minutes)
665
- * @returns Number of runs marked as timed out
666
- */
667
- async cleanupStaleRuns(defaultTimeoutMs: number = 600000): Promise<number> {
668
- this.ensureInitialized();
669
- const result = await sql<{ id: string }>`
670
- UPDATE ${sql.table(`${this.schema}.runs`)}
671
- SET
672
- status = 'timeout',
673
- error_json = jsonb_build_object(
674
- 'code', 'WORKFLOW_TIMEOUT',
675
- 'message', 'Workflow exceeded maximum execution time and was marked as timed out'
676
- ),
677
- finished_at = NOW()
678
- WHERE status = 'running'
679
- AND started_at IS NOT NULL
680
- AND (
681
- (timeout_ms IS NOT NULL AND started_at < NOW() - (timeout_ms || ' milliseconds')::interval)
682
- OR
683
- (timeout_ms IS NULL AND started_at < NOW() - (${defaultTimeoutMs} || ' milliseconds')::interval)
684
- )
685
- RETURNING id
686
- `.execute(this.db);
687
-
688
- return result.rows.length;
689
- }
690
-
691
- /**
692
- * Mark multiple runs as failed with a given reason.
693
- * Useful for cleanup when a worker shuts down unexpectedly.
694
- *
695
- * @param runIds - Array of run IDs to mark as failed
696
- * @param reason - Reason message for the failure
697
- */
698
- async markRunsAsFailed(runIds: string[], reason: string): Promise<void> {
699
- this.ensureInitialized();
700
- if (runIds.length === 0) return;
701
-
702
- await sql`
703
- UPDATE ${sql.table(`${this.schema}.runs`)}
704
- SET
705
- status = 'failed',
706
- error_json = jsonb_build_object(
707
- 'code', 'WORKER_SHUTDOWN',
708
- 'message', ${reason}
709
- ),
710
- finished_at = NOW()
711
- WHERE id = ANY(${runIds})
712
- AND status = 'running'
713
- `.execute(this.db);
714
- }
715
-
716
- /**
717
- * Get a specific step result by run ID and step name.
718
- */
719
- async getStepResult(runId: string, stepName: string): Promise<StepResult | undefined> {
720
- this.ensureInitialized();
721
- const row = await this.extQb
722
- .selectFrom('stepflow_step_results')
723
- .selectAll()
724
- .where('run_id', '=', runId)
725
- .where('step_name', '=', stepName)
726
- .executeTakeFirst();
727
-
728
- return row ? this.mapStepResultRow(row) : undefined;
729
- }
730
-
731
- /**
732
- * Get all step results for a run.
733
- */
734
- async getStepResults(runId: string): Promise<StepResult[]> {
735
- this.ensureInitialized();
736
- const rows = await this.extQb
737
- .selectFrom('stepflow_step_results')
738
- .selectAll()
739
- .where('run_id', '=', runId)
740
- .orderBy('started_at', 'asc')
741
- .execute();
742
-
743
- return rows.map(row => this.mapStepResultRow(row));
744
- }
745
-
746
- /**
747
- * Save or update a step result.
748
- * Uses upsert to handle both new and existing results.
749
- */
750
- async saveStepResult(result: Omit<StepResult, 'id'> & { id?: string }): Promise<void> {
751
- this.ensureInitialized();
752
- const id = result.id ?? generateId();
753
-
754
- await this.extQb
755
- .insertInto('stepflow_step_results')
756
- .values({
757
- id,
758
- run_id: result.runId,
759
- step_name: result.stepName,
760
- status: result.status,
761
- output_json: result.output ? JSON.stringify(result.output) : null,
762
- error_json: result.error ? JSON.stringify(stripStack(result.error)) : null,
763
- attempt: result.attempt,
764
- started_at: result.startedAt ?? null,
765
- completed_at: result.completedAt ?? null,
766
- })
767
- .onConflict(oc =>
768
- oc.columns(['run_id', 'step_name']).doUpdateSet({
769
- status: result.status,
770
- output_json: result.output ? JSON.stringify(result.output) : null,
771
- error_json: result.error ? JSON.stringify(stripStack(result.error)) : null,
772
- attempt: result.attempt,
773
- started_at: result.startedAt ?? null,
774
- completed_at: result.completedAt ?? null,
775
- })
776
- )
777
- .execute();
778
- }
779
-
780
- // ============================================================================
781
- // Utility Methods
782
- // ============================================================================
783
-
784
- /**
785
- * Get database statistics.
786
- */
787
- async getStats(): Promise<{ runs: number; steps: number; events: number }> {
788
- this.ensureInitialized();
789
- const [runsCount, stepsCount, eventsCount] = await Promise.all([
790
- this.qb.selectFrom('runs').select(sql`count(*)`.as('count')).executeTakeFirst(),
791
- this.qb.selectFrom('workflow_run_steps').select(sql`count(*)`.as('count')).executeTakeFirst(),
792
- this.qb.selectFrom('workflow_events').select(sql`count(*)`.as('count')).executeTakeFirst(),
793
- ]) as { count?: string | number }[];
794
-
795
- return {
796
- runs: Number(runsCount?.count ?? 0),
797
- steps: Number(stepsCount?.count ?? 0),
798
- events: Number(eventsCount?.count ?? 0),
799
- };
800
- }
801
- }
802
-
803
- // ============================================================================
804
- // Transaction Adapter (wraps Kysely transaction)
805
- // ============================================================================
806
-
807
- /**
808
- * A StorageAdapter wrapper for use within transactions.
809
- *
810
- * Used internally by `PostgresStorageAdapter.transaction()`. All CRUD methods
811
- * are inherited from `PostgresStorageCore`; this class only binds the
812
- * transaction-scoped query builder.
813
- *
814
- * Exported for regression testing; not part of the public API.
815
- */
816
- export class PostgresTransactionAdapter extends PostgresStorageCore {
817
- private readonly _qb: CoreSchemaQueryBuilder;
818
-
819
- constructor(trx: KyselyType<StepflowDatabase>, schema: string, logger?: Logger) {
820
- super(schema, logger);
821
- this._qb = trx.withSchema(schema) as unknown as CoreSchemaQueryBuilder;
822
- }
823
-
824
- protected override get qb(): CoreSchemaQueryBuilder {
825
- return this._qb;
826
- }
827
-
828
- protected override get warnLabel(): string {
829
- return 'PostgresTransactionAdapter';
830
- }
831
- }
832
-
833
- // Convenience aliases for shorter naming
834
- export { PostgresStorageAdapter as PostgresStorage };
835
- export type { PostgresStorageConfig as PostgresStorageOptions };