@multiplier-labs/stepflow 0.3.3 → 0.3.4

package/src/storage/sqlite.ts

@@ -2,6 +2,21 @@
  * SQLite storage adapter using better-sqlite3.
  *
  * Provides durable persistence for workflow runs, steps, and events.
+ *
+ * ## Module layout
+ *
+ *   - `SQLiteStorageAdapter` (this file) owns the better-sqlite3 connection,
+ *     the prepared-statement cache, and every method on `StorageAdapter`. It
+ *     is the only adapter class — unlike the Postgres adapter there is no
+ *     transaction-scoped twin, so no abstract CRUD base class is required.
+ *
+ *   - `sqlite-core.ts` holds the pure row-type interfaces and row mappers
+ *     (`mapRunRow`, `mapStepRow`, `mapEventRow`). The mappers route every
+ *     JSON-column parse through `safeParseField` / `safeParseOptionalField`
+ *     in `src/utils/safe-json.ts`, sharing the corruption-counter +
+ *     structured-warn behavior with the Postgres core (audit M2 / L3,
+ *     PR #52). The split mirrors `postgres-core.ts` so both adapters
+ *     expose mappers in the same shape.
  */
 
 import type Database from 'better-sqlite3';
@@ -15,8 +30,17 @@ import type {
   ListEventsOptions,
   PaginatedResult,
 } from './types';
-import type { RunStatus, StepStatus, WorkflowError } from '../core/types';
+import type { Logger } from '../core/types';
 import { sanitizeErrorForStorage } from '../utils/logger';
+import type { MapperContext } from '../utils/safe-json';
+import {
+  mapRunRow,
+  mapStepRow,
+  mapEventRow,
+  type SQLiteRunRow,
+  type SQLiteStepRow,
+  type SQLiteEventRow,
+} from './sqlite-core';
 
 // ============================================================================
 // Schema SQL
@@ -82,6 +106,22 @@ CREATE INDEX IF NOT EXISTS idx_workflow_events_run ON workflow_events(run_id);
 CREATE INDEX IF NOT EXISTS idx_workflow_events_run_ts ON workflow_events(run_id, timestamp);
 `;
 
+/**
+ * Columns added to {@link CREATE_TABLES_SQL} after the initial schema, expressed
+ * as `ALTER TABLE ADD COLUMN` migrations. SQLite's `CREATE TABLE IF NOT EXISTS`
+ * does not reconcile column lists against existing tables, so a deploy that ran
+ * an older version of this adapter will be missing any column added later. The
+ * adapter applies these on startup so existing databases stay usable across
+ * upgrades.
+ *
+ * Append-only: SQLite does not support DROP COLUMN or general ALTER COLUMN, so
+ * removing or changing an entry here would break upgrades from older versions.
+ * Only add new entries when a column is added to {@link CREATE_TABLES_SQL}.
+ */
+const SCHEMA_MIGRATIONS: ReadonlyArray<{ table: string; column: string; type: string }> = [
+  { table: 'workflow_runs', column: 'completed_steps_json', type: 'TEXT' },
+];
+
 // ============================================================================
 // Configuration
 // ============================================================================
@@ -107,6 +147,61 @@ export interface SQLiteStorageConfig {
    * Table names are always prefixed with 'workflow_'.
    */
   tablePrefix?: string;
+
+  /**
+   * Optional structured logger used to surface JSON corruption events.
+   * Without one, parse failures still increment the corruption counter
+   * exposed via `getJsonParseCorruptionCount()` but produce no log output.
+   */
+  logger?: Logger;
+}
+
+// ============================================================================
+// Helpers
+// ============================================================================
+
+/**
+ * better-sqlite3 raises `SqliteError { code: 'SQLITE_ERROR', message:
+ * 'duplicate column name: <name>' }` when an `ALTER TABLE ADD COLUMN` targets
+ * a column that already exists. Treat that as a benign no-op so concurrent
+ * startups (or any race between the {@link runMigrations} existence check and
+ * the ALTER) don't crash. All other errors propagate.
+ */
+function isDuplicateColumnError(err: unknown): boolean {
+  if (!err || typeof err !== 'object') return false;
+  const code = (err as { code?: unknown }).code;
+  const message = (err as { message?: unknown }).message;
+  return (
+    code === 'SQLITE_ERROR' &&
+    typeof message === 'string' &&
+    message.toLowerCase().includes('duplicate column name')
+  );
+}
+
+/**
+ * Resolve to `true` if `p` settles within a few microtask ticks of being
+ * inspected, otherwise `false`. Used by the deprecated async `transaction()`
+ * to detect callbacks that suspend on real async I/O — by the time we ask,
+ * the synchronous COMMIT has already run, so a still-pending promise means
+ * any post-await operations land outside the transaction.
+ */
+async function isPromiseSettled(p: Promise<unknown>): Promise<boolean> {
+  let settled = false;
+  p.then(
+    () => {
+      settled = true;
+    },
+    () => {
+      settled = true;
+    }
+  );
+  // Drain a handful of microtask ticks. `await Promise.resolve()` callbacks
+  // and other already-resolved continuations finish in 1-2 ticks; anything
+  // still pending after this is doing real async I/O (timers, network, etc.).
+  for (let i = 0; i < 5 && !settled; i++) {
+    await Promise.resolve();
+  }
+  return settled;
 }
 
 // ============================================================================
@@ -136,6 +231,7 @@ export interface SQLiteStorageConfig {
 export class SQLiteStorageAdapter implements StorageAdapter {
   private db: Database.Database;
   private prefix: string;
+  private logger: Logger | undefined;
 
   // Prepared statements (cached for performance)
   private stmts: {
@@ -160,6 +256,7 @@ export class SQLiteStorageAdapter implements StorageAdapter {
   constructor(config: SQLiteStorageConfig) {
     this.db = config.db;
     this.prefix = config.tablePrefix ?? 'workflow';
+    this.logger = config.logger;
 
     // Enable foreign keys
     this.db.pragma('foreign_keys = ON');
@@ -169,6 +266,14 @@ export class SQLiteStorageAdapter implements StorageAdapter {
       this.createTables();
     }
 
+    // Always reconcile column-level migrations: the prepared statements below
+    // reference the latest schema, so an adapter sitting on top of an older
+    // table layout (e.g., a database created by a previous version, or a
+    // caller-managed schema with `autoCreateTables: false` that has not been
+    // hand-upgraded) would crash on the first INSERT/SELECT. Migrations are
+    // idempotent and skip tables that don't exist.
+    this.runMigrations();
+
     // Prepare statements
     this.prepareStatements();
   }
@@ -180,6 +285,39 @@ export class SQLiteStorageAdapter implements StorageAdapter {
     this.db.exec(CREATE_TABLES_SQL);
   }
 
+  /**
+   * Apply additive schema migrations to existing tables.
+   *
+   * `CREATE TABLE IF NOT EXISTS` is a no-op when the table already exists, so
+   * any column added to {@link CREATE_TABLES_SQL} after the initial schema is
+   * silently absent on databases created by older versions of this adapter.
+   * Without this step, the first INSERT/SELECT crashes with
+   * `SqliteError: table X has no column named Y`. This mirrors the
+   * `ALTER TABLE ADD COLUMN IF NOT EXISTS` block in `postgres.ts`'s
+   * `createTables()`.
+   *
+   * For each entry in {@link SCHEMA_MIGRATIONS}: if the table exists and the
+   * column does not, run `ALTER TABLE ADD COLUMN`. The ALTER is wrapped in a
+   * try/catch that tolerates `duplicate column` errors so concurrent startups
+   * (or any race between the existence check and the ALTER) do not crash.
+   * Other SQLite errors propagate.
+   */
+  private runMigrations(): void {
+    for (const { table, column, type } of SCHEMA_MIGRATIONS) {
+      const tableInfo = this.db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>;
+      // PRAGMA table_info on a missing table returns []; CREATE TABLE handles that case.
+      if (tableInfo.length === 0) continue;
+      if (tableInfo.some((col) => col.name === column)) continue;
+
+      try {
+        this.db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${type}`);
+      } catch (err) {
+        if (isDuplicateColumnError(err)) continue;
+        throw err;
+      }
+    }
+  }
+
   /**
    * Prepare all SQL statements for better performance.
    */
@@ -307,7 +445,7 @@ export class SQLiteStorageAdapter implements StorageAdapter {
 
   async getRun(runId: string): Promise<WorkflowRunRecord | null> {
     const row = this.stmts!.getRun.get(runId) as SQLiteRunRow | undefined;
-    return row ? this.mapRunRow(row) : null;
+    return row ? mapRunRow(row, this.mapperCtx) : null;
   }
 
   async updateRun(runId: string, updates: Partial<WorkflowRunRecord>): Promise<void> {
@@ -344,7 +482,7 @@ export class SQLiteStorageAdapter implements StorageAdapter {
     ) as { count: number };
 
     return {
-      items: rows.map(row => this.mapRunRow(row)),
+      items: rows.map(row => mapRunRow(row, this.mapperCtx)),
       total: countResult.count,
       limit,
       offset,
@@ -376,7 +514,7 @@ export class SQLiteStorageAdapter implements StorageAdapter {
 
   async getStep(stepId: string): Promise<WorkflowRunStepRecord | null> {
     const row = this.stmts!.getStep.get(stepId) as SQLiteStepRow | undefined;
-    return row ? this.mapStepRow(row) : null;
+    return row ? mapStepRow(row, this.mapperCtx) : null;
   }
 
   async updateStep(stepId: string, updates: Partial<WorkflowRunStepRecord>): Promise<void> {
@@ -392,7 +530,7 @@ export class SQLiteStorageAdapter implements StorageAdapter {
 
   async getStepsForRun(runId: string): Promise<WorkflowRunStepRecord[]> {
     const rows = this.stmts!.getStepsForRun.all(runId) as SQLiteStepRow[];
-    return rows.map(row => this.mapStepRow(row));
+    return rows.map(row => mapStepRow(row, this.mapperCtx));
   }
 
   // ============================================================================
@@ -426,7 +564,7 @@ export class SQLiteStorageAdapter implements StorageAdapter {
       limit, offset
     ) as SQLiteEventRow[];
 
-    return rows.map(row => this.mapEventRow(row));
+    return rows.map(row => mapEventRow(row, this.mapperCtx));
   }
 
   // ============================================================================
@@ -436,16 +574,36 @@ export class SQLiteStorageAdapter implements StorageAdapter {
   /**
    * Execute a function within a database transaction (async interface).
    *
-   * @deprecated Use `transactionSync()` instead. This method only works when
-   * the callback performs purely synchronous operations wrapped in async/await.
-   * If the callback awaits real async I/O (network, timers, etc.), it will
-   * throw an error. `transactionSync()` makes the synchronous requirement explicit.
+   * @deprecated Use {@link transactionSync} instead. better-sqlite3 transactions
+   * are synchronous: COMMIT happens the moment the callback returns control to
+   * `db.transaction(fn)()`. If the callback awaits real async I/O (network,
+   * timers, etc.), the COMMIT has already happened by the time the awaited work
+   * runs, so its operations are **not** transactional.
+   *
+   * To make the footgun visible the adapter installs a runtime guard: when the
+   * callback's promise has not settled by the time the COMMIT returns, the
+   * deprecated method emits a `console.warn` so that the misuse surfaces in
+   * production logs. The promise is still returned so existing callers keep
+   * working — switch to `transactionSync` to silence the warning.
    */
   async transaction<T>(fn: (tx: StorageAdapter) => Promise<T>): Promise<T> {
     let fnPromise!: Promise<T>;
     this.transactionSync(() => {
       fnPromise = fn(this);
     });
+
+    // COMMIT has already executed at this point. If the callback's promise is
+    // still pending, the callback is doing async work outside the transaction
+    // boundary — log a warning so the misuse is visible in production logs.
+    if (!(await isPromiseSettled(fnPromise))) {
+      console.warn(
+        '[SQLiteStorageAdapter] transaction() callback did not settle synchronously. ' +
+        'better-sqlite3 transactions are synchronous and the COMMIT has already executed; ' +
+        'any async work inside the callback is NOT transactional. ' +
+        'Use transactionSync() with purely synchronous operations instead.'
+      );
+    }
+
     return fnPromise;
   }
 
@@ -497,7 +655,7 @@ export class SQLiteStorageAdapter implements StorageAdapter {
    */
   async getInterruptedRuns(): Promise<WorkflowRunRecord[]> {
     const rows = this.stmts!.getInterruptedRuns.all() as SQLiteRunRow[];
-    return rows.map(row => this.mapRunRow(row));
+    return rows.map(row => mapRunRow(row, this.mapperCtx));
   }
 
   /**
@@ -506,55 +664,22 @@ export class SQLiteStorageAdapter implements StorageAdapter {
    */
   async getLastCompletedStep(runId: string): Promise<WorkflowRunStepRecord | null> {
     const row = this.stmts!.getLastCompletedStep.get(runId) as SQLiteStepRow | undefined;
-    return row ? this.mapStepRow(row) : null;
+    return row ? mapStepRow(row, this.mapperCtx) : null;
   }
 
   // ============================================================================
   // Row Mapping
   // ============================================================================
-
-  private mapRunRow(row: SQLiteRunRow): WorkflowRunRecord {
-    return {
-      id: row.id,
-      kind: row.kind,
-      status: row.status as RunStatus,
-      parentRunId: row.parent_run_id ?? undefined,
-      input: JSON.parse(row.input_json),
-      metadata: JSON.parse(row.metadata_json),
-      context: JSON.parse(row.context_json),
-      completedSteps: row.completed_steps_json ? JSON.parse(row.completed_steps_json) : undefined,
-      error: row.error_json ? JSON.parse(row.error_json) : undefined,
-      createdAt: new Date(row.created_at),
-      startedAt: row.started_at ? new Date(row.started_at) : undefined,
-      finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,
-    };
-  }
-
-  private mapStepRow(row: SQLiteStepRow): WorkflowRunStepRecord {
-    return {
-      id: row.id,
-      runId: row.run_id,
-      stepKey: row.step_key,
-      stepName: row.step_name,
-      status: row.status as StepStatus,
-      attempt: row.attempt,
-      result: row.result_json ? JSON.parse(row.result_json) : undefined,
-      error: row.error_json ? JSON.parse(row.error_json) : undefined,
-      startedAt: row.started_at ? new Date(row.started_at) : undefined,
-      finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,
-    };
-  }
-
-  private mapEventRow(row: SQLiteEventRow): WorkflowEventRecord {
-    return {
-      id: row.id,
-      runId: row.run_id,
-      stepKey: row.step_key ?? undefined,
-      eventType: row.event_type,
-      level: row.level as 'info' | 'warn' | 'error',
-      payload: row.payload_json ? JSON.parse(row.payload_json) : undefined,
-      timestamp: new Date(row.timestamp),
-    };
+  //
+  // The pure row mappers (`mapRunRow`, `mapStepRow`, `mapEventRow`) live in
+  // `./sqlite-core.ts`. Each call site here builds a `MapperContext` once via
+  // `this.mapperCtx` and hands it to the mapper, which extends it with
+  // `rowId` + `column` per JSON-parse so corruption logs identify the bad row
+  // (audit M2 / L3 — PR #52).
+
+  /** Mapper context paired with the adapter's component label + optional logger. */
+  private get mapperCtx(): MapperContext {
+    return { component: 'SQLiteStorageAdapter', logger: this.logger };
   }
 
   // ============================================================================
@@ -587,41 +712,6 @@ export class SQLiteStorageAdapter implements StorageAdapter {
 // ============================================================================
 // SQLite Row Types
 // ============================================================================
-
-interface SQLiteRunRow {
-  id: string;
-  kind: string;
-  status: string;
-  parent_run_id: string | null;
-  input_json: string;
-  metadata_json: string;
-  context_json: string;
-  completed_steps_json: string | null;
-  error_json: string | null;
-  created_at: string;
-  started_at: string | null;
-  finished_at: string | null;
-}
-
-interface SQLiteStepRow {
-  id: string;
-  run_id: string;
-  step_key: string;
-  step_name: string;
-  status: string;
-  attempt: number;
-  result_json: string | null;
-  error_json: string | null;
-  started_at: string | null;
-  finished_at: string | null;
-}
-
-interface SQLiteEventRow {
-  id: string;
-  run_id: string;
-  step_key: string | null;
-  event_type: string;
-  level: string;
-  payload_json: string | null;
-  timestamp: string;
-}
+//
+// `SQLiteRunRow`, `SQLiteStepRow`, and `SQLiteEventRow` now live in
+// `./sqlite-core.ts` alongside their pure row mappers.

package/src/utils/index.ts

@@ -6,3 +6,5 @@ export * from './id';
 export * from './errors';
 export * from './logger';
 export * from './retry';
+export * from './safe-json';
+export * from './webhook-signature';

package/src/utils/pinned-fetch.test.ts

@@ -0,0 +1,84 @@
+/**
+ * pinnedFetch is the linchpin of the M1 (DNS rebinding TOCTOU) fix: by
+ * passing a `lookup` function that returns a caller-supplied IP, the TCP
+ * connection cannot be re-resolved to a different address. We test it
+ * against a real loopback HTTP server because ESM forbids spying on the
+ * `http` module's exports and we want to exercise the real request path
+ * end-to-end.
+ */
+
+import { describe, it, expect, beforeAll, afterAll } from 'vitest';
+import * as http from 'node:http';
+import { pinnedFetch } from './pinned-fetch';
+
+describe('pinnedFetch', () => {
+  let server: http.Server;
+  let port: number;
+  let lastRequest: { hostHeader?: string; method?: string; path?: string; body: string } = {
+    body: '',
+  };
+
+  beforeAll(async () => {
+    server = http.createServer((req, res) => {
+      const chunks: Buffer[] = [];
+      req.on('data', (c: Buffer) => chunks.push(c));
+      req.on('end', () => {
+        lastRequest = {
+          hostHeader: req.headers.host,
+          method: req.method,
+          path: req.url,
+          body: Buffer.concat(chunks).toString('utf8'),
+        };
+        res.writeHead(204, 'No Content');
+        res.end();
+      });
+    });
+    await new Promise<void>((resolve) => {
+      server.listen(0, '127.0.0.1', () => resolve());
+    });
+    port = (server.address() as { port: number }).port;
+  });
+
+  afterAll(async () => {
+    await new Promise<void>((resolve) => server.close(() => resolve()));
+  });
+
+  it('connects to the pinned IP even when the URL hostname is unrelated', async () => {
+    // The URL hostname is one that would never resolve to 127.0.0.1 via DNS.
+    // pinnedFetch must ignore the URL's would-be DNS answer and connect to
+    // the pinned address. If pinning is broken, the connection will fail
+    // or hit a different host.
+    const response = await pinnedFetch(
+      `http://rebind.invalid:${port}/hook`,
+      { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: '{"ok":1}' },
+      '127.0.0.1',
+      4,
+    );
+
+    expect(response.ok).toBe(true);
+    expect(response.status).toBe(204);
+    expect(lastRequest.method).toBe('POST');
+    expect(lastRequest.path).toBe('/hook');
+    expect(lastRequest.body).toBe('{"ok":1}');
+  });
+
+  it('preserves the original hostname in the Host header (vhost / TLS SNI)', async () => {
+    await pinnedFetch(
+      `http://example.test:${port}/`,
+      { method: 'GET' },
+      '127.0.0.1',
+      4,
+    );
+
+    // The Host header must reflect the URL hostname, not the pinned IP, so
+    // that virtual-host routing and TLS certificate validation work.
+    expect(lastRequest.hostHeader).toContain('example.test');
+  });
+
+  it('rejects when the connection cannot be established', async () => {
+    // Pin to a non-listening port on loopback.
+    await expect(
+      pinnedFetch(`http://example.test:1/`, { method: 'GET' }, '127.0.0.1', 4),
+    ).rejects.toThrow();
+  });
+});

package/src/utils/pinned-fetch.ts

@@ -0,0 +1,118 @@
+/**
+ * DNS-pinned fetch helper.
+ *
+ * Performs an HTTP(S) request that connects to a caller-supplied IP address
+ * instead of re-resolving the URL hostname. Used by the webhook transport to
+ * close the SSRF DNS-rebinding TOCTOU window: the hostname is resolved
+ * exactly once (by the caller), the resolved IP is validated, and the
+ * resolved IP is then pinned for the connection so the request cannot be
+ * rerouted by a hostile DNS server.
+ *
+ * The original hostname is preserved for TLS SNI / certificate verification
+ * and for the HTTP `Host` header.
+ *
+ * Audit: 2026-04-27 finding M1 — DNS TOCTOU between SSRF validation and
+ * connect.
+ */
+
+import * as http from 'node:http';
+import * as https from 'node:https';
+import type { LookupAddress, LookupOptions } from 'node:dns';
+
+/** Minimal fetch-compatible Response surface used by the webhook transport. */
+export interface PinnedFetchResponse {
+  ok: boolean;
+  status: number;
+  statusText: string;
+}
+
+/** Subset of `RequestInit` accepted by `pinnedFetch`. */
+export interface PinnedFetchInit {
+  method?: string;
+  headers?: Record<string, string>;
+  body?: string;
+  signal?: AbortSignal;
+}
+
+/**
+ * Make an HTTP(S) request that connects to `address` instead of resolving
+ * `url`'s hostname. Returns a minimal `Response`-shaped object compatible
+ * with the webhook transport's status / ok checks.
+ *
+ * @throws Error on transport failure or abort
+ */
+export function pinnedFetch(
+  url: string,
+  init: PinnedFetchInit,
+  address: string,
+  family: 4 | 6 = 4,
+): Promise<PinnedFetchResponse> {
+  const parsed = new URL(url);
+  const isHttps = parsed.protocol === 'https:';
+  const lib = isHttps ? https : http;
+  const port = parsed.port ? Number(parsed.port) : isHttps ? 443 : 80;
+
+  // `lookup` is called by the agent's socket-create path. By returning the
+  // pre-validated address here we guarantee the TCP connection target is the
+  // IP whose safety the caller already verified. Node may invoke this with
+  // `{ all: true }` (expects an array result) or single-address mode
+  // (expects positional address/family) — handle both for cross-version safety.
+  const lookup = (
+    _hostname: string,
+    options: LookupOptions,
+    cb: (err: NodeJS.ErrnoException | null, addr: string | LookupAddress[], fam?: number) => void,
+  ): void => {
+    if (options && options.all === true) {
+      cb(null, [{ address, family }]);
+    } else {
+      cb(null, address, family);
+    }
+  };
+
+  return new Promise<PinnedFetchResponse>((resolve, reject) => {
+    const req = lib.request(
+      {
+        hostname: parsed.hostname,
+        port,
+        path: `${parsed.pathname || '/'}${parsed.search}`,
+        method: init.method ?? 'GET',
+        headers: init.headers,
+        lookup,
+        servername: parsed.hostname,
+      },
+      (res) => {
+        // Drain the body — we only inspect status, but leaving the socket
+        // half-read would leak it.
+        res.resume();
+        res.on('end', () => {
+          const status = res.statusCode ?? 0;
+          resolve({
+            ok: status >= 200 && status < 300,
+            status,
+            statusText: res.statusMessage ?? '',
+          });
+        });
+        res.on('error', reject);
+      },
+    );
+
+    req.on('error', reject);
+
+    if (init.signal) {
+      if (init.signal.aborted) {
+        req.destroy(new Error('Request aborted'));
+      } else {
+        init.signal.addEventListener(
+          'abort',
+          () => req.destroy(new Error('Request aborted')),
+          { once: true },
+        );
+      }
+    }
+
+    if (init.body !== undefined) {
+      req.write(init.body);
+    }
+    req.end();
+  });
+}

package/src/utils/postgres-deps.ts

@@ -61,3 +61,19 @@ export async function loadPostgresDeps(): Promise<PostgresDeps> {
 
   return loadingPromise;
 }
+
+/**
+ * Synchronously access the cached PostgreSQL deps. Throws if
+ * `loadPostgresDeps()` has not yet completed. Used by code paths that run
+ * after `PostgresStorageAdapter.initialize()` and need direct access to
+ * `sql` / `Kysely` without re-awaiting.
+ */
+export function requirePostgresDeps(): PostgresDeps {
+  if (!cached) {
+    throw new Error(
+      'PostgreSQL dependencies have not been loaded yet. ' +
+      'Call loadPostgresDeps() (or PostgresStorageAdapter.initialize()) first.'
+    );
+  }
+  return cached;
+}