@multiplier-labs/stepflow 0.3.3 → 0.3.4

package/src/storage/sqlite-core.ts

@@ -0,0 +1,151 @@
+/**
+ * Shared core for the SQLite storage adapter.
+ *
+ * Mirrors the structure introduced for Postgres in `postgres-core.ts` (PR #50)
+ * so all storage adapters expose row mappers and types in the same shape.
+ *
+ * The SQLite case is simpler than Postgres: there is only one adapter class
+ * (`SQLiteStorageAdapter`), with no transaction-scoped twin. So this module
+ * carries only the *pure* portion of the postgres-core split:
+ *
+ *   - SQLite row type interfaces for the three core tables
+ *     (`workflow_runs`, `workflow_run_steps`, `workflow_events`).
+ *   - Pure row mappers (`mapRunRow`, `mapStepRow`, `mapEventRow`) that take
+ *     a `MapperContext = { component, logger }` and route every JSON-column
+ *     parse through the shared `safeParseField` / `safeParseOptionalField`
+ *     helpers. Corruption is reported via the configured logger with safe
+ *     metadata only — the raw value is never logged (audit M2 / L3, PR #52).
+ *
+ * No abstract base class is needed: the adapter class binds the prepared
+ * statements + better-sqlite3 connection in one place and would gain nothing
+ * from inheritance. The mappers and row types are extracted purely so they
+ * are reusable, testable, and consistent with `postgres-core.ts`.
+ */
+
+import {
+  safeParseField as sharedSafeParseField,
+  safeParseOptionalField as sharedSafeParseOptionalField,
+  type MapperContext,
+  type SafeJsonParseContext,
+} from '../utils/safe-json';
+import type {
+  WorkflowRunRecord,
+  WorkflowRunStepRecord,
+  WorkflowEventRecord,
+} from './types';
+import type { RunStatus, StepStatus, WorkflowError } from '../core/types';
+
+/**
+ * Re-export the shared `MapperContext` so callers can `import { MapperContext }`
+ * from either storage core module.
+ */
+export type { MapperContext };
+
+// ============================================================================
+// SQLite row types
+// ============================================================================
+//
+// SQLite stores everything as text/integer/real, so timestamps come back as
+// ISO-8601 strings (the adapter writes them via `Date.prototype.toISOString`)
+// and JSON columns come back as the original `JSON.stringify` output. The
+// mappers below convert each row back to the public record shape.
+
+/** Row shape for the `workflow_runs` table. */
+export interface SQLiteRunRow {
+  id: string;
+  kind: string;
+  status: string;
+  parent_run_id: string | null;
+  input_json: string;
+  metadata_json: string;
+  context_json: string;
+  completed_steps_json: string | null;
+  error_json: string | null;
+  created_at: string;
+  started_at: string | null;
+  finished_at: string | null;
+}
+
+/** Row shape for the `workflow_run_steps` table. */
+export interface SQLiteStepRow {
+  id: string;
+  run_id: string;
+  step_key: string;
+  step_name: string;
+  status: string;
+  attempt: number;
+  result_json: string | null;
+  error_json: string | null;
+  started_at: string | null;
+  finished_at: string | null;
+}
+
+/** Row shape for the `workflow_events` table. */
+export interface SQLiteEventRow {
+  id: string;
+  run_id: string;
+  step_key: string | null;
+  event_type: string;
+  level: string;
+  payload_json: string | null;
+  timestamp: string;
+}
+
+// ============================================================================
+// Row mappers (pure)
+// ============================================================================
+//
+// Adapters supply a `MapperContext` ({ component, logger }); each helper
+// extends it with the row id and column being parsed so a structured log
+// emitted on corruption identifies the bad row. The raw value is never
+// logged — see `src/utils/safe-json.ts`.
+
+/** Map a `workflow_runs` row to the public `WorkflowRunRecord` shape. */
+export function mapRunRow(row: SQLiteRunRow, ctx: MapperContext): WorkflowRunRecord {
+  const fieldCtx = (column: string): SafeJsonParseContext => ({ ...ctx, rowId: row.id, column });
+  return {
+    id: row.id,
+    kind: row.kind,
+    status: row.status as RunStatus,
+    parentRunId: row.parent_run_id ?? undefined,
+    input: sharedSafeParseField(row.input_json, {}, fieldCtx('input_json')) as Record<string, unknown>,
+    metadata: sharedSafeParseField(row.metadata_json, {}, fieldCtx('metadata_json')) as Record<string, unknown>,
+    context: sharedSafeParseField(row.context_json, {}, fieldCtx('context_json')) as Record<string, unknown>,
+    completedSteps: sharedSafeParseOptionalField(row.completed_steps_json, fieldCtx('completed_steps_json')) as string[] | undefined,
+    error: sharedSafeParseOptionalField(row.error_json, fieldCtx('error_json')) as WorkflowError | undefined,
+    createdAt: new Date(row.created_at),
+    startedAt: row.started_at ? new Date(row.started_at) : undefined,
+    finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,
+  };
+}
+
+/** Map a `workflow_run_steps` row to the public `WorkflowRunStepRecord` shape. */
+export function mapStepRow(row: SQLiteStepRow, ctx: MapperContext): WorkflowRunStepRecord {
+  const fieldCtx = (column: string): SafeJsonParseContext => ({ ...ctx, rowId: row.id, column });
+  return {
+    id: row.id,
+    runId: row.run_id,
+    stepKey: row.step_key,
+    stepName: row.step_name,
+    status: row.status as StepStatus,
+    attempt: row.attempt,
+    result: sharedSafeParseOptionalField(row.result_json, fieldCtx('result_json')),
+    error: sharedSafeParseOptionalField(row.error_json, fieldCtx('error_json')) as WorkflowError | undefined,
+    startedAt: row.started_at ? new Date(row.started_at) : undefined,
+    finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,
+  };
+}
+
+/** Map a `workflow_events` row to the public `WorkflowEventRecord` shape. */
+export function mapEventRow(row: SQLiteEventRow, ctx: MapperContext): WorkflowEventRecord {
+  const fieldCtx: SafeJsonParseContext = { ...ctx, rowId: row.id, column: 'payload_json' };
+  return {
+    id: row.id,
+    runId: row.run_id,
+    stepKey: row.step_key ?? undefined,
+    eventType: row.event_type,
+    level: row.level as 'info' | 'warn' | 'error',
+    payload: sharedSafeParseOptionalField(row.payload_json, fieldCtx),
+    timestamp: new Date(row.timestamp),
+  };
+}

package/src/storage/sqlite.test.ts

@@ -1,4 +1,4 @@
-import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
 import Database from 'better-sqlite3';
 import { SQLiteStorageAdapter } from './sqlite';
 import type { WorkflowRunRecord, WorkflowRunStepRecord } from './types';
@@ -450,6 +450,38 @@ describe('SQLiteStorageAdapter', () => {
       });
       expect(result).toBe('hello');
     });
+
+    it('warns when the callback awaits real async I/O (post-COMMIT work is not transactional)', async () => {
+      const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
+      try {
+        const result = await storage.transaction(async () => {
+          // setTimeout is a macrotask — by the time it fires, COMMIT is long done.
+          await new Promise((resolve) => setTimeout(resolve, 5));
+          return 'late';
+        });
+        expect(result).toBe('late');
+        expect(warnSpy).toHaveBeenCalled();
+        const message = warnSpy.mock.calls[0]?.[0] ?? '';
+        expect(message).toContain('did not settle synchronously');
+        expect(message).toContain('transactionSync');
+      } finally {
+        warnSpy.mockRestore();
+      }
+    });
+
+    it('does not warn when the callback completes synchronously', async () => {
+      const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
+      try {
+        await storage.transaction(async () => 1);
+        await storage.transaction(async () => {
+          await Promise.resolve();
+          return 2;
+        });
+        expect(warnSpy).not.toHaveBeenCalled();
+      } finally {
+        warnSpy.mockRestore();
+      }
+    });
   });
 
   describe('getStats', () => {
@@ -463,4 +495,148 @@ describe('SQLiteStorageAdapter', () => {
       expect(stats.events).toBe(0);
     });
   });
+
+  describe('schema migrations', () => {
+    /**
+     * The pre-`completed_steps_json` schema. Reproduces the state of a database
+     * created by an older version of the adapter, before the column was added.
+     * Includes the unchanged sibling tables so prepared statements that touch
+     * `workflow_run_steps` / `workflow_events` find them on construction.
+     */
+    const LEGACY_SCHEMA_SQL = `
+      CREATE TABLE workflow_runs (
+        id TEXT PRIMARY KEY,
+        kind TEXT NOT NULL,
+        status TEXT NOT NULL,
+        parent_run_id TEXT,
+        input_json TEXT NOT NULL,
+        metadata_json TEXT NOT NULL,
+        context_json TEXT NOT NULL,
+        error_json TEXT,
+        created_at TEXT NOT NULL,
+        started_at TEXT,
+        finished_at TEXT
+      );
+      CREATE TABLE workflow_run_steps (
+        id TEXT PRIMARY KEY,
+        run_id TEXT NOT NULL,
+        step_key TEXT NOT NULL,
+        step_name TEXT,
+        status TEXT NOT NULL,
+        attempt INTEGER NOT NULL DEFAULT 1,
+        result_json TEXT,
+        error_json TEXT,
+        started_at TEXT,
+        finished_at TEXT,
+        FOREIGN KEY (run_id) REFERENCES workflow_runs(id) ON DELETE CASCADE
+      );
+      CREATE TABLE workflow_events (
+        id TEXT PRIMARY KEY,
+        run_id TEXT NOT NULL,
+        step_key TEXT,
+        event_type TEXT NOT NULL,
+        level TEXT NOT NULL,
+        payload_json TEXT,
+        timestamp TEXT NOT NULL,
+        FOREIGN KEY (run_id) REFERENCES workflow_runs(id) ON DELETE CASCADE
+      );
+    `;
+
+    function columnNames(database: Database.Database, table: string): string[] {
+      const rows = database.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>;
+      return rows.map((r) => r.name);
+    }
+
+    it('adds completed_steps_json to a legacy workflow_runs table on startup', () => {
+      // Regression: this is the exact column that broke dev-benchmark when an
+      // older deploy's `workflow_runs` survived an upgrade. Renaming the column
+      // requires updating this assertion AND `SCHEMA_MIGRATIONS` in sqlite.ts.
+      const legacyDb = new Database(':memory:');
+      legacyDb.exec(LEGACY_SCHEMA_SQL);
+      expect(columnNames(legacyDb, 'workflow_runs')).not.toContain('completed_steps_json');
+
+      const adapter = new SQLiteStorageAdapter({ db: legacyDb });
+      try {
+        expect(columnNames(legacyDb, 'workflow_runs')).toContain('completed_steps_json');
+      } finally {
+        adapter.close();
+      }
+    });
+
+    it('persists completedSteps after migrating a legacy table', async () => {
+      const legacyDb = new Database(':memory:');
+      legacyDb.exec(LEGACY_SCHEMA_SQL);
+
+      const adapter = new SQLiteStorageAdapter({ db: legacyDb });
+      try {
+        const run = await adapter.createRun({
+          kind: 'test.workflow',
+          status: 'running',
+          input: {},
+          metadata: {},
+          context: {},
+          completedSteps: ['step1', 'step2'],
+        });
+
+        const retrieved = await adapter.getRun(run.id);
+        expect(retrieved!.completedSteps).toEqual(['step1', 'step2']);
+      } finally {
+        adapter.close();
+      }
+    });
+
+    it('is a no-op on a fresh database', () => {
+      // Pure additive: the second adapter on the same DB must not error or
+      // change anything when every column is already present.
+      const freshDb = new Database(':memory:');
+      const adapter1 = new SQLiteStorageAdapter({ db: freshDb });
+      const before = columnNames(freshDb, 'workflow_runs');
+
+      // Simulate a second startup against a fully-migrated DB.
+      const adapter2 = new SQLiteStorageAdapter({ db: freshDb });
+      const after = columnNames(freshDb, 'workflow_runs');
+
+      try {
+        expect(after).toEqual(before);
+        expect(after).toContain('completed_steps_json');
+      } finally {
+        adapter1.close();
+        adapter2.close();
+      }
+    });
+
+    it('still applies migrations when autoCreateTables is false', () => {
+      // The prepared statements always reference the latest schema, so we have
+      // to reconcile columns even when the caller manages CREATE TABLE.
+      const legacyDb = new Database(':memory:');
+      legacyDb.exec(LEGACY_SCHEMA_SQL);
+
+      const adapter = new SQLiteStorageAdapter({ db: legacyDb, autoCreateTables: false });
+      try {
+        expect(columnNames(legacyDb, 'workflow_runs')).toContain('completed_steps_json');
+      } finally {
+        adapter.close();
+      }
+    });
+
+    it('is idempotent across multiple instantiations', () => {
+      // Migrating a DB twice must not error or duplicate columns — exercises
+      // the existence-check skip path for an already-migrated column.
+      const legacyDb = new Database(':memory:');
+      legacyDb.exec(LEGACY_SCHEMA_SQL);
+
+      // First adapter migrates the legacy table.
+      new SQLiteStorageAdapter({ db: legacyDb });
+      const after1 = columnNames(legacyDb, 'workflow_runs');
+      expect(after1).toContain('completed_steps_json');
+
+      // Second adapter on the same DB must be a no-op.
+      const second = new SQLiteStorageAdapter({ db: legacyDb });
+      try {
+        expect(columnNames(legacyDb, 'workflow_runs')).toEqual(after1);
+      } finally {
+        second.close();
+      }
+    });
+  });
 });