@multiplier-labs/stepflow 0.3.3 → 0.3.6

package/src/storage/sqlite.ts

@@ -1,627 +0,0 @@
-/**
- * SQLite storage adapter using better-sqlite3.
- *
- * Provides durable persistence for workflow runs, steps, and events.
- */
-
-import type Database from 'better-sqlite3';
-import { generateId } from '../utils/id';
-import type {
-  StorageAdapter,
-  WorkflowRunRecord,
-  WorkflowRunStepRecord,
-  WorkflowEventRecord,
-  ListRunsOptions,
-  ListEventsOptions,
-  PaginatedResult,
-} from './types';
-import type { RunStatus, StepStatus, WorkflowError } from '../core/types';
-import { sanitizeErrorForStorage } from '../utils/logger';
-
-// ============================================================================
-// Schema SQL
-// ============================================================================
-
-/**
- * SQL statements to create the workflow tables.
- * These create the workflow tables in your database.
- */
-const CREATE_TABLES_SQL = `
--- Workflow runs table
-CREATE TABLE IF NOT EXISTS workflow_runs (
-  id TEXT PRIMARY KEY,
-  kind TEXT NOT NULL,
-  status TEXT NOT NULL,
-  parent_run_id TEXT,
-  input_json TEXT NOT NULL,
-  metadata_json TEXT NOT NULL,
-  context_json TEXT NOT NULL,
-  completed_steps_json TEXT,
-  error_json TEXT,
-  created_at TEXT NOT NULL,
-  started_at TEXT,
-  finished_at TEXT
-);
-
-CREATE INDEX IF NOT EXISTS idx_workflow_runs_kind_status ON workflow_runs(kind, status);
-CREATE INDEX IF NOT EXISTS idx_workflow_runs_parent ON workflow_runs(parent_run_id);
-CREATE INDEX IF NOT EXISTS idx_workflow_runs_created ON workflow_runs(created_at);
-CREATE INDEX IF NOT EXISTS idx_workflow_runs_status ON workflow_runs(status);
-
--- Workflow run steps table
-CREATE TABLE IF NOT EXISTS workflow_run_steps (
-  id TEXT PRIMARY KEY,
-  run_id TEXT NOT NULL,
-  step_key TEXT NOT NULL,
-  step_name TEXT,
-  status TEXT NOT NULL,
-  attempt INTEGER NOT NULL DEFAULT 1,
-  result_json TEXT,
-  error_json TEXT,
-  started_at TEXT,
-  finished_at TEXT,
-  FOREIGN KEY (run_id) REFERENCES workflow_runs(id) ON DELETE CASCADE
-);
-
-CREATE INDEX IF NOT EXISTS idx_workflow_run_steps_run ON workflow_run_steps(run_id);
-CREATE INDEX IF NOT EXISTS idx_workflow_run_steps_run_key ON workflow_run_steps(run_id, step_key);
-
--- Workflow events table
-CREATE TABLE IF NOT EXISTS workflow_events (
-  id TEXT PRIMARY KEY,
-  run_id TEXT NOT NULL,
-  step_key TEXT,
-  event_type TEXT NOT NULL,
-  level TEXT NOT NULL,
-  payload_json TEXT,
-  timestamp TEXT NOT NULL,
-  FOREIGN KEY (run_id) REFERENCES workflow_runs(id) ON DELETE CASCADE
-);
-
-CREATE INDEX IF NOT EXISTS idx_workflow_events_run ON workflow_events(run_id);
-CREATE INDEX IF NOT EXISTS idx_workflow_events_run_ts ON workflow_events(run_id, timestamp);
-`;
-
-// ============================================================================
-// Configuration
-// ============================================================================
-
-/**
- * Configuration options for the SQLite storage adapter.
- */
-export interface SQLiteStorageConfig {
-  /**
-   * The better-sqlite3 database instance.
-   * Can be an in-memory database (`:memory:`) or a file path.
-   */
-  db: Database.Database;
-
-  /**
-   * Whether to automatically create tables on initialization.
-   * Default: true
-   */
-  autoCreateTables?: boolean;
-
-  /**
-   * @deprecated This option is not currently supported and will be ignored.
-   * Table names are always prefixed with 'workflow_'.
-   */
-  tablePrefix?: string;
-}
-
-// ============================================================================
-// SQLite Storage Adapter
-// ============================================================================
-
-/**
- * SQLite implementation of StorageAdapter using better-sqlite3.
- *
- * Features:
- * - Synchronous operations (better-sqlite3 is sync by design)
- * - Automatic table creation
- * - Transaction support
- * - Customizable table prefix
- *
- * @example
- * ```typescript
- * import Database from 'better-sqlite3';
- * import { SQLiteStorageAdapter } from 'stepflow/storage';
- *
- * const db = new Database('./workflows.db');
- * const storage = new SQLiteStorageAdapter({ db });
- *
- * const engine = new WorkflowEngine({ storage });
- * ```
- */
-export class SQLiteStorageAdapter implements StorageAdapter {
-  private db: Database.Database;
-  private prefix: string;
-
-  // Prepared statements (cached for performance)
-  private stmts: {
-    insertRun: Database.Statement;
-    getRun: Database.Statement;
-    updateRun: Database.Statement;
-    listRuns: Database.Statement;
-    countRuns: Database.Statement;
-    insertStep: Database.Statement;
-    getStep: Database.Statement;
-    updateStep: Database.Statement;
-    getStepsForRun: Database.Statement;
-    insertEvent: Database.Statement;
-    getEventsForRun: Database.Statement;
-    deleteOldRuns: Database.Statement;
-    deleteStepsForRuns: Database.Statement;
-    deleteEventsForRuns: Database.Statement;
-    getInterruptedRuns: Database.Statement;
-    getLastCompletedStep: Database.Statement;
-  } | null = null;
-
-  constructor(config: SQLiteStorageConfig) {
-    this.db = config.db;
-    this.prefix = config.tablePrefix ?? 'workflow';
-
-    // Enable foreign keys
-    this.db.pragma('foreign_keys = ON');
-
-    // Create tables if needed
-    if (config.autoCreateTables !== false) {
-      this.createTables();
-    }
-
-    // Prepare statements
-    this.prepareStatements();
-  }
-
-  /**
-   * Create the workflow tables if they don't exist.
-   */
-  private createTables(): void {
-    this.db.exec(CREATE_TABLES_SQL);
-  }
-
-  /**
-   * Prepare all SQL statements for better performance.
-   */
-  private prepareStatements(): void {
-    this.stmts = {
-      insertRun: this.db.prepare(`
-        INSERT INTO workflow_runs (id, kind, status, parent_run_id, input_json, metadata_json, context_json, completed_steps_json, error_json, created_at, started_at, finished_at)
-        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-      `),
-      getRun: this.db.prepare(`
-        SELECT * FROM workflow_runs WHERE id = ?
-      `),
-      updateRun: this.db.prepare(`
-        UPDATE workflow_runs
-        SET status = COALESCE(?, status),
-            context_json = COALESCE(?, context_json),
-            completed_steps_json = COALESCE(?, completed_steps_json),
-            error_json = COALESCE(?, error_json),
-            started_at = COALESCE(?, started_at),
-            finished_at = COALESCE(?, finished_at)
-        WHERE id = ?
-      `),
-      // json_each() is a SQLite table-valued function that expands a JSON array into rows.
-      // We use it here to support filtering by multiple statuses in a single prepared statement:
-      // the caller passes a JSON array like '["running","failed"]', and json_each unpacks it
-      // so the IN clause can match against each value. When status is NULL, the entire
-      // condition is bypassed via the (? IS NULL OR ...) guard.
-      listRuns: this.db.prepare(`
-        SELECT * FROM workflow_runs
-        WHERE (? IS NULL OR kind = ?)
-          AND (? IS NULL OR status IN (SELECT value FROM json_each(?)))
-          AND (? IS NULL OR parent_run_id = ?)
-        ORDER BY created_at DESC
-        LIMIT ? OFFSET ?
-      `),
-      countRuns: this.db.prepare(`
-        SELECT COUNT(*) as count FROM workflow_runs
-        WHERE (? IS NULL OR kind = ?)
-          AND (? IS NULL OR status IN (SELECT value FROM json_each(?)))
-          AND (? IS NULL OR parent_run_id = ?)
-      `),
-      insertStep: this.db.prepare(`
-        INSERT INTO workflow_run_steps (id, run_id, step_key, step_name, status, attempt, result_json, error_json, started_at, finished_at)
-        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-      `),
-      getStep: this.db.prepare(`
-        SELECT * FROM workflow_run_steps WHERE id = ?
-      `),
-      updateStep: this.db.prepare(`
-        UPDATE workflow_run_steps
-        SET status = COALESCE(?, status),
-            attempt = COALESCE(?, attempt),
-            result_json = COALESCE(?, result_json),
-            error_json = COALESCE(?, error_json),
-            finished_at = COALESCE(?, finished_at)
-        WHERE id = ?
-      `),
-      getStepsForRun: this.db.prepare(`
-        SELECT * FROM workflow_run_steps WHERE run_id = ? ORDER BY started_at ASC
-      `),
-      insertEvent: this.db.prepare(`
-        INSERT INTO workflow_events (id, run_id, step_key, event_type, level, payload_json, timestamp)
-        VALUES (?, ?, ?, ?, ?, ?, ?)
-      `),
-      getEventsForRun: this.db.prepare(`
-        SELECT * FROM workflow_events
-        WHERE run_id = ?
-          AND (? IS NULL OR step_key = ?)
-          AND (? IS NULL OR level = ?)
-        ORDER BY timestamp ASC
-        LIMIT ? OFFSET ?
-      `),
-      deleteOldRuns: this.db.prepare(`
-        DELETE FROM workflow_runs WHERE created_at < ?
-      `),
-      deleteStepsForRuns: this.db.prepare(`
-        DELETE FROM workflow_run_steps WHERE run_id IN (SELECT id FROM workflow_runs WHERE created_at < ?)
-      `),
-      deleteEventsForRuns: this.db.prepare(`
-        DELETE FROM workflow_events WHERE run_id IN (SELECT id FROM workflow_runs WHERE created_at < ?)
-      `),
-      getInterruptedRuns: this.db.prepare(`
-        SELECT * FROM workflow_runs
-        WHERE status IN ('queued', 'running')
-        ORDER BY created_at ASC
-      `),
-      getLastCompletedStep: this.db.prepare(`
-        SELECT * FROM workflow_run_steps
-        WHERE run_id = ? AND status = 'succeeded'
-        ORDER BY finished_at DESC
-        LIMIT 1
-      `),
-    };
-  }
-
-  // ============================================================================
-  // Run Operations
-  // ============================================================================
-
-  async createRun(run: Omit<WorkflowRunRecord, 'id' | 'createdAt'>): Promise<WorkflowRunRecord> {
-    const id = generateId();
-    const createdAt = new Date();
-
-    this.stmts!.insertRun.run(
-      id,
-      run.kind,
-      run.status,
-      run.parentRunId ?? null,
-      JSON.stringify(run.input),
-      JSON.stringify(run.metadata),
-      JSON.stringify(run.context),
-      run.completedSteps ? JSON.stringify(run.completedSteps) : null,
-      run.error ? JSON.stringify(sanitizeErrorForStorage(run.error)) : null,
-      createdAt.toISOString(),
-      run.startedAt?.toISOString() ?? null,
-      run.finishedAt?.toISOString() ?? null
-    );
-
-    return {
-      ...run,
-      id,
-      createdAt,
-    };
-  }
-
-  async getRun(runId: string): Promise<WorkflowRunRecord | null> {
-    const row = this.stmts!.getRun.get(runId) as SQLiteRunRow | undefined;
-    return row ? this.mapRunRow(row) : null;
-  }
-
-  async updateRun(runId: string, updates: Partial<WorkflowRunRecord>): Promise<void> {
-    this.stmts!.updateRun.run(
-      updates.status ?? null,
-      updates.context ? JSON.stringify(updates.context) : null,
-      updates.completedSteps ? JSON.stringify(updates.completedSteps) : null,
-      updates.error ? JSON.stringify(sanitizeErrorForStorage(updates.error)) : null,
-      updates.startedAt?.toISOString() ?? null,
-      updates.finishedAt?.toISOString() ?? null,
-      runId
-    );
-  }
-
-  async listRuns(options: ListRunsOptions = {}): Promise<PaginatedResult<WorkflowRunRecord>> {
-    const limit = options.limit ?? 50;
-    const offset = options.offset ?? 0;
-    const kind = options.kind ?? null;
-    const status = options.status ?? null;
-    const statusJson = status ? JSON.stringify(Array.isArray(status) ? status : [status]) : null;
-    const parentRunId = options.parentRunId ?? null;
-
-    const rows = this.stmts!.listRuns.all(
-      kind, kind,
-      statusJson, statusJson,
-      parentRunId, parentRunId,
-      limit, offset
-    ) as SQLiteRunRow[];
-
-    const countResult = this.stmts!.countRuns.get(
-      kind, kind,
-      statusJson, statusJson,
-      parentRunId, parentRunId
-    ) as { count: number };
-
-    return {
-      items: rows.map(row => this.mapRunRow(row)),
-      total: countResult.count,
-      limit,
-      offset,
-    };
-  }
-
-  // ============================================================================
-  // Step Operations
-  // ============================================================================
-
-  async createStep(step: Omit<WorkflowRunStepRecord, 'id'>): Promise<WorkflowRunStepRecord> {
-    const id = generateId();
-
-    this.stmts!.insertStep.run(
-      id,
-      step.runId,
-      step.stepKey,
-      step.stepName,
-      step.status,
-      step.attempt,
-      step.result !== undefined ? JSON.stringify(step.result) : null,
-      step.error ? JSON.stringify(sanitizeErrorForStorage(step.error)) : null,
-      step.startedAt?.toISOString() ?? null,
-      step.finishedAt?.toISOString() ?? null
-    );
-
-    return { ...step, id };
-  }
-
-  async getStep(stepId: string): Promise<WorkflowRunStepRecord | null> {
-    const row = this.stmts!.getStep.get(stepId) as SQLiteStepRow | undefined;
-    return row ? this.mapStepRow(row) : null;
-  }
-
-  async updateStep(stepId: string, updates: Partial<WorkflowRunStepRecord>): Promise<void> {
-    this.stmts!.updateStep.run(
-      updates.status ?? null,
-      updates.attempt ?? null,
-      updates.result !== undefined ? JSON.stringify(updates.result) : null,
-      updates.error ? JSON.stringify(sanitizeErrorForStorage(updates.error)) : null,
-      updates.finishedAt?.toISOString() ?? null,
-      stepId
-    );
-  }
-
-  async getStepsForRun(runId: string): Promise<WorkflowRunStepRecord[]> {
-    const rows = this.stmts!.getStepsForRun.all(runId) as SQLiteStepRow[];
-    return rows.map(row => this.mapStepRow(row));
-  }
-
-  // ============================================================================
-  // Event Operations
-  // ============================================================================
-
-  async saveEvent(event: Omit<WorkflowEventRecord, 'id'>): Promise<void> {
-    const id = generateId();
-
-    this.stmts!.insertEvent.run(
-      id,
-      event.runId,
-      event.stepKey ?? null,
-      event.eventType,
-      event.level,
-      event.payload !== undefined ? JSON.stringify(event.payload) : null,
-      event.timestamp.toISOString()
-    );
-  }
-
-  async getEventsForRun(runId: string, options: ListEventsOptions = {}): Promise<WorkflowEventRecord[]> {
-    const limit = options.limit ?? 1000;
-    const offset = options.offset ?? 0;
-    const stepKey = options.stepKey ?? null;
-    const level = options.level ?? null;
-
-    const rows = this.stmts!.getEventsForRun.all(
-      runId,
-      stepKey, stepKey,
-      level, level,
-      limit, offset
-    ) as SQLiteEventRow[];
-
-    return rows.map(row => this.mapEventRow(row));
-  }
-
-  // ============================================================================
-  // Transaction Support
-  // ============================================================================
-
-  /**
-   * Execute a function within a database transaction (async interface).
-   *
-   * @deprecated Use `transactionSync()` instead. This method only works when
-   * the callback performs purely synchronous operations wrapped in async/await.
-   * If the callback awaits real async I/O (network, timers, etc.), it will
-   * throw an error. `transactionSync()` makes the synchronous requirement explicit.
-   */
-  async transaction<T>(fn: (tx: StorageAdapter) => Promise<T>): Promise<T> {
-    let fnPromise!: Promise<T>;
-    this.transactionSync(() => {
-      fnPromise = fn(this);
-    });
-    return fnPromise;
-  }
-
-  /**
-   * Execute a synchronous transaction (preferred for better-sqlite3).
-   * Use this when you need transaction guarantees.
-   *
-   * @example
-   * ```typescript
-   * storage.transactionSync(() => {
-   *   // Multiple operations in a single transaction
-   *   const run = storage.createRunSync(...);
-   *   storage.updateRunSync(run.id, ...);
-   * });
-   * ```
-   */
-  transactionSync<T>(fn: () => T): T {
-    return this.db.transaction(fn)();
-  }
-
-  // ============================================================================
-  // Cleanup Operations
-  // ============================================================================
-
-  /**
-   * Delete runs older than the specified date.
-   * Also deletes associated steps and events (via CASCADE).
-   */
-  async deleteOldRuns(olderThan: Date): Promise<number> {
-    const isoDate = olderThan.toISOString();
-
-    // Due to CASCADE, steps and events are deleted automatically
-    // But we do it explicitly for databases without CASCADE support
-    return this.db.transaction(() => {
-      this.stmts!.deleteEventsForRuns.run(isoDate);
-      this.stmts!.deleteStepsForRuns.run(isoDate);
-      const result = this.stmts!.deleteOldRuns.run(isoDate);
-      return result.changes;
-    })();
-  }
-
-  // ============================================================================
-  // Resume Support
-  // ============================================================================
-
-  /**
-   * Get all runs that were interrupted (status is 'running' or 'queued').
-   * These runs can potentially be resumed.
-   */
-  async getInterruptedRuns(): Promise<WorkflowRunRecord[]> {
-    const rows = this.stmts!.getInterruptedRuns.all() as SQLiteRunRow[];
-    return rows.map(row => this.mapRunRow(row));
-  }
-
-  /**
-   * Get the last completed step for a run.
-   * Useful for resuming from a checkpoint.
-   */
-  async getLastCompletedStep(runId: string): Promise<WorkflowRunStepRecord | null> {
-    const row = this.stmts!.getLastCompletedStep.get(runId) as SQLiteStepRow | undefined;
-    return row ? this.mapStepRow(row) : null;
-  }
-
-  // ============================================================================
-  // Row Mapping
-  // ============================================================================
-
-  private mapRunRow(row: SQLiteRunRow): WorkflowRunRecord {
-    return {
-      id: row.id,
-      kind: row.kind,
-      status: row.status as RunStatus,
-      parentRunId: row.parent_run_id ?? undefined,
-      input: JSON.parse(row.input_json),
-      metadata: JSON.parse(row.metadata_json),
-      context: JSON.parse(row.context_json),
-      completedSteps: row.completed_steps_json ? JSON.parse(row.completed_steps_json) : undefined,
-      error: row.error_json ? JSON.parse(row.error_json) : undefined,
-      createdAt: new Date(row.created_at),
-      startedAt: row.started_at ? new Date(row.started_at) : undefined,
-      finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,
-    };
-  }
-
-  private mapStepRow(row: SQLiteStepRow): WorkflowRunStepRecord {
-    return {
-      id: row.id,
-      runId: row.run_id,
-      stepKey: row.step_key,
-      stepName: row.step_name,
-      status: row.status as StepStatus,
-      attempt: row.attempt,
-      result: row.result_json ? JSON.parse(row.result_json) : undefined,
-      error: row.error_json ? JSON.parse(row.error_json) : undefined,
-      startedAt: row.started_at ? new Date(row.started_at) : undefined,
-      finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,
-    };
-  }
-
-  private mapEventRow(row: SQLiteEventRow): WorkflowEventRecord {
-    return {
-      id: row.id,
-      runId: row.run_id,
-      stepKey: row.step_key ?? undefined,
-      eventType: row.event_type,
-      level: row.level as 'info' | 'warn' | 'error',
-      payload: row.payload_json ? JSON.parse(row.payload_json) : undefined,
-      timestamp: new Date(row.timestamp),
-    };
-  }
-
-  // ============================================================================
-  // Utility Methods
-  // ============================================================================
-
-  /**
-   * Close the database connection.
-   */
-  close(): void {
-    this.db.close();
-  }
-
-  /**
-   * Get database statistics.
-   */
-  getStats(): { runs: number; steps: number; events: number } {
-    const runsCount = this.db.prepare('SELECT COUNT(*) as count FROM workflow_runs').get() as { count: number };
-    const stepsCount = this.db.prepare('SELECT COUNT(*) as count FROM workflow_run_steps').get() as { count: number };
-    const eventsCount = this.db.prepare('SELECT COUNT(*) as count FROM workflow_events').get() as { count: number };
-
-    return {
-      runs: runsCount.count,
-      steps: stepsCount.count,
-      events: eventsCount.count,
-    };
-  }
-}
-
-// ============================================================================
-// SQLite Row Types
-// ============================================================================
-
-interface SQLiteRunRow {
-  id: string;
-  kind: string;
-  status: string;
-  parent_run_id: string | null;
-  input_json: string;
-  metadata_json: string;
-  context_json: string;
-  completed_steps_json: string | null;
-  error_json: string | null;
-  created_at: string;
-  started_at: string | null;
-  finished_at: string | null;
-}
-
-interface SQLiteStepRow {
-  id: string;
-  run_id: string;
-  step_key: string;
-  step_name: string;
-  status: string;
-  attempt: number;
-  result_json: string | null;
-  error_json: string | null;
-  started_at: string | null;
-  finished_at: string | null;
-}
-
-interface SQLiteEventRow {
-  id: string;
-  run_id: string;
-  step_key: string | null;
-  event_type: string;
-  level: string;
-  payload_json: string | null;
-  timestamp: string;
-}