@multiplier-labs/stepflow 0.3.1 → 0.3.3

package/src/storage/sqlite.ts

@@ -4,8 +4,8 @@
  * Provides durable persistence for workflow runs, steps, and events.
  */
 
-import type Database from "better-sqlite3";
-import { generateId } from "../utils/id";
+import type Database from 'better-sqlite3';
+import { generateId } from '../utils/id';
 import type {
   StorageAdapter,
   WorkflowRunRecord,
@@ -14,9 +14,9 @@ import type {
   ListRunsOptions,
   ListEventsOptions,
   PaginatedResult,
-} from "./types";
-import type { RunStatus, StepStatus, WorkflowError } from "../core/types";
-import { sanitizeErrorForStorage } from "../utils/logger";
+} from './types';
+import type { RunStatus, StepStatus, WorkflowError } from '../core/types';
+import { sanitizeErrorForStorage } from '../utils/logger';
 
 // ============================================================================
 // Schema SQL
@@ -36,6 +36,7 @@ CREATE TABLE IF NOT EXISTS workflow_runs (
   input_json TEXT NOT NULL,
   metadata_json TEXT NOT NULL,
   context_json TEXT NOT NULL,
+  completed_steps_json TEXT,
   error_json TEXT,
   created_at TEXT NOT NULL,
   started_at TEXT,
@@ -158,10 +159,10 @@ export class SQLiteStorageAdapter implements StorageAdapter {
 
   constructor(config: SQLiteStorageConfig) {
     this.db = config.db;
-    this.prefix = config.tablePrefix ?? "workflow";
+    this.prefix = config.tablePrefix ?? 'workflow';
 
     // Enable foreign keys
-    this.db.pragma("foreign_keys = ON");
+    this.db.pragma('foreign_keys = ON');
 
     // Create tables if needed
     if (config.autoCreateTables !== false) {
@@ -185,8 +186,8 @@ export class SQLiteStorageAdapter implements StorageAdapter {
   private prepareStatements(): void {
     this.stmts = {
       insertRun: this.db.prepare(`
-        INSERT INTO workflow_runs (id, kind, status, parent_run_id, input_json, metadata_json, context_json, error_json, created_at, started_at, finished_at)
-        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        INSERT INTO workflow_runs (id, kind, status, parent_run_id, input_json, metadata_json, context_json, completed_steps_json, error_json, created_at, started_at, finished_at)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
       `),
       getRun: this.db.prepare(`
         SELECT * FROM workflow_runs WHERE id = ?
@@ -195,6 +196,7 @@ export class SQLiteStorageAdapter implements StorageAdapter {
         UPDATE workflow_runs
         SET status = COALESCE(?, status),
             context_json = COALESCE(?, context_json),
+            completed_steps_json = COALESCE(?, completed_steps_json),
             error_json = COALESCE(?, error_json),
             started_at = COALESCE(?, started_at),
             finished_at = COALESCE(?, finished_at)
@@ -277,9 +279,7 @@ export class SQLiteStorageAdapter implements StorageAdapter {
   // Run Operations
   // ============================================================================
 
-  async createRun(
-    run: Omit<WorkflowRunRecord, "id" | "createdAt">,
-  ): Promise<WorkflowRunRecord> {
+  async createRun(run: Omit<WorkflowRunRecord, 'id' | 'createdAt'>): Promise<WorkflowRunRecord> {
     const id = generateId();
     const createdAt = new Date();
 
@@ -291,10 +291,11 @@ export class SQLiteStorageAdapter implements StorageAdapter {
       JSON.stringify(run.input),
       JSON.stringify(run.metadata),
       JSON.stringify(run.context),
+      run.completedSteps ? JSON.stringify(run.completedSteps) : null,
       run.error ? JSON.stringify(sanitizeErrorForStorage(run.error)) : null,
       createdAt.toISOString(),
       run.startedAt?.toISOString() ?? null,
-      run.finishedAt?.toISOString() ?? null,
+      run.finishedAt?.toISOString() ?? null
     );
 
     return {
@@ -309,56 +310,41 @@ export class SQLiteStorageAdapter implements StorageAdapter {
     return row ? this.mapRunRow(row) : null;
   }
 
-  async updateRun(
-    runId: string,
-    updates: Partial<WorkflowRunRecord>,
-  ): Promise<void> {
+  async updateRun(runId: string, updates: Partial<WorkflowRunRecord>): Promise<void> {
     this.stmts!.updateRun.run(
       updates.status ?? null,
       updates.context ? JSON.stringify(updates.context) : null,
-      updates.error
-        ? JSON.stringify(sanitizeErrorForStorage(updates.error))
-        : null,
+      updates.completedSteps ? JSON.stringify(updates.completedSteps) : null,
+      updates.error ? JSON.stringify(sanitizeErrorForStorage(updates.error)) : null,
       updates.startedAt?.toISOString() ?? null,
       updates.finishedAt?.toISOString() ?? null,
-      runId,
+      runId
     );
   }
 
-  async listRuns(
-    options: ListRunsOptions = {},
-  ): Promise<PaginatedResult<WorkflowRunRecord>> {
+  async listRuns(options: ListRunsOptions = {}): Promise<PaginatedResult<WorkflowRunRecord>> {
     const limit = options.limit ?? 50;
     const offset = options.offset ?? 0;
     const kind = options.kind ?? null;
     const status = options.status ?? null;
-    const statusJson = status
-      ? JSON.stringify(Array.isArray(status) ? status : [status])
-      : null;
+    const statusJson = status ? JSON.stringify(Array.isArray(status) ? status : [status]) : null;
     const parentRunId = options.parentRunId ?? null;
 
     const rows = this.stmts!.listRuns.all(
-      kind,
-      kind,
-      statusJson,
-      statusJson,
-      parentRunId,
-      parentRunId,
-      limit,
-      offset,
+      kind, kind,
+      statusJson, statusJson,
+      parentRunId, parentRunId,
+      limit, offset
     ) as SQLiteRunRow[];
 
     const countResult = this.stmts!.countRuns.get(
-      kind,
-      kind,
-      statusJson,
-      statusJson,
-      parentRunId,
-      parentRunId,
+      kind, kind,
+      statusJson, statusJson,
+      parentRunId, parentRunId
     ) as { count: number };
 
     return {
-      items: rows.map((row) => this.mapRunRow(row)),
+      items: rows.map(row => this.mapRunRow(row)),
       total: countResult.count,
       limit,
       offset,
@@ -369,9 +355,7 @@ export class SQLiteStorageAdapter implements StorageAdapter {
   // Step Operations
   // ============================================================================
 
-  async createStep(
-    step: Omit<WorkflowRunStepRecord, "id">,
-  ): Promise<WorkflowRunStepRecord> {
+  async createStep(step: Omit<WorkflowRunStepRecord, 'id'>): Promise<WorkflowRunStepRecord> {
     const id = generateId();
 
     this.stmts!.insertStep.run(
@@ -384,7 +368,7 @@ export class SQLiteStorageAdapter implements StorageAdapter {
       step.result !== undefined ? JSON.stringify(step.result) : null,
       step.error ? JSON.stringify(sanitizeErrorForStorage(step.error)) : null,
       step.startedAt?.toISOString() ?? null,
-      step.finishedAt?.toISOString() ?? null,
+      step.finishedAt?.toISOString() ?? null
     );
 
     return { ...step, id };
@@ -395,32 +379,27 @@ export class SQLiteStorageAdapter implements StorageAdapter {
     return row ? this.mapStepRow(row) : null;
   }
 
-  async updateStep(
-    stepId: string,
-    updates: Partial<WorkflowRunStepRecord>,
-  ): Promise<void> {
+  async updateStep(stepId: string, updates: Partial<WorkflowRunStepRecord>): Promise<void> {
     this.stmts!.updateStep.run(
       updates.status ?? null,
       updates.attempt ?? null,
       updates.result !== undefined ? JSON.stringify(updates.result) : null,
-      updates.error
-        ? JSON.stringify(sanitizeErrorForStorage(updates.error))
-        : null,
+      updates.error ? JSON.stringify(sanitizeErrorForStorage(updates.error)) : null,
       updates.finishedAt?.toISOString() ?? null,
-      stepId,
+      stepId
     );
   }
 
   async getStepsForRun(runId: string): Promise<WorkflowRunStepRecord[]> {
     const rows = this.stmts!.getStepsForRun.all(runId) as SQLiteStepRow[];
-    return rows.map((row) => this.mapStepRow(row));
+    return rows.map(row => this.mapStepRow(row));
   }
 
   // ============================================================================
   // Event Operations
   // ============================================================================
 
-  async saveEvent(event: Omit<WorkflowEventRecord, "id">): Promise<void> {
+  async saveEvent(event: Omit<WorkflowEventRecord, 'id'>): Promise<void> {
     const id = generateId();
 
     this.stmts!.insertEvent.run(
@@ -430,14 +409,11 @@ export class SQLiteStorageAdapter implements StorageAdapter {
       event.eventType,
       event.level,
       event.payload !== undefined ? JSON.stringify(event.payload) : null,
-      event.timestamp.toISOString(),
+      event.timestamp.toISOString()
     );
   }
 
-  async getEventsForRun(
-    runId: string,
-    options: ListEventsOptions = {},
-  ): Promise<WorkflowEventRecord[]> {
+  async getEventsForRun(runId: string, options: ListEventsOptions = {}): Promise<WorkflowEventRecord[]> {
     const limit = options.limit ?? 1000;
     const offset = options.offset ?? 0;
     const stepKey = options.stepKey ?? null;
@@ -445,15 +421,12 @@ export class SQLiteStorageAdapter implements StorageAdapter {
 
     const rows = this.stmts!.getEventsForRun.all(
       runId,
-      stepKey,
-      stepKey,
-      level,
-      level,
-      limit,
-      offset,
+      stepKey, stepKey,
+      level, level,
+      limit, offset
     ) as SQLiteEventRow[];
 
-    return rows.map((row) => this.mapEventRow(row));
+    return rows.map(row => this.mapEventRow(row));
   }
 
   // ============================================================================
@@ -524,19 +497,15 @@ export class SQLiteStorageAdapter implements StorageAdapter {
    */
   async getInterruptedRuns(): Promise<WorkflowRunRecord[]> {
     const rows = this.stmts!.getInterruptedRuns.all() as SQLiteRunRow[];
-    return rows.map((row) => this.mapRunRow(row));
+    return rows.map(row => this.mapRunRow(row));
   }
 
   /**
    * Get the last completed step for a run.
    * Useful for resuming from a checkpoint.
    */
-  async getLastCompletedStep(
-    runId: string,
-  ): Promise<WorkflowRunStepRecord | null> {
-    const row = this.stmts!.getLastCompletedStep.get(runId) as
-      | SQLiteStepRow
-      | undefined;
+  async getLastCompletedStep(runId: string): Promise<WorkflowRunStepRecord | null> {
+    const row = this.stmts!.getLastCompletedStep.get(runId) as SQLiteStepRow | undefined;
     return row ? this.mapStepRow(row) : null;
   }
 
@@ -553,6 +522,7 @@ export class SQLiteStorageAdapter implements StorageAdapter {
       input: JSON.parse(row.input_json),
       metadata: JSON.parse(row.metadata_json),
       context: JSON.parse(row.context_json),
+      completedSteps: row.completed_steps_json ? JSON.parse(row.completed_steps_json) : undefined,
       error: row.error_json ? JSON.parse(row.error_json) : undefined,
       createdAt: new Date(row.created_at),
       startedAt: row.started_at ? new Date(row.started_at) : undefined,
@@ -581,7 +551,7 @@ export class SQLiteStorageAdapter implements StorageAdapter {
       runId: row.run_id,
       stepKey: row.step_key ?? undefined,
       eventType: row.event_type,
-      level: row.level as "info" | "warn" | "error",
+      level: row.level as 'info' | 'warn' | 'error',
       payload: row.payload_json ? JSON.parse(row.payload_json) : undefined,
       timestamp: new Date(row.timestamp),
     };
@@ -602,15 +572,9 @@ export class SQLiteStorageAdapter implements StorageAdapter {
    * Get database statistics.
    */
   getStats(): { runs: number; steps: number; events: number } {
-    const runsCount = this.db
-      .prepare("SELECT COUNT(*) as count FROM workflow_runs")
-      .get() as { count: number };
-    const stepsCount = this.db
-      .prepare("SELECT COUNT(*) as count FROM workflow_run_steps")
-      .get() as { count: number };
-    const eventsCount = this.db
-      .prepare("SELECT COUNT(*) as count FROM workflow_events")
-      .get() as { count: number };
+    const runsCount = this.db.prepare('SELECT COUNT(*) as count FROM workflow_runs').get() as { count: number };
+    const stepsCount = this.db.prepare('SELECT COUNT(*) as count FROM workflow_run_steps').get() as { count: number };
+    const eventsCount = this.db.prepare('SELECT COUNT(*) as count FROM workflow_events').get() as { count: number };
 
     return {
       runs: runsCount.count,
@@ -632,6 +596,7 @@ interface SQLiteRunRow {
   input_json: string;
   metadata_json: string;
   context_json: string;
+  completed_steps_json: string | null;
   error_json: string | null;
   created_at: string;
   started_at: string | null;

package/src/storage/types.ts

@@ -3,12 +3,7 @@
  * Implement StorageAdapter to use your preferred database.
  */
 
-import type {
-  WorkflowKind,
-  RunStatus,
-  StepStatus,
-  WorkflowError,
-} from "../core/types";
+import type { WorkflowKind, RunStatus, StepStatus, WorkflowError } from '../core/types';
 
 // ============================================================================
 // Extended Status Types (New)
@@ -19,25 +14,25 @@ import type {
  * Adds 'pending' and 'timeout' to the core statuses.
  */
 export type ExtendedRunStatus =
-  | "pending"
-  | "queued"
-  | "running"
-  | "succeeded"
-  | "failed"
-  | "canceled"
-  | "timeout";
+  | 'pending'
+  | 'queued'
+  | 'running'
+  | 'succeeded'
+  | 'failed'
+  | 'canceled'
+  | 'timeout';
 
 /**
  * Extended status of a workflow step.
  * Uses 'completed' instead of 'succeeded' for consistency.
  */
 export type ExtendedStepStatus =
-  | "pending"
-  | "running"
-  | "completed"
-  | "failed"
-  | "skipped"
-  | "canceled";
+  | 'pending'
+  | 'running'
+  | 'completed'
+  | 'failed'
+  | 'skipped'
+  | 'canceled';
 
 // ============================================================================
 // Legacy Record Types (Backward Compatible)
@@ -53,6 +48,7 @@ export interface WorkflowRunRecord {
   parentRunId?: string;
   input: Record<string, unknown>;
   context: Record<string, unknown>;
+  completedSteps?: string[];
   output?: Record<string, unknown>;
   error?: { code: string; message: string };
   metadata?: Record<string, unknown>;
@@ -87,7 +83,7 @@ export interface WorkflowEventRecord {
   runId: string;
   stepKey?: string;
   eventType: string;
-  level: "info" | "warn" | "error";
+  level: 'info' | 'warn' | 'error';
   payload?: unknown;
   timestamp: Date;
 }
@@ -189,8 +185,8 @@ export interface ListRunsOptions {
   parentRunId?: string;
   limit?: number;
   offset?: number;
-  orderBy?: "createdAt" | "startedAt" | "finishedAt";
-  orderDirection?: "asc" | "desc";
+  orderBy?: 'createdAt' | 'startedAt' | 'finishedAt';
+  orderDirection?: 'asc' | 'desc';
 }
 
 /**
@@ -201,8 +197,8 @@ export interface ExtendedListRunsOptions {
   status?: ExtendedRunStatus | ExtendedRunStatus[];
   limit?: number;
   offset?: number;
-  orderBy?: "createdAt" | "startedAt" | "finishedAt";
-  orderDir?: "asc" | "desc";
+  orderBy?: 'createdAt' | 'startedAt' | 'finishedAt';
+  orderDir?: 'asc' | 'desc';
 }
 
 /**
@@ -210,7 +206,7 @@ export interface ExtendedListRunsOptions {
  */
 export interface ListEventsOptions {
   stepKey?: string;
-  level?: "info" | "warn" | "error";
+  level?: 'info' | 'warn' | 'error';
   limit?: number;
   offset?: number;
 }
@@ -238,30 +234,21 @@ export interface WorkflowStorage {
   createRun(run: CreateRunInput): Promise<ExtendedWorkflowRunRecord>;
   getRun(id: string): Promise<ExtendedWorkflowRunRecord | null>;
   updateRun(id: string, updates: UpdateRunInput): Promise<void>;
-  listRuns(
-    options?: ExtendedListRunsOptions,
-  ): Promise<PaginatedResult<ExtendedWorkflowRunRecord>>;
+  listRuns(options?: ExtendedListRunsOptions): Promise<PaginatedResult<ExtendedWorkflowRunRecord>>;
   deleteRun(id: string): Promise<void>;
 
   // Atomic dequeue for concurrency control
-  dequeueRun(
-    workflowKinds: string[],
-  ): Promise<ExtendedWorkflowRunRecord | null>;
+  dequeueRun(workflowKinds: string[]): Promise<ExtendedWorkflowRunRecord | null>;
 
   // Stale workflow cleanup
   cleanupStaleRuns(defaultTimeoutMs?: number): Promise<number>;
   markRunsAsFailed(runIds: string[], reason: string): Promise<void>;
 
   // Step operations
-  getStepResult(
-    runId: string,
-    stepName: string,
-  ): Promise<StepResult | undefined>;
+  getStepResult(runId: string, stepName: string): Promise<StepResult | undefined>;
   getStepResults(runId: string): Promise<StepResult[]>;
   getStepsForRun(runId: string): Promise<StepRecord[]>;
-  saveStepResult(
-    result: Omit<StepResult, "id"> & { id?: string },
-  ): Promise<void>;
+  saveStepResult(result: Omit<StepResult, 'id'> & { id?: string }): Promise<void>;
 
   // Lifecycle
   initialize(): Promise<void>;
@@ -278,32 +265,20 @@ export interface WorkflowStorage {
  */
 export interface StorageAdapter {
   // Run operations
-  createRun(
-    run: Omit<WorkflowRunRecord, "id" | "createdAt">,
-  ): Promise<WorkflowRunRecord>;
+  createRun(run: Omit<WorkflowRunRecord, 'id' | 'createdAt'>): Promise<WorkflowRunRecord>;
   getRun(runId: string): Promise<WorkflowRunRecord | null>;
   updateRun(runId: string, updates: Partial<WorkflowRunRecord>): Promise<void>;
-  listRuns(
-    options?: ListRunsOptions,
-  ): Promise<PaginatedResult<WorkflowRunRecord>>;
+  listRuns(options?: ListRunsOptions): Promise<PaginatedResult<WorkflowRunRecord>>;
 
   // Step operations
-  createStep(
-    step: Omit<WorkflowRunStepRecord, "id">,
-  ): Promise<WorkflowRunStepRecord>;
+  createStep(step: Omit<WorkflowRunStepRecord, 'id'>): Promise<WorkflowRunStepRecord>;
   getStep(stepId: string): Promise<WorkflowRunStepRecord | null>;
-  updateStep(
-    stepId: string,
-    updates: Partial<WorkflowRunStepRecord>,
-  ): Promise<void>;
+  updateStep(stepId: string, updates: Partial<WorkflowRunStepRecord>): Promise<void>;
   getStepsForRun(runId: string): Promise<WorkflowRunStepRecord[]>;
 
   // Event operations
-  saveEvent(event: Omit<WorkflowEventRecord, "id">): Promise<void>;
-  getEventsForRun(
-    runId: string,
-    options?: ListEventsOptions,
-  ): Promise<WorkflowEventRecord[]>;
+  saveEvent(event: Omit<WorkflowEventRecord, 'id'>): Promise<void>;
+  getEventsForRun(runId: string, options?: ListEventsOptions): Promise<WorkflowEventRecord[]>;
 
   /**
    * Optional transaction support.
@@ -363,6 +338,6 @@ export interface StepflowStepResultsTable {
  * Combined database schema for Stepflow.
  */
 export interface StepflowDatabase {
-  "stepflow.runs": StepflowRunsTable;
-  "stepflow.step_results": StepflowStepResultsTable;
+  'stepflow.runs': StepflowRunsTable;
+  'stepflow.step_results': StepflowStepResultsTable;
 }