keystone-cli 0.8.0 → 1.0.0

package/src/db/memory-db.ts

@@ -16,6 +16,7 @@ export class MemoryDb {
   private db: Database;
   // Cache connections by path to avoid reloading extensions
   private static connectionCache = new Map<string, { db: Database; refCount: number }>();
+  static readonly EMBEDDING_DIMENSION = 384;
 
   constructor(public readonly dbPath = '.keystone/memory.db') {
     const cached = MemoryDb.connectionCache.get(dbPath);
@@ -44,7 +45,7 @@ export class MemoryDb {
     this.db.run(`
       CREATE VIRTUAL TABLE IF NOT EXISTS vec_memory USING vec0(
         id TEXT PRIMARY KEY,
-        embedding FLOAT[384]
+        embedding FLOAT[${MemoryDb.EMBEDDING_DIMENSION}]
       );
     `);
 
@@ -58,6 +59,26 @@ export class MemoryDb {
     `);
   }
 
+  private static assertEmbeddingDimension(embedding: number[]): void {
+    if (embedding.length !== MemoryDb.EMBEDDING_DIMENSION) {
+      throw new Error(
+        `Embedding dimension mismatch: expected ${MemoryDb.EMBEDDING_DIMENSION}, got ${embedding.length}`
+      );
+    }
+  }
+
+  /**
+   * Store an embedding and its associated text/metadata.
+   *
+   * Note: The async signature provides interface compatibility with potentially
+   * async backends (e.g., remote vector DBs). The current implementation uses
+   * synchronous bun:sqlite operations internally.
+   *
+   * @param text - The text content to store
+   * @param embedding - The embedding vector (384 dimensions)
+   * @param metadata - Optional metadata to associate with the entry
+   * @returns The generated entry ID
+   */
   async store(
     text: string,
     embedding: number[],
@@ -65,6 +86,7 @@ export class MemoryDb {
   ): Promise<string> {
     const id = randomUUID();
     const createdAt = new Date().toISOString();
+    MemoryDb.assertEmbeddingDimension(embedding);
 
     // bun:sqlite transaction wrapper ensures atomicity synchronously
     const insertTransaction = this.db.transaction(() => {
@@ -83,7 +105,19 @@ export class MemoryDb {
     return id;
   }
 
+  /**
+   * Search for similar embeddings using vector similarity.
+   *
+   * Note: The async signature provides interface compatibility with potentially
+   * async backends (e.g., remote vector DBs). The current implementation uses
+   * synchronous bun:sqlite operations internally.
+   *
+   * @param embedding - The query embedding vector
+   * @param limit - Maximum number of results to return (default: 5)
+   * @returns Array of matching entries with distance scores
+   */
   async search(embedding: number[], limit = 5): Promise<MemoryEntry[]> {
+    MemoryDb.assertEmbeddingDimension(embedding);
     const query = `
       SELECT 
         v.id, 

package/src/db/workflow-db.test.ts

@@ -47,6 +47,30 @@ describe('WorkflowDb', () => {
     expect(steps[0].status).toBe('success');
   });
 
+  it('should persist falsy outputs', async () => {
+    const runId = 'run-falsy';
+    await db.createRun(runId, 'test-wf', {});
+
+    await db.createStep('exec-false', runId, 'step-false');
+    await db.startStep('exec-false');
+    await db.completeStep('exec-false', 'success', false);
+
+    await db.createStep('exec-zero', runId, 'step-zero');
+    await db.startStep('exec-zero');
+    await db.completeStep('exec-zero', 'success', 0);
+
+    await db.createStep('exec-empty', runId, 'step-empty');
+    await db.startStep('exec-empty');
+    await db.completeStep('exec-empty', 'success', '');
+
+    const steps = await db.getStepsByRun(runId);
+    const outputsById = Object.fromEntries(steps.map((step) => [step.step_id, step.output]));
+
+    expect(JSON.parse(outputsById['step-false'] || 'null')).toBe(false);
+    expect(JSON.parse(outputsById['step-zero'] || 'null')).toBe(0);
+    expect(JSON.parse(outputsById['step-empty'] || 'null')).toBe('');
+  });
+
   it('should handle iterations in steps', async () => {
     const runId = 'run-4';
     await db.createRun(runId, 'test-wf', {});

package/src/db/workflow-db.ts

@@ -8,8 +8,8 @@ import {
   WorkflowStatus as WorkflowStatusConst,
   type WorkflowStatusType,
 } from '../types/status';
+import { PathResolver } from '../utils/paths';
 
-// Re-export for backward compatibility - these map to the database column values
 export type RunStatus = WorkflowStatusType | 'pending';
 export type StepStatus = StepStatusType;
 
@@ -38,10 +38,44 @@ export interface StepExecution {
   usage: string | null; // JSON
 }
 
+export interface IdempotencyRecord {
+  idempotency_key: string;
+  run_id: string;
+  step_id: string;
+  status: StepStatus;
+  output: string | null; // JSON
+  error: string | null;
+  created_at: string;
+  expires_at: string | null;
+}
+
+export interface DurableTimer {
+  id: string;
+  run_id: string;
+  step_id: string;
+  timer_type: 'sleep' | 'human';
+  wake_at: string | null; // ISO datetime - null for human-triggered timers
+  created_at: string;
+  completed_at: string | null;
+}
+
+export interface CompensationRecord {
+  id: string;
+  run_id: string;
+  step_id: string;
+  compensation_step_id: string; // The ID of the compensation step definition (usually randomUUID)
+  definition: string; // JSON definition of the compensation step
+  status: StepStatus;
+  output: string | null;
+  error: string | null;
+  created_at: string;
+  completed_at: string | null;
+}
+
 export class WorkflowDb {
   private db: Database;
 
-  constructor(public readonly dbPath = '.keystone/state.db') {
+  constructor(public readonly dbPath = PathResolver.resolveDbPath()) {
     const dir = dirname(dbPath);
     if (!existsSync(dir)) {
       mkdirSync(dir, { recursive: true });
@@ -96,6 +130,16 @@ export class WorkflowDb {
     throw lastError || new Error('SQLite operation failed after retries');
   }
 
+  private formatExpiresAt(ttlSeconds?: number): string | null {
+    if (!ttlSeconds || ttlSeconds <= 0) return null;
+    const date = new Date(Date.now() + ttlSeconds * 1000);
+    return date
+      .toISOString()
+      .replace('T', ' ')
+      .replace('Z', '')
+      .replace(/\.\d{3}$/, '');
+  }
+
   private initSchema(): void {
     this.db.exec(`
       CREATE TABLE IF NOT EXISTS workflow_runs (
@@ -130,17 +174,54 @@ export class WorkflowDb {
       CREATE INDEX IF NOT EXISTS idx_steps_run ON step_executions(run_id);
       CREATE INDEX IF NOT EXISTS idx_steps_status ON step_executions(status);
       CREATE INDEX IF NOT EXISTS idx_steps_iteration ON step_executions(run_id, step_id, iteration_index);
-    `);
 
-    // Ensure usage column exists (migration for older databases)
-    // Use PRAGMA table_info to check column existence - more reliable than catching errors
-    const columns = this.db.prepare('PRAGMA table_info(step_executions)').all() as {
-      name: string;
-    }[];
-    const hasUsageColumn = columns.some((col) => col.name === 'usage');
-    if (!hasUsageColumn) {
-      this.db.exec('ALTER TABLE step_executions ADD COLUMN usage TEXT;');
-    }
+      CREATE TABLE IF NOT EXISTS idempotency_records (
+        idempotency_key TEXT PRIMARY KEY,
+        run_id TEXT NOT NULL,
+        step_id TEXT NOT NULL,
+        status TEXT NOT NULL,
+        output TEXT,
+        error TEXT,
+        created_at TEXT NOT NULL,
+        expires_at TEXT,
+        FOREIGN KEY (run_id) REFERENCES workflow_runs(id) ON DELETE CASCADE
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_idempotency_run ON idempotency_records(run_id);
+      CREATE INDEX IF NOT EXISTS idx_idempotency_expires ON idempotency_records(expires_at);
+
+      CREATE TABLE IF NOT EXISTS durable_timers (
+        id TEXT PRIMARY KEY,
+        run_id TEXT NOT NULL,
+        step_id TEXT NOT NULL,
+        timer_type TEXT NOT NULL,
+        wake_at TEXT,
+        created_at TEXT NOT NULL,
+        completed_at TEXT,
+        FOREIGN KEY (run_id) REFERENCES workflow_runs(id) ON DELETE CASCADE
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_timers_wake ON durable_timers(wake_at);
+      CREATE INDEX IF NOT EXISTS idx_timers_run ON durable_timers(run_id);
+      CREATE INDEX IF NOT EXISTS idx_timers_pending ON durable_timers(wake_at, completed_at);
+
+      CREATE TABLE IF NOT EXISTS compensations (
+        id TEXT PRIMARY KEY,
+        run_id TEXT NOT NULL,
+        step_id TEXT NOT NULL,
+        compensation_step_id TEXT NOT NULL,
+        definition TEXT NOT NULL,
+        status TEXT NOT NULL,
+        output TEXT,
+        error TEXT,
+        created_at TEXT NOT NULL,
+        completed_at TEXT,
+        FOREIGN KEY (run_id) REFERENCES workflow_runs(id) ON DELETE CASCADE
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_compensations_run ON compensations(run_id);
+      CREATE INDEX IF NOT EXISTS idx_compensations_status ON compensations(status);
+    `);
   }
 
   // ===== Workflow Runs =====
@@ -274,10 +355,10 @@ export class WorkflowDb {
       `);
       stmt.run(
         status,
-        output ? JSON.stringify(output) : null,
+        output === undefined ? null : JSON.stringify(output),
         error || null,
         new Date().toISOString(),
-        usage ? JSON.stringify(usage) : null,
+        usage === undefined ? null : JSON.stringify(usage),
         id
       );
     });
@@ -314,6 +395,21 @@ export class WorkflowDb {
     });
   }
 
+  /**
+   * Get the main execution (non-iteration) of a step
+   */
+  public async getMainStep(runId: string, stepId: string): Promise<StepExecution | null> {
+    return this.withRetry(() => {
+      const stmt = this.db.prepare(`
+        SELECT * FROM step_executions
+        WHERE run_id = ? AND step_id = ? AND iteration_index IS NULL
+        ORDER BY started_at DESC
+        LIMIT 1
+      `);
+      return stmt.get(runId, stepId) as StepExecution | null;
+    });
+  }
+
   /**
    * Get all step executions for a workflow run
    * @note Synchronous method - wrapped in sync retry logic
@@ -356,6 +452,365 @@ export class WorkflowDb {
       return stmt.get(workflowName) as WorkflowRun | null;
     });
   }
+  // ===== Idempotency Records =====
+
+  /**
+   * Get an idempotency record by key
+   * Returns null if not found or expired
+   */
+  async getIdempotencyRecord(key: string): Promise<IdempotencyRecord | null> {
+    return this.withRetry(() => {
+      const stmt = this.db.prepare(`
+        SELECT * FROM idempotency_records
+        WHERE idempotency_key = ?
+        AND (expires_at IS NULL OR datetime(expires_at) > datetime('now'))
+      `);
+      return stmt.get(key) as IdempotencyRecord | null;
+    });
+  }
+
+  /**
+   * Remove an expired idempotency record by key
+   */
+  async clearExpiredIdempotencyRecord(key: string): Promise<number> {
+    return await this.withRetry(() => {
+      const stmt = this.db.prepare(`
+        DELETE FROM idempotency_records
+        WHERE idempotency_key = ?
+        AND expires_at IS NOT NULL
+        AND datetime(expires_at) < datetime('now')
+      `);
+      const result = stmt.run(key);
+      return result.changes;
+    });
+  }
+
+  /**
+   * Insert an idempotency record only if it doesn't exist
+   */
+  async insertIdempotencyRecordIfAbsent(
+    key: string,
+    runId: string,
+    stepId: string,
+    status: StepStatus,
+    ttlSeconds?: number
+  ): Promise<boolean> {
+    return await this.withRetry(() => {
+      const expiresAt = this.formatExpiresAt(ttlSeconds);
+      const stmt = this.db.prepare(`
+        INSERT OR IGNORE INTO idempotency_records
+        (idempotency_key, run_id, step_id, status, output, error, created_at, expires_at)
+        VALUES (?, ?, ?, ?, NULL, NULL, datetime('now'), ?)
+      `);
+      const result = stmt.run(key, runId, stepId, status, expiresAt);
+      return result.changes > 0;
+    });
+  }
+
+  /**
+   * Mark an idempotency record as running if it's not already running or successful
+   */
+  async markIdempotencyRecordRunning(
+    key: string,
+    runId: string,
+    stepId: string,
+    ttlSeconds?: number
+  ): Promise<boolean> {
+    return await this.withRetry(() => {
+      const expiresAt = this.formatExpiresAt(ttlSeconds);
+      const stmt = this.db.prepare(`
+        UPDATE idempotency_records
+        SET status = ?, run_id = ?, step_id = ?, output = NULL, error = NULL, created_at = datetime('now'), expires_at = ?
+        WHERE idempotency_key = ?
+        AND status NOT IN (?, ?)
+      `);
+      const result = stmt.run(
+        StepStatusConst.RUNNING,
+        runId,
+        stepId,
+        expiresAt,
+        key,
+        StepStatusConst.RUNNING,
+        StepStatusConst.SUCCESS
+      );
+      return result.changes > 0;
+    });
+  }
+
+  /**
+   * Store an idempotency record
+   * If a record with the same key exists, it will be replaced
+   */
+  async storeIdempotencyRecord(
+    key: string,
+    runId: string,
+    stepId: string,
+    status: StepStatus,
+    output?: unknown,
+    error?: string,
+    ttlSeconds?: number
+  ): Promise<void> {
+    await this.withRetry(() => {
+      const expiresAt = this.formatExpiresAt(ttlSeconds);
+      const stmt = this.db.prepare(`
+        INSERT OR REPLACE INTO idempotency_records
+        (idempotency_key, run_id, step_id, status, output, error, created_at, expires_at)
+        VALUES (?, ?, ?, ?, ?, ?, datetime('now'), ?)
+      `);
+      stmt.run(
+        key,
+        runId,
+        stepId,
+        status,
+        output === undefined ? null : JSON.stringify(output),
+        error || null,
+        expiresAt
+      );
+    });
+  }
+
+  /**
+   * Remove expired idempotency records
+   */
+  async pruneIdempotencyRecords(): Promise<number> {
+    return await this.withRetry(() => {
+      const stmt = this.db.prepare(`
+        DELETE FROM idempotency_records
+        WHERE expires_at IS NOT NULL AND datetime(expires_at) < datetime('now')
+      `);
+      const result = stmt.run();
+      return result.changes;
+    });
+  }
+
+  /**
+   * Clear idempotency records for a specific run
+   */
+  async clearIdempotencyRecords(runId: string): Promise<number> {
+    return await this.withRetry(() => {
+      const stmt = this.db.prepare('DELETE FROM idempotency_records WHERE run_id = ?');
+      const result = stmt.run(runId);
+      return result.changes;
+    });
+  }
+
+  /**
+   * List idempotency records, optionally filtered by run ID
+   */
+  async listIdempotencyRecords(runId?: string): Promise<IdempotencyRecord[]> {
+    return this.withRetry(() => {
+      if (runId) {
+        const stmt = this.db.prepare(`
+          SELECT * FROM idempotency_records
+          WHERE run_id = ?
+          ORDER BY created_at DESC
+        `);
+        return stmt.all(runId) as IdempotencyRecord[];
+      }
+      const stmt = this.db.prepare(`
+        SELECT * FROM idempotency_records
+        ORDER BY created_at DESC
+        LIMIT 100
+      `);
+      return stmt.all() as IdempotencyRecord[];
+    });
+  }
+
+  /**
+   * Clear all idempotency records
+   */
+  async clearAllIdempotencyRecords(): Promise<number> {
+    return await this.withRetry(() => {
+      const stmt = this.db.prepare('DELETE FROM idempotency_records');
+      const result = stmt.run();
+      return result.changes;
+    });
+  }
+
+  // ===== Durable Timers =====
+
+  /**
+   * Create a durable timer for a step
+   */
+  async createTimer(
+    id: string,
+    runId: string,
+    stepId: string,
+    timerType: 'sleep' | 'human',
+    wakeAt?: string
+  ): Promise<void> {
+    await this.withRetry(() => {
+      const stmt = this.db.prepare(`
+        INSERT INTO durable_timers (id, run_id, step_id, timer_type, wake_at, created_at)
+        VALUES (?, ?, ?, ?, ?, datetime('now'))
+      `);
+      stmt.run(id, runId, stepId, timerType, wakeAt || null);
+    });
+  }
+
+  /**
+   * Get a durable timer by ID
+   */
+  async getTimer(id: string): Promise<DurableTimer | null> {
+    return this.withRetry(() => {
+      const stmt = this.db.prepare('SELECT * FROM durable_timers WHERE id = ?');
+      return stmt.get(id) as DurableTimer | null;
+    });
+  }
+
+  /**
+   * Get a durable timer by run ID and step ID
+   */
+  async getTimerByStep(runId: string, stepId: string): Promise<DurableTimer | null> {
+    return this.withRetry(() => {
+      const stmt = this.db.prepare(`
+        SELECT * FROM durable_timers
+        WHERE run_id = ? AND step_id = ? AND completed_at IS NULL
+        ORDER BY created_at DESC
+        LIMIT 1
+      `);
+      return stmt.get(runId, stepId) as DurableTimer | null;
+    });
+  }
+
+  /**
+   * Get pending timers that are ready to fire
+   * @param before Optional cutoff time (defaults to now)
+   */
+  async getPendingTimers(
+    before?: Date,
+    timerType: 'sleep' | 'human' | 'all' = 'sleep'
+  ): Promise<DurableTimer[]> {
+    return this.withRetry(() => {
+      const cutoff = (before || new Date()).toISOString();
+      const filterType = timerType !== 'all';
+      const stmt = this.db.prepare(`
+        SELECT * FROM durable_timers
+        WHERE completed_at IS NULL
+        AND (wake_at IS NULL OR wake_at <= ?)
+        ${filterType ? 'AND timer_type = ?' : ''}
+        ORDER BY wake_at ASC
+      `);
+      return (filterType ? stmt.all(cutoff, timerType) : stmt.all(cutoff)) as DurableTimer[];
+    });
+  }
+
+  /**
+   * Complete a durable timer
+   */
+  async completeTimer(id: string): Promise<void> {
+    await this.withRetry(() => {
+      const stmt = this.db.prepare(`
+        UPDATE durable_timers
+        SET completed_at = datetime('now')
+        WHERE id = ?
+      `);
+      stmt.run(id);
+    });
+  }
+
+  /**
+   * List all timers, optionally filtered by run ID
+   */
+  async listTimers(runId?: string, limit = 50): Promise<DurableTimer[]> {
+    return this.withRetry(() => {
+      if (runId) {
+        const stmt = this.db.prepare(`
+          SELECT * FROM durable_timers
+          WHERE run_id = ?
+          ORDER BY created_at DESC
+          LIMIT ?
+        `);
+        return stmt.all(runId, limit) as DurableTimer[];
+      }
+      const stmt = this.db.prepare(`
+        SELECT * FROM durable_timers
+        ORDER BY created_at DESC
+        LIMIT ?
+      `);
+      return stmt.all(limit) as DurableTimer[];
+    });
+  }
+
+  /**
+   * Clear timers for a specific run or all timers
+   */
+  async clearTimers(runId?: string): Promise<number> {
+    return await this.withRetry(() => {
+      if (runId) {
+        const stmt = this.db.prepare('DELETE FROM durable_timers WHERE run_id = ?');
+        const result = stmt.run(runId);
+        return result.changes;
+      }
+      const stmt = this.db.prepare('DELETE FROM durable_timers');
+      const result = stmt.run();
+      return result.changes;
+    });
+  }
+
+  // ===== Compensations =====
+
+  async registerCompensation(
+    id: string,
+    runId: string,
+    stepId: string,
+    compensationStepId: string,
+    definition: string
+  ): Promise<void> {
+    await this.withRetry(() => {
+      const stmt = this.db.prepare(`
+        INSERT INTO compensations (id, run_id, step_id, compensation_step_id, definition, status, created_at)
+        VALUES (?, ?, ?, ?, ?, 'pending', datetime('now'))
+      `);
+      stmt.run(id, runId, stepId, compensationStepId, definition);
+    });
+  }
+
+  async updateCompensationStatus(
+    id: string,
+    status: StepStatus,
+    output?: unknown,
+    error?: string
+  ): Promise<void> {
+    await this.withRetry(() => {
+      const stmt = this.db.prepare(`
+        UPDATE compensations
+        SET status = ?, output = ?, error = ?, completed_at = ?
+        WHERE id = ?
+      `);
+      const completedAt =
+        status === 'success' || status === 'failed' ? new Date().toISOString() : null;
+      stmt.run(
+        status,
+        output === undefined ? null : JSON.stringify(output),
+        error || null,
+        completedAt,
+        id
+      );
+    });
+  }
+
+  async getPendingCompensations(runId: string): Promise<CompensationRecord[]> {
+    return this.withRetry(() => {
+      const stmt = this.db.prepare(`
+        SELECT * FROM compensations
+        WHERE run_id = ? AND status = 'pending'
+        ORDER BY created_at DESC, rowid DESC
+      `);
+      return stmt.all(runId) as CompensationRecord[];
+    });
+  }
+
+  async getAllCompensations(runId: string): Promise<CompensationRecord[]> {
+    return this.withRetry(() => {
+      const stmt = this.db.prepare(`
+        SELECT * FROM compensations
+        WHERE run_id = ?
+        ORDER BY created_at DESC, rowid DESC
+      `);
+      return stmt.all(runId) as CompensationRecord[];
+    });
+  }
 
   close(): void {
     this.db.close();