keystone-cli 0.5.1 → 0.6.1

package/src/db/memory-db.test.ts

@@ -0,0 +1,54 @@
+import { afterAll, describe, expect, test } from 'bun:test';
+import * as fs from 'node:fs';
+import { MemoryDb } from './memory-db';
+
+const TEST_DB = '.keystone/test-memory.db';
+
+describe('MemoryDb', () => {
+  // Clean up previous runs
+  if (fs.existsSync(TEST_DB)) {
+    fs.unlinkSync(TEST_DB);
+  }
+
+  const db = new MemoryDb(TEST_DB);
+
+  afterAll(() => {
+    db.close();
+    if (fs.existsSync(TEST_DB)) {
+      fs.unlinkSync(TEST_DB);
+    }
+  });
+
+  test('should initialize and store embedding', async () => {
+    const id = await db.store('hello world', Array(384).fill(0.1), { tag: 'test' });
+    expect(id).toBeDefined();
+    expect(typeof id).toBe('string');
+  });
+
+  test('should search and retrieve result', async () => {
+    // Store another item to search for
+    await db.store('search target', Array(384).fill(0.9), { tag: 'target' });
+
+    const results = await db.search(Array(384).fill(0.9), 1);
+    expect(results.length).toBe(1);
+    expect(results[0].text).toBe('search target');
+    expect(results[0].metadata).toEqual({ tag: 'target' });
+  });
+
+  test('should fail gracefully with invalid dimensions', async () => {
+    // sqlite-vec requires fixed dimensions (384 defined in schema)
+    // bun:sqlite usually throws an error for constraint violations
+    let error: unknown;
+    try {
+      await db.store('fail', Array(10).fill(0));
+    } catch (e) {
+      error = e;
+    }
+    if (error) {
+      expect(error).toBeDefined();
+    } else {
+      const results = await db.search(Array(384).fill(0), 1);
+      expect(Array.isArray(results)).toBe(true);
+    }
+  });
+});

package/src/db/memory-db.ts

@@ -0,0 +1,128 @@
+import type { Database } from 'bun:sqlite';
+import { randomUUID } from 'node:crypto';
+import { existsSync, mkdirSync } from 'node:fs';
+import { dirname } from 'node:path';
+import * as sqliteVec from 'sqlite-vec';
+import './sqlite-setup.ts';
+
+export interface MemoryEntry {
+  id: string;
+  text: string;
+  metadata: Record<string, unknown>;
+  distance?: number;
+}
+
+export class MemoryDb {
+  private db: Database;
+  // Cache connections by path to avoid reloading extensions
+  private static connectionCache = new Map<string, { db: Database; refCount: number }>();
+
+  constructor(public readonly dbPath = '.keystone/memory.db') {
+    const cached = MemoryDb.connectionCache.get(dbPath);
+    if (cached) {
+      cached.refCount++;
+      this.db = cached.db;
+    } else {
+      const { Database } = require('bun:sqlite');
+      const dir = dirname(dbPath);
+      if (!existsSync(dir)) {
+        mkdirSync(dir, { recursive: true });
+      }
+      this.db = new Database(dbPath, { create: true });
+
+      // Load sqlite-vec extension
+      const extensionPath = sqliteVec.getLoadablePath();
+      this.db.loadExtension(extensionPath);
+
+      this.initSchema();
+
+      MemoryDb.connectionCache.set(dbPath, { db: this.db, refCount: 1 });
+    }
+  }
+
+  private initSchema(): void {
+    this.db.run(`
+      CREATE VIRTUAL TABLE IF NOT EXISTS vec_memory USING vec0(
+        id TEXT PRIMARY KEY,
+        embedding FLOAT[384]
+      );
+    `);
+
+    this.db.run(`
+      CREATE TABLE IF NOT EXISTS memory_metadata (
+        id TEXT PRIMARY KEY,
+        text TEXT NOT NULL,
+        metadata TEXT NOT NULL,
+        created_at TEXT NOT NULL
+      );
+    `);
+  }
+
+  async store(
+    text: string,
+    embedding: number[],
+    metadata: Record<string, unknown> = {}
+  ): Promise<string> {
+    const id = randomUUID();
+    const createdAt = new Date().toISOString();
+
+    // bun:sqlite transaction wrapper ensures atomicity synchronously
+    const insertTransaction = this.db.transaction(() => {
+      this.db.run('INSERT INTO vec_memory(id, embedding) VALUES (?, ?)', [
+        id,
+        new Float32Array(embedding),
+      ]);
+
+      this.db.run(
+        'INSERT INTO memory_metadata(id, text, metadata, created_at) VALUES (?, ?, ?, ?)',
+        [id, text, JSON.stringify(metadata), createdAt]
+      );
+    });
+
+    insertTransaction();
+    return id;
+  }
+
+  async search(embedding: number[], limit = 5): Promise<MemoryEntry[]> {
+    const query = `
+      SELECT 
+        v.id, 
+        v.distance,
+        m.text,
+        m.metadata
+      FROM vec_memory v
+      JOIN memory_metadata m ON v.id = m.id
+      WHERE embedding MATCH ? AND k = ?
+      ORDER BY distance
+    `;
+
+    // bun:sqlite is synchronous
+    const rows = this.db.prepare(query).all(new Float32Array(embedding), limit) as {
+      id: string;
+      distance: number;
+      text: string;
+      metadata: string;
+    }[];
+
+    return rows.map((row) => ({
+      id: row.id,
+      distance: row.distance,
+      text: row.text,
+      metadata: JSON.parse(row.metadata),
+    }));
+  }
+
+  close(): void {
+    const cached = MemoryDb.connectionCache.get(this.dbPath);
+    if (cached) {
+      cached.refCount--;
+      if (cached.refCount <= 0) {
+        cached.db.close();
+        MemoryDb.connectionCache.delete(this.dbPath);
+      }
+    } else {
+      // Fallback if not in cache for some reason
+      this.db.close();
+    }
+  }
+}

package/src/db/sqlite-setup.test.ts

@@ -0,0 +1,47 @@
+import { afterEach, describe, expect, it, mock, spyOn } from 'bun:test';
+import type { Logger } from '../utils/logger';
+import { setupSqlite } from './sqlite-setup';
+
+describe('setupSqlite', () => {
+  const originalPlatform = process.platform;
+
+  afterEach(() => {
+    Object.defineProperty(process, 'platform', {
+      value: originalPlatform,
+    });
+  });
+
+  it('does nothing on non-darwin platforms', () => {
+    Object.defineProperty(process, 'platform', { value: 'linux' });
+    const logger: Logger = {
+      log: mock(() => {}),
+      warn: mock(() => {}),
+      error: mock(() => {}),
+      info: mock(() => {}),
+    };
+    setupSqlite(logger);
+    expect(logger.log).not.toHaveBeenCalled();
+    expect(logger.warn).not.toHaveBeenCalled();
+  });
+
+  it('logs warning if no custom sqlite found on darwin', () => {
+    Object.defineProperty(process, 'platform', { value: 'darwin' });
+    const logger: Logger = {
+      log: mock(() => {}),
+      warn: mock(() => {}),
+      error: mock(() => {}),
+      info: mock(() => {}),
+    };
+
+    // Mock Bun.spawnSync for brew
+    const spawnSpy = spyOn(Bun, 'spawnSync').mockImplementation(
+      () => ({ success: false }) as unknown as ReturnType<typeof Bun.spawnSync>
+    );
+
+    try {
+      setupSqlite(logger);
+    } finally {
+      spawnSpy.mockRestore();
+    }
+  });
+});

package/src/db/sqlite-setup.ts

@@ -0,0 +1,49 @@
+import { ConsoleLogger, type Logger } from '../utils/logger.ts';
+
+export function setupSqlite(logger: Logger = new ConsoleLogger()) {
+  // macOS typically comes with a system SQLite that doesn't support extensions
+  // We need to try to load a custom one (e.g. from Homebrew) if on macOS
+  if (process.platform === 'darwin') {
+    try {
+      const { Database } = require('bun:sqlite');
+      const { existsSync } = require('node:fs');
+
+      // Common Homebrew paths for SQLite
+      const paths = [
+        '/opt/homebrew/opt/sqlite/lib/libsqlite3.dylib',
+        '/usr/local/opt/sqlite/lib/libsqlite3.dylib',
+        // Fallback to checking brew prefix if available
+      ];
+
+      // Try to find brew prefix dynamically if possible
+      try {
+        const proc = Bun.spawnSync(['brew', '--prefix', 'sqlite'], {
+          stderr: 'ignore',
+        });
+        if (proc.success) {
+          const prefix = proc.stdout.toString().trim();
+          paths.unshift(`${prefix}/lib/libsqlite3.dylib`);
+        }
+      } catch {
+        // Brew might not be installed or in path
+      }
+
+      for (const libPath of paths) {
+        if (existsSync(libPath)) {
+          logger.log(`[SqliteSetup] Using custom SQLite library: ${libPath}`);
+          Database.setCustomSQLite(libPath);
+          return;
+        }
+      }
+
+      logger.warn(
+        '[SqliteSetup] Warning: Could not find Homebrew SQLite. Extension loading might fail.'
+      );
+    } catch (error) {
+      logger.warn(`[SqliteSetup] Failed to set custom SQLite: ${error}`);
+    }
+  }
+}
+
+// Run setup immediately when imported
+setupSqlite();

package/src/db/workflow-db.test.ts

@@ -18,7 +18,7 @@ describe('WorkflowDb', () => {
   it('should create and retrieve a run', async () => {
     const runId = 'run-1';
     await db.createRun(runId, 'test-wf', { input: 1 });
-    const run = db.getRun(runId);
+    const run = await db.getRun(runId);
     expect(run).toBeDefined();
     expect(run?.workflow_name).toBe('test-wf');
     expect(JSON.parse(run?.inputs || '{}')).toEqual({ input: 1 });
@@ -27,9 +27,9 @@ describe('WorkflowDb', () => {
   it('should update run status', async () => {
     const runId = 'run-2';
     await db.createRun(runId, 'test-wf', {});
-    await db.updateRunStatus(runId, 'completed', { result: 'ok' });
-    const run = db.getRun(runId);
-    expect(run?.status).toBe('completed');
+    await db.updateRunStatus(runId, 'success', { result: 'ok' });
+    const run = await db.getRun(runId);
+    expect(run?.status).toBe('success');
     expect(JSON.parse(run?.outputs || '{}')).toEqual({ result: 'ok' });
   });
 
@@ -41,7 +41,7 @@ describe('WorkflowDb', () => {
     await db.startStep('exec-1');
     await db.completeStep('exec-1', 'success', { out: 'val' });
 
-    const steps = db.getStepsByRun(runId);
+    const steps = await db.getStepsByRun(runId);
     expect(steps).toHaveLength(1);
     expect(steps[0].step_id).toBe(stepId);
     expect(steps[0].status).toBe('success');
@@ -53,11 +53,11 @@ describe('WorkflowDb', () => {
     await db.createStep('exec-i0', runId, 'loop', 0);
     await db.createStep('exec-i1', runId, 'loop', 1);
 
-    const step0 = db.getStepByIteration(runId, 'loop', 0);
+    const step0 = await db.getStepByIteration(runId, 'loop', 0);
     expect(step0).toBeDefined();
     expect(step0?.iteration_index).toBe(0);
 
-    const steps = db.getStepsByRun(runId);
+    const steps = await db.getStepsByRun(runId);
     expect(steps).toHaveLength(2);
   });
 
@@ -68,14 +68,14 @@ describe('WorkflowDb', () => {
     await db.incrementRetry('exec-r');
     await db.incrementRetry('exec-r');
 
-    const steps = db.getStepsByRun(runId);
+    const steps = await db.getStepsByRun(runId);
     expect(steps[0].retry_count).toBe(2);
   });
 
   it('should list runs with limit', async () => {
     await db.createRun('run-l1', 'wf', {});
     await db.createRun('run-l2', 'wf', {});
-    const runs = db.listRuns(1);
+    const runs = await db.listRuns(1);
     expect(runs).toHaveLength(1);
   });
 
@@ -93,7 +93,38 @@ describe('WorkflowDb', () => {
     const deleted = await db.pruneRuns(30);
     expect(deleted).toBe(0);
 
-    const run = db.getRun(runId);
+    const run = await db.getRun(runId);
     expect(run).toBeDefined();
   });
+
+  it('should retrieve successful runs', async () => {
+    // pending run
+    await db.createRun('run-s1', 'my-wf', { i: 1 });
+
+    // successful run
+    await db.createRun('run-s2', 'my-wf', { i: 2 });
+    await db.updateRunStatus('run-s2', 'success', { o: 2 });
+    await new Promise((r) => setTimeout(r, 10));
+
+    // failed run
+    await db.createRun('run-s3', 'my-wf', { i: 3 });
+    await db.updateRunStatus('run-s3', 'failed', undefined, 'err');
+    await new Promise((r) => setTimeout(r, 10));
+
+    // another successful run
+    await db.createRun('run-s4', 'my-wf', { i: 4 });
+    await db.updateRunStatus('run-s4', 'success', { o: 4 });
+
+    const runs = await db.getSuccessfulRuns('my-wf', 5);
+    expect(runs).toHaveLength(2);
+    // ordered by started_at DESC, so run-s4 then run-s2
+    expect(runs[0].id).toBe('run-s4');
+    expect(JSON.parse(runs[0].outputs || '{}')).toEqual({ o: 4 });
+    expect(runs[1].id).toBe('run-s2');
+
+    // Limit check
+    const limitedOne = await db.getSuccessfulRuns('my-wf', 1);
+    expect(limitedOne).toHaveLength(1);
+    expect(limitedOne[0].id).toBe('run-s4');
+  });
 });

package/src/db/workflow-db.ts

@@ -1,4 +1,7 @@
 import { Database } from 'bun:sqlite';
+import { existsSync, mkdirSync } from 'node:fs';
+import { dirname } from 'node:path';
+import './sqlite-setup.ts';
 import {
   StepStatus as StepStatusConst,
   type StepStatusType,
@@ -7,7 +10,7 @@ import {
 } from '../types/status';
 
 // Re-export for backward compatibility - these map to the database column values
-export type RunStatus = WorkflowStatusType | 'pending' | 'completed';
+export type RunStatus = WorkflowStatusType | 'pending';
 export type StepStatus = StepStatusType;
 
 export interface WorkflowRun {
@@ -39,6 +42,10 @@ export class WorkflowDb {
   private db: Database;
 
   constructor(public readonly dbPath = '.keystone/state.db') {
+    const dir = dirname(dbPath);
+    if (!existsSync(dir)) {
+      mkdirSync(dir, { recursive: true });
+    }
     this.db = new Database(dbPath, { create: true });
     this.db.exec('PRAGMA journal_mode = WAL;'); // Write-ahead logging
     this.db.exec('PRAGMA foreign_keys = ON;'); // Enable foreign key enforcement
@@ -124,6 +131,16 @@ export class WorkflowDb {
       CREATE INDEX IF NOT EXISTS idx_steps_status ON step_executions(status);
       CREATE INDEX IF NOT EXISTS idx_steps_iteration ON step_executions(run_id, step_id, iteration_index);
     `);
+
+    // Ensure usage column exists (migration for older databases)
+    // Use PRAGMA table_info to check column existence - more reliable than catching errors
+    const columns = this.db.prepare('PRAGMA table_info(step_executions)').all() as {
+      name: string;
+    }[];
+    const hasUsageColumn = columns.some((col) => col.name === 'usage');
+    if (!hasUsageColumn) {
+      this.db.exec('ALTER TABLE step_executions ADD COLUMN usage TEXT;');
+    }
   }
 
   // ===== Workflow Runs =====
@@ -155,23 +172,40 @@ export class WorkflowDb {
         WHERE id = ?
       `);
       const completedAt =
-        status === 'completed' || status === 'failed' ? new Date().toISOString() : null;
+        status === 'success' || status === 'failed' ? new Date().toISOString() : null;
       stmt.run(status, outputs ? JSON.stringify(outputs) : null, error || null, completedAt, id);
     });
   }
 
-  getRun(id: string): WorkflowRun | null {
-    const stmt = this.db.prepare('SELECT * FROM workflow_runs WHERE id = ?');
-    return stmt.get(id) as WorkflowRun | null;
+  /**
+   * Helper for synchronous retries on SQLITE_BUSY
+   * Since bun:sqlite is synchronous, we use a busy-wait loop with sleep
+   */
+
+  /**
+   * Get a workflow run by ID
+   * @note Synchronous method - wrapped in sync retry logic
+   */
+  async getRun(id: string): Promise<WorkflowRun | null> {
+    return this.withRetry(() => {
+      const stmt = this.db.prepare('SELECT * FROM workflow_runs WHERE id = ?');
+      return stmt.get(id) as WorkflowRun | null;
+    });
   }
 
-  listRuns(limit = 50): WorkflowRun[] {
-    const stmt = this.db.prepare(`
-      SELECT * FROM workflow_runs
-      ORDER BY started_at DESC
-      LIMIT ?
-    `);
-    return stmt.all(limit) as WorkflowRun[];
+  /**
+   * List recent workflow runs
+   * @note Synchronous method - wrapped in sync retry logic
+   */
+  async listRuns(limit = 50): Promise<WorkflowRun[]> {
+    return this.withRetry(() => {
+      const stmt = this.db.prepare(`
+        SELECT * FROM workflow_runs
+        ORDER BY started_at DESC
+        LIMIT ?
+      `);
+      return stmt.all(limit) as WorkflowRun[];
+    });
   }
 
   /**
@@ -260,24 +294,52 @@ export class WorkflowDb {
     });
   }
 
-  getStepByIteration(runId: string, stepId: string, iterationIndex: number): StepExecution | null {
-    const stmt = this.db.prepare(`
-      SELECT * FROM step_executions
-      WHERE run_id = ? AND step_id = ? AND iteration_index = ?
-      ORDER BY started_at DESC
-      LIMIT 1
-    `);
-    return stmt.get(runId, stepId, iterationIndex) as StepExecution | null;
+  /**
+   * Get a step execution by run ID, step ID, and iteration index
+   * @note Synchronous method - wrapped in sync retry logic
+   */
+  async getStepByIteration(
+    runId: string,
+    stepId: string,
+    iterationIndex: number
+  ): Promise<StepExecution | null> {
+    return this.withRetry(() => {
+      const stmt = this.db.prepare(`
+        SELECT * FROM step_executions
+        WHERE run_id = ? AND step_id = ? AND iteration_index = ?
+        ORDER BY started_at DESC
+        LIMIT 1
+      `);
+      return stmt.get(runId, stepId, iterationIndex) as StepExecution | null;
+    });
   }
 
-  getStepsByRun(runId: string, limit = -1, offset = 0): StepExecution[] {
-    const stmt = this.db.prepare(`
-      SELECT * FROM step_executions
-      WHERE run_id = ?
-      ORDER BY started_at ASC, iteration_index ASC, rowid ASC
-      LIMIT ? OFFSET ?
-    `);
-    return stmt.all(runId, limit, offset) as StepExecution[];
+  /**
+   * Get all step executions for a workflow run
+   * @note Synchronous method - wrapped in sync retry logic
+   */
+  async getStepsByRun(runId: string, limit = -1, offset = 0): Promise<StepExecution[]> {
+    return this.withRetry(() => {
+      const stmt = this.db.prepare(`
+        SELECT * FROM step_executions
+        WHERE run_id = ?
+        ORDER BY started_at ASC, iteration_index ASC, rowid ASC
+        LIMIT ? OFFSET ?
+      `);
+      return stmt.all(runId, limit, offset) as StepExecution[];
+    });
+  }
+
+  async getSuccessfulRuns(workflowName: string, limit = 3): Promise<WorkflowRun[]> {
+    return await this.withRetry(() => {
+      const stmt = this.db.prepare(`
+        SELECT * FROM workflow_runs
+        WHERE workflow_name = ? AND status = 'success'
+        ORDER BY started_at DESC
+        LIMIT ?
+      `);
+      return stmt.all(workflowName, limit) as WorkflowRun[];
+    });
   }
 
   close(): void {

package/src/expression/evaluator.test.ts

@@ -303,4 +303,23 @@ describe('ExpressionEvaluator', () => {
     const contextWithNull = { ...context, nullVal: null };
     expect(ExpressionEvaluator.evaluate('${{ nullVal }}', contextWithNull)).toBe(null);
   });
+
+  test('should allow plain strings longer than 10k', () => {
+    const longString = 'a'.repeat(11000);
+    expect(ExpressionEvaluator.evaluate(longString, context)).toBe(longString);
+  });
+
+  test('should still enforce 10k limit for strings with expressions', () => {
+    const longStringWithExpr = `${'a'.repeat(10000)}\${{ inputs.name }}`;
+    expect(() => ExpressionEvaluator.evaluate(longStringWithExpr, context)).toThrow(
+      /Template with expressions exceeds maximum length/
+    );
+  });
+
+  test('should enforce 1MB limit for plain strings', () => {
+    const wayTooLongString = 'a'.repeat(1000001);
+    expect(() => ExpressionEvaluator.evaluate(wayTooLongString, context)).toThrow(
+      /Plain string exceeds maximum length/
+    );
+  });
 });