keystone-cli 0.5.0 → 0.6.0

package/README.md

@@ -25,6 +25,8 @@ Keystone allows you to define complex automation workflows using a simple YAML s
 - 🛠️ **Extensible:** Support for shell, file, HTTP request, LLM, and sub-workflow steps.
 - 🔌 **MCP Support:** Integrated Model Context Protocol server.
 - 🛡️ **Secret Redaction:** Automatically redacts environment variables and secrets from logs and outputs.
+- 🧠 **Semantic Memory:** Store and retrieve step outputs using vector embeddings/RAG.
+- 🎯 **Prompt Optimization:** Automatically optimize prompts using iterative evaluation (DSPy-style).
 
 ---
 
@@ -281,10 +283,26 @@ Keystone supports several specialized step types:
 - `script`: Run arbitrary JavaScript in a sandbox. On Bun, uses `node:vm` (since `isolated-vm` requires V8).
   - ⚠️ **Security Note:** The `node:vm` sandbox is not secure against malicious code. Only run scripts from trusted sources.
 - `sleep`: Pause execution for a specified duration.
+- `memory`: Store or retrieve information from the semantic memory vector database.
 
-All steps support common features like `needs` (dependencies), `if` (conditionals), `retry`, `timeout`, `foreach` (parallel iteration), `concurrency` (max parallel items for foreach), and `transform` (post-process output using expressions).
+All steps support common features like `needs` (dependencies), `if` (conditionals), `retry`, `timeout`, `foreach` (parallel iteration), `concurrency` (max parallel items for foreach), `transform` (post-process output using expressions), `learn` (auto-index for few-shot), and `reflexion` (self-correction loop).
 
-Workflows also support a top-level `concurrency` field to limit how many steps can run in parallel across the entire workflow.
+Workflows also support a top-level `concurrency` field to limit how many steps can run in parallel across the entire workflow. This must be a positive integer.
+
+### Self-Healing Steps
+Steps can be configured to automatically recover from failures using an LLM agent.
+
+```yaml
+- id: build
+  type: shell
+  run: bun build
+  auto_heal:
+    agent: debugger_agent
+    maxAttempts: 3
+    model: gpt-4o # Optional override
+```
+
+When a step fails, the specified agent is invoked with the error details. The agent proposes a fix (e.g., a corrected command), and the step is automatically retried.
 
 #### Example: Transform & Foreach Concurrency
 ```yaml
@@ -297,13 +315,14 @@ Workflows also support a top-level `concurrency` field to limit how many steps c
 - id: process_files
   type: shell
   foreach: ${{ steps.list_files.output }}
-  concurrency: 5 # Process 5 files at a time
+  concurrency: 5 # Process 5 files at a time (must be a positive integer)
   run: echo "Processing ${{ item }}"
 
 #### Example: Script Step
 ```yaml
 - id: calculate
   type: script
+  allowInsecure: true
   run: |
     const data = context.steps.fetch_data.output;
     return data.map(i => i.value * 2).reduce((a, b) => a + b, 0);
@@ -427,7 +446,8 @@ In these examples, the agent will have access to all tools provided by the MCP s
 | Command | Description |
 | :--- | :--- |
 | `init` | Initialize a new Keystone project |
-| `run <workflow>` | Execute a workflow (use `-i key=val` for inputs, `--dry-run` to test) |
+| `run <workflow>` | Execute a workflow (use `-i key=val` for inputs, `--dry-run` to test, `--debug` for REPL) |
+| `optimize <workflow>` | Optimize a specific step in a workflow (requires --target) |
 | `resume <run_id>` | Resume a failed or paused workflow |
 | `validate [path]` | Check workflow files for errors |
 | `workflows` | List available workflows |
@@ -442,11 +462,38 @@ In these examples, the agent will have access to all tools provided by the MCP s
 | `mcp start` | Start the Keystone MCP server |
 | `mcp login <server>` | Login to a remote MCP server |
 | `completion [shell]` | Generate shell completion script (zsh, bash) |
-| `prune [--days N]` | Cleanup old run data from the database |
+| `maintenance [--days N]` | Perform database maintenance (prune old runs and vacuum) |
 
 ---
-
-## 📂 Project Structure
+ 
+ ## 🛡️ Security
+ 
+ ### Shell Execution
+ By default, Keystone analyzes shell commands for potentially dangerous patterns (like shell injection, `rm -rf`, piped commands). If a risk is detected:
+ - In interactive mode, the user is prompted for confirmation.
+ - In non-interactive mode, the step is suspended or failed.
+ 
+ You can bypass this check if you trust the command:
+ ```yaml
+ - id: deploy
+   type: shell
+   run: ./deploy.sh ${{ inputs.env }}
+   allowInsecure: true
+ ```
+ 
+ ### Expression Safety
+ Expressions `${{ }}` are evaluated using a safe AST parser (`jsep`) which:
+ - Prevents arbitrary code execution (no `eval` or `Function`).
+ - Whitelists safe global objects (`Math`, `JSON`, `Date`, etc.).
+ - Blocks access to sensitive properties (`constructor`, `__proto__`).
+ - Enforces a maximum template length to prevent ReDoS attacks.
+ 
+ ### Script Sandboxing
+ The `script` step uses Node.js `vm` module. While it provides isolation for variables, it is **not a security boundary** for malicious code. Only run scripts from trusted sources.
+ 
+ ---
+ 
+ ## 📂 Project Structure
 
 - `src/db/`: SQLite persistence layer.
 - `src/runner/`: The core execution engine, handles parallelization and retries.
@@ -460,4 +507,4 @@ In these examples, the agent will have access to all tools provided by the MCP s
 
 ## 📄 License
 
-MIT
+MIT

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "keystone-cli",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "description": "A local-first, declarative, agentic workflow orchestrator built on Bun",
   "type": "module",
   "bin": {
@@ -25,7 +25,8 @@
   "dependencies": {
     "@jsep-plugin/arrow": "^1.0.6",
     "@jsep-plugin/object": "^1.2.2",
-    "@types/react": "^19.2.7",
+    "@types/react": "^19.0.0",
+    "@xenova/transformers": "^2.17.2",
     "commander": "^12.1.0",
     "dagre": "^0.8.5",
     "ink": "^6.5.1",
@@ -33,7 +34,8 @@
     "ink-spinner": "^5.0.0",
     "js-yaml": "^4.1.0",
     "jsep": "^1.4.0",
-    "react": "^19.2.3",
+    "react": "^19.0.0",
+    "sqlite-vec": "0.1.6",
     "zod": "^3.23.8"
   },
   "devDependencies": {

package/src/cli.ts

@@ -12,6 +12,7 @@ import scaffoldWorkflow from './templates/scaffold-feature.yaml' with { type: 't
 import { WorkflowDb } from './db/workflow-db.ts';
 import { WorkflowParser } from './parser/workflow-parser.ts';
 import { ConfigLoader } from './utils/config-loader.ts';
+import { ConsoleLogger } from './utils/logger.ts';
 import { generateMermaidGraph, renderWorkflowAsAscii } from './utils/mermaid.ts';
 import { WorkflowRegistry } from './utils/workflow-registry.ts';
 
@@ -226,6 +227,7 @@ program
   .argument('<workflow>', 'Workflow name or path to workflow file')
   .option('-i, --input <key=value...>', 'Input values')
   .option('--dry-run', 'Show what would be executed without actually running it')
+  .option('--debug', 'Enable interactive debug mode on failure')
   .action(async (workflowPath, options) => {
     // Parse inputs
     const inputs: Record<string, unknown> = {};
@@ -250,25 +252,15 @@ program
       const resolvedPath = WorkflowRegistry.resolvePath(workflowPath);
       const workflow = WorkflowParser.loadWorkflow(resolvedPath);
 
-      // Auto-prune old runs
-      try {
-        const config = ConfigLoader.load();
-        const db = new WorkflowDb();
-        const deleted = await db.pruneRuns(config.storage.retention_days);
-        if (deleted > 0) {
-          await db.vacuum();
-        }
-        db.close();
-      } catch (error) {
-        // Non-fatal
-      }
-
       // Import WorkflowRunner dynamically
       const { WorkflowRunner } = await import('./runner/workflow-runner.ts');
+      const logger = new ConsoleLogger();
       const runner = new WorkflowRunner(workflow, {
         inputs,
         workflowDir: dirname(resolvedPath),
         dryRun: !!options.dryRun,
+        debug: !!options.debug,
+        logger,
       });
 
       const outputs = await runner.run();
@@ -287,6 +279,8 @@ program
     }
   });
 
+// ... (optimize command remains here) ...
+
 // ===== keystone resume =====
 program
   .command('resume')
@@ -295,21 +289,10 @@ program
   .option('-w, --workflow <path>', 'Path to workflow file (auto-detected if not specified)')
   .action(async (runId, options) => {
     try {
-      const config = ConfigLoader.load();
       const db = new WorkflowDb();
 
-      // Auto-prune old runs
-      try {
-        const deleted = await db.pruneRuns(config.storage.retention_days);
-        if (deleted > 0) {
-          await db.vacuum();
-        }
-      } catch (error) {
-        // Non-fatal
-      }
-
       // Load run from database to get workflow name
-      const run = db.getRun(runId);
+      const run = await db.getRun(runId);
 
       if (!run) {
         console.error(`✗ Run not found: ${runId}`);
@@ -344,9 +327,11 @@ program
 
       // Import WorkflowRunner dynamically
       const { WorkflowRunner } = await import('./runner/workflow-runner.ts');
+      const logger = new ConsoleLogger();
       const runner = new WorkflowRunner(workflow, {
         resumeRunId: runId,
         workflowDir: dirname(workflowPath),
+        logger,
       });
 
       const outputs = await runner.run();
@@ -362,156 +347,13 @@ program
     }
   });
 
-// ===== keystone workflows =====
-program
-  .command('workflows')
-  .description('List available workflows')
-  .action(() => {
-    try {
-      const workflows = WorkflowRegistry.listWorkflows();
-      if (workflows.length === 0) {
-        console.log('No workflows found.');
-        return;
-      }
-
-      console.log('\nAvailable workflows:\n');
-      for (const w of workflows) {
-        const description = w.description ? ` - ${w.description}` : '';
-        console.log(`  ${w.name.padEnd(25)}${description}`);
-      }
-      console.log();
-    } catch (error) {
-      console.error('✗ Failed to list workflows:', error instanceof Error ? error.message : error);
-      process.exit(1);
-    }
-  });
-
-// ===== keystone history =====
-program
-  .command('history')
-  .description('List recent workflow runs')
-  .option('-n, --limit <number>', 'Number of runs to show', '20')
-  .action((options) => {
-    try {
-      const db = new WorkflowDb();
-      const runs = db.listRuns(Number.parseInt(options.limit));
-
-      if (runs.length === 0) {
-        console.log('No workflow runs found.');
-        return;
-      }
-
-      console.log('\nRecent workflow runs:\n');
-      for (const run of runs) {
-        const status = run.status.toUpperCase().padEnd(10);
-        const date = new Date(run.started_at).toLocaleString();
-        console.log(
-          `${run.id.substring(0, 8)}  ${status}  ${run.workflow_name.padEnd(20)}  ${date}`
-        );
-      }
-
-      db.close();
-    } catch (error) {
-      console.error('✗ Failed to list runs:', error instanceof Error ? error.message : error);
-      process.exit(1);
-    }
-  });
-
-// ===== keystone logs =====
-program
-  .command('logs')
-  .description('Show logs for a workflow run')
-  .argument('<run_id>', 'Run ID')
-  .option('-v, --verbose', 'Show full output without truncation')
-  .action((runId, options) => {
-    try {
-      const db = new WorkflowDb();
-      const run = db.getRun(runId);
-
-      if (!run) {
-        console.error(`✗ Run not found: ${runId}`);
-        process.exit(1);
-      }
-
-      console.log(`\n📋 Workflow: ${run.workflow_name}`);
-      console.log(`Status: ${run.status}`);
-      console.log(`Started: ${new Date(run.started_at).toLocaleString()}`);
-      if (run.completed_at) {
-        console.log(`Completed: ${new Date(run.completed_at).toLocaleString()}`);
-      }
-      if (run.error) {
-        console.log(`\n❌ Error: ${run.error}`);
-      }
-
-      const steps = db.getStepsByRun(runId);
-      if (steps.length > 0) {
-        console.log('\nSteps:');
-        for (const step of steps) {
-          const statusColors: Record<string, string> = {
-            success: '\x1b[32m', // green
-            failed: '\x1b[31m', // red
-            pending: '\x1b[33m', // yellow
-            skipped: '\x1b[90m', // gray
-            suspended: '\x1b[35m', // magenta
-          };
-          const RESET = '\x1b[0m';
-          const color = statusColors[step.status] || '';
-          const status = `${color}${step.status.toUpperCase().padEnd(10)}${RESET}`;
-          const iteration = step.iteration_index !== null ? ` [${step.iteration_index}]` : '';
-          console.log(`  ${(step.step_id + iteration).padEnd(25)}  ${status}`);
-
-          // Show error if present
-          if (step.error) {
-            console.log(`    ❌ Error: ${step.error}`);
-          }
-
-          // Show output if present
-          if (step.output) {
-            try {
-              const output = JSON.parse(step.output);
-              let outputStr = JSON.stringify(output, null, 2);
-              if (!options.verbose && outputStr.length > 500) {
-                outputStr = `${outputStr.substring(0, 500)}... (use --verbose for full output)`;
-              }
-              // Indent output
-              const indentedOutput = outputStr
-                .split('\n')
-                .map((line: string) => `      ${line}`)
-                .join('\n');
-              console.log(`    📤 Output:\n${indentedOutput}`);
-            } catch {
-              console.log(`    📤 Output: ${step.output.substring(0, 200)}`);
-            }
-          }
+// ... (other commands) ...
 
-          // Show usage if present
-          if (step.usage) {
-            try {
-              const usage = JSON.parse(step.usage);
-              if (usage.total_tokens) {
-                console.log(
-                  `    📊 Tokens: ${usage.total_tokens} (prompt: ${usage.prompt_tokens}, completion: ${usage.completion_tokens})`
-                );
-              }
-            } catch {
-              // Ignore parse errors
-            }
-          }
-        }
-      }
-
-      db.close();
-    } catch (error) {
-      console.error('✗ Failed to show logs:', error instanceof Error ? error.message : error);
-      process.exit(1);
-    }
-  });
-
-// ===== keystone prune =====
+// ===== keystone maintenance =====
 program
-  .command('prune')
-  .description('Delete old workflow runs from the database')
-  .option('--days <days>', 'Delete runs older than this many days', '7')
+  .command('maintenance')
+  .description('Perform database maintenance (prune old runs and vacuum)')
+  .option('--days <days>', 'Delete runs older than this many days', '30')
   .action(async (options) => {
     try {
       const days = Number.parseInt(options.days, 10);
@@ -520,16 +362,21 @@ program
         process.exit(1);
       }
 
+      console.log('🧹 Starting maintenance...');
       const db = new WorkflowDb();
+
+      console.log(`   Pruning runs older than ${days} days...`);
       const deleted = await db.pruneRuns(days);
-      if (deleted > 0) {
-        await db.vacuum();
-      }
-      db.close();
+      console.log(`   ✓ Deleted ${deleted} run(s)`);
 
-      console.log(`✓ Deleted ${deleted} workflow run(s) older than ${days} days`);
+      console.log('   Vacuuming database (reclaiming space)...');
+      await db.vacuum();
+      console.log('   ✓ Vacuum complete');
+
+      db.close();
+      console.log('\n✨ Maintenance completed successfully!');
     } catch (error) {
-      console.error('✗ Failed to prune runs:', error instanceof Error ? error.message : error);
+      console.error('✗ Maintenance failed:', error instanceof Error ? error.message : error);
       process.exit(1);
     }
   });
@@ -591,14 +438,8 @@ mcp
     console.log('   You can still manually provide an OAuth token below if you have one.');
     console.log('\n2. Paste the access token below:\n');
 
-    const prompt = 'Access Token: ';
-    process.stdout.write(prompt);
-
-    let token = '';
-    for await (const line of console) {
-      token = line.trim();
-      break;
-    }
+    const { promptSecret } = await import('./utils/prompt.ts');
+    const token = await promptSecret('Access Token: ');
 
     if (token) {
       const auth = AuthManager.load();
@@ -859,10 +700,10 @@ program.command('_list-workflows', { hidden: true }).action(() => {
   }
 });
 
-program.command('_list-runs', { hidden: true }).action(() => {
+program.command('_list-runs', { hidden: true }).action(async () => {
   try {
     const db = new WorkflowDb();
-    const runs = db.listRuns(50);
+    const runs = await db.listRuns(50);
     for (const run of runs) {
       console.log(run.id);
     }
@@ -959,11 +800,11 @@ __keystone_runs() {
       console.log(`_keystone_completion() {
   local cur prev opts
   COMPREPLY=()
-  cur="${COMP_WORDS[COMP_CWORD]}"
-  prev="${COMP_WORDS[COMP_CWORD - 1]}"
+  cur="\${COMP_WORDS[COMP_CWORD]}"
+  prev="\${COMP_WORDS[COMP_CWORD - 1]}"
   opts="init validate graph run resume workflows history logs prune ui mcp config auth completion"
 
-  case "${prev}" in
+  case "\${prev}" in
     run|graph)
       local workflows=$(keystone _list-workflows 2>/dev/null)
       COMPREPLY=( $(compgen -W "\${workflows}" -- \${cur}) )

package/src/db/memory-db.test.ts

@@ -0,0 +1,54 @@
+import { afterAll, describe, expect, test } from 'bun:test';
+import * as fs from 'node:fs';
+import { MemoryDb } from './memory-db';
+
+const TEST_DB = '.keystone/test-memory.db';
+
+describe('MemoryDb', () => {
+  // Clean up previous runs
+  if (fs.existsSync(TEST_DB)) {
+    fs.unlinkSync(TEST_DB);
+  }
+
+  const db = new MemoryDb(TEST_DB);
+
+  afterAll(() => {
+    db.close();
+    if (fs.existsSync(TEST_DB)) {
+      fs.unlinkSync(TEST_DB);
+    }
+  });
+
+  test('should initialize and store embedding', async () => {
+    const id = await db.store('hello world', Array(384).fill(0.1), { tag: 'test' });
+    expect(id).toBeDefined();
+    expect(typeof id).toBe('string');
+  });
+
+  test('should search and retrieve result', async () => {
+    // Store another item to search for
+    await db.store('search target', Array(384).fill(0.9), { tag: 'target' });
+
+    const results = await db.search(Array(384).fill(0.9), 1);
+    expect(results.length).toBe(1);
+    expect(results[0].text).toBe('search target');
+    expect(results[0].metadata).toEqual({ tag: 'target' });
+  });
+
+  test('should fail gracefully with invalid dimensions', async () => {
+    // sqlite-vec requires fixed dimensions (384 defined in schema)
+    // bun:sqlite usually throws an error for constraint violations
+    let error: unknown;
+    try {
+      await db.store('fail', Array(10).fill(0));
+    } catch (e) {
+      error = e;
+    }
+    if (error) {
+      expect(error).toBeDefined();
+    } else {
+      const results = await db.search(Array(384).fill(0), 1);
+      expect(Array.isArray(results)).toBe(true);
+    }
+  });
+});

package/src/db/memory-db.ts

@@ -0,0 +1,122 @@
+import type { Database } from 'bun:sqlite';
+import { randomUUID } from 'node:crypto';
+import * as sqliteVec from 'sqlite-vec';
+import './sqlite-setup.ts';
+
+export interface MemoryEntry {
+  id: string;
+  text: string;
+  metadata: Record<string, unknown>;
+  distance?: number;
+}
+
+export class MemoryDb {
+  private db: Database;
+  // Cache connections by path to avoid reloading extensions
+  private static connectionCache = new Map<string, { db: Database; refCount: number }>();
+
+  constructor(public readonly dbPath = '.keystone/memory.db') {
+    const cached = MemoryDb.connectionCache.get(dbPath);
+    if (cached) {
+      cached.refCount++;
+      this.db = cached.db;
+    } else {
+      const { Database } = require('bun:sqlite');
+      this.db = new Database(dbPath, { create: true });
+
+      // Load sqlite-vec extension
+      const extensionPath = sqliteVec.getLoadablePath();
+      this.db.loadExtension(extensionPath);
+
+      this.initSchema();
+
+      MemoryDb.connectionCache.set(dbPath, { db: this.db, refCount: 1 });
+    }
+  }
+
+  private initSchema(): void {
+    this.db.run(`
+      CREATE VIRTUAL TABLE IF NOT EXISTS vec_memory USING vec0(
+        id TEXT PRIMARY KEY,
+        embedding FLOAT[384]
+      );
+    `);
+
+    this.db.run(`
+      CREATE TABLE IF NOT EXISTS memory_metadata (
+        id TEXT PRIMARY KEY,
+        text TEXT NOT NULL,
+        metadata TEXT NOT NULL,
+        created_at TEXT NOT NULL
+      );
+    `);
+  }
+
+  async store(
+    text: string,
+    embedding: number[],
+    metadata: Record<string, unknown> = {}
+  ): Promise<string> {
+    const id = randomUUID();
+    const createdAt = new Date().toISOString();
+
+    // bun:sqlite transaction wrapper ensures atomicity synchronously
+    const insertTransaction = this.db.transaction(() => {
+      this.db.run('INSERT INTO vec_memory(id, embedding) VALUES (?, ?)', [
+        id,
+        new Float32Array(embedding),
+      ]);
+
+      this.db.run(
+        'INSERT INTO memory_metadata(id, text, metadata, created_at) VALUES (?, ?, ?, ?)',
+        [id, text, JSON.stringify(metadata), createdAt]
+      );
+    });
+
+    insertTransaction();
+    return id;
+  }
+
+  async search(embedding: number[], limit = 5): Promise<MemoryEntry[]> {
+    const query = `
+      SELECT 
+        v.id, 
+        v.distance,
+        m.text,
+        m.metadata
+      FROM vec_memory v
+      JOIN memory_metadata m ON v.id = m.id
+      WHERE embedding MATCH ? AND k = ?
+      ORDER BY distance
+    `;
+
+    // bun:sqlite is synchronous
+    const rows = this.db.prepare(query).all(new Float32Array(embedding), limit) as {
+      id: string;
+      distance: number;
+      text: string;
+      metadata: string;
+    }[];
+
+    return rows.map((row) => ({
+      id: row.id,
+      distance: row.distance,
+      text: row.text,
+      metadata: JSON.parse(row.metadata),
+    }));
+  }
+
+  close(): void {
+    const cached = MemoryDb.connectionCache.get(this.dbPath);
+    if (cached) {
+      cached.refCount--;
+      if (cached.refCount <= 0) {
+        cached.db.close();
+        MemoryDb.connectionCache.delete(this.dbPath);
+      }
+    } else {
+      // Fallback if not in cache for some reason
+      this.db.close();
+    }
+  }
+}

package/src/db/sqlite-setup.ts

@@ -0,0 +1,49 @@
+import { ConsoleLogger, type Logger } from '../utils/logger.ts';
+
+export function setupSqlite(logger: Logger = new ConsoleLogger()) {
+  // macOS typically comes with a system SQLite that doesn't support extensions
+  // We need to try to load a custom one (e.g. from Homebrew) if on macOS
+  if (process.platform === 'darwin') {
+    try {
+      const { Database } = require('bun:sqlite');
+      const { existsSync } = require('node:fs');
+
+      // Common Homebrew paths for SQLite
+      const paths = [
+        '/opt/homebrew/opt/sqlite/lib/libsqlite3.dylib',
+        '/usr/local/opt/sqlite/lib/libsqlite3.dylib',
+        // Fallback to checking brew prefix if available
+      ];
+
+      // Try to find brew prefix dynamically if possible
+      try {
+        const proc = Bun.spawnSync(['brew', '--prefix', 'sqlite'], {
+          stderr: 'ignore',
+        });
+        if (proc.success) {
+          const prefix = proc.stdout.toString().trim();
+          paths.unshift(`${prefix}/lib/libsqlite3.dylib`);
+        }
+      } catch {
+        // Brew might not be installed or in path
+      }
+
+      for (const libPath of paths) {
+        if (existsSync(libPath)) {
+          logger.log(`[SqliteSetup] Using custom SQLite library: ${libPath}`);
+          Database.setCustomSQLite(libPath);
+          return;
+        }
+      }
+
+      logger.warn(
+        '[SqliteSetup] Warning: Could not find Homebrew SQLite. Extension loading might fail.'
+      );
+    } catch (error) {
+      logger.warn(`[SqliteSetup] Failed to set custom SQLite: ${error}`);
+    }
+  }
+}
+
+// Run setup immediately when imported
+setupSqlite();