keystone-cli 0.7.1 → 0.8.0

package/README.md

@@ -372,6 +372,16 @@ You are a technical communications expert. Your goal is to take technical output
 
 Agents can be equipped with tools, which are essentially workflow steps they can choose to execute. You can define tools in the agent definition, or directly in an LLM step within a workflow.
 
+Keystone comes with a set of **Standard Tools** that can be enabled for any agent by setting `useStandardTools: true` in the step definition:
+
+- `read_file`: Read the contents of a file (arguments: `path`)
+- `read_file_lines`: Read a specific range of lines from a file (arguments: `path`, `start`, `count`)
+- `write_file`: Write or overwrite a file (arguments: `path`, `content`)
+- `list_files`: List files in a directory (arguments: `path`)
+- `search_files`: Search for files by glob pattern (arguments: `pattern`, `dir`)
+- `search_content`: Search for string or regex within files (arguments: `query`, `dir`, `pattern`)
+- `run_command`: Run a shell command (arguments: `command`, `dir`). Requires `allowInsecure: true` on the step unless whitelisted.
+
 Tool arguments are passed to the tool's execution step via the `args` variable.
 
 **`.keystone/workflows/agents/developer.md`**
@@ -379,28 +389,25 @@ Tool arguments are passed to the tool's execution step via the `args` variable.
 ---
 name: developer
 tools:
-  - name: list_files
-    description: List files in the current directory
+  - name: custom_tool
+    description: A custom tool definition
     execution:
-      id: list-files-tool
       type: shell
-      run: ls -F
-  - name: read_file
-    description: Read a specific file
-    parameters:
-      type: object
-      properties:
-        path: { type: string }
-      required: [path]
-    execution:
-      id: read-file-tool
-      type: file
-      op: read
-      path: ${{ args.path }}
+      run: echo "custom"
 ---
 You are a software developer. You can use tools to explore the codebase.
 ```
 
+To enable standard tools in a workflow step:
+
+```yaml
+- id: explore
+  type: llm
+  agent: developer
+  useStandardTools: true
+  prompt: "Explore the src directory"
+```
+
 ### Keystone as an MCP Server
 
 Keystone can itself act as an MCP server, allowing other agents (like Claude Desktop or GitHub Copilot) to discover and run your workflows as tools.
@@ -480,9 +487,9 @@ In these examples, the agent will have access to all tools provided by the MCP s
 | Command | Description |
 | :--- | :--- |
 | `init` | Initialize a new Keystone project |
-| `run <workflow>` | Execute a workflow (use `-i key=val` for inputs, `--dry-run` to test, `--debug` for REPL) |
+| `run <workflow>` | Execute a workflow (use `-i key=val`, `--resume` to auto-resume, `--dry-run`, `--debug`) |
 | `optimize <workflow>` | Optimize a specific step in a workflow (requires --target) |
-| `resume <run_id>` | Resume a failed or paused workflow |
+| `resume <run_id>` | Resume a failed/paused/crashed workflow by ID |
 | `validate [path]` | Check workflow files for errors |
 | `workflows` | List available workflows |
 | `history` | Show recent workflow runs |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "keystone-cli",
-  "version": "0.7.1",
+  "version": "0.8.0",
   "description": "A local-first, declarative, agentic workflow orchestrator built on Bun",
   "type": "module",
   "bin": {

package/src/cli.ts

@@ -228,7 +228,8 @@ program
   .option('-i, --input <key=value...>', 'Input values')
   .option('--dry-run', 'Show what would be executed without actually running it')
   .option('--debug', 'Enable interactive debug mode on failure')
-  .action(async (workflowPath, options) => {
+  .option('--resume', 'Resume the last run of this workflow if it failed or was paused')
+  .action(async (workflowPathArg, options) => {
     // Parse inputs
     const inputs: Record<string, unknown> = {};
     if (options.input) {
@@ -249,17 +250,47 @@ program
 
     // Load and validate workflow
     try {
-      const resolvedPath = WorkflowRegistry.resolvePath(workflowPath);
+      const resolvedPath = WorkflowRegistry.resolvePath(workflowPathArg);
       const workflow = WorkflowParser.loadWorkflow(resolvedPath);
 
       // Import WorkflowRunner dynamically
       const { WorkflowRunner } = await import('./runner/workflow-runner.ts');
       const logger = new ConsoleLogger();
+
+      let resumeRunId: string | undefined;
+
+      // Handle auto-resume
+      if (options.resume) {
+        const db = new WorkflowDb();
+        const lastRun = await db.getLastRun(workflow.name);
+        db.close();
+
+        if (lastRun) {
+          if (
+            lastRun.status === 'failed' ||
+            lastRun.status === 'paused' ||
+            lastRun.status === 'running'
+          ) {
+            resumeRunId = lastRun.id;
+            console.log(
+              `Resuming run ${lastRun.id} (status: ${lastRun.status}) from ${new Date(
+                lastRun.started_at
+              ).toLocaleString()}`
+            );
+          } else {
+            console.log(`Last run ${lastRun.id} completed successfully. Starting new run.`);
+          }
+        } else {
+          console.log('No previous run found. Starting new run.');
+        }
+      }
+
       const runner = new WorkflowRunner(workflow, {
         inputs,
         workflowDir: dirname(resolvedPath),
         dryRun: !!options.dryRun,
         debug: !!options.debug,
+        resumeRunId,
         logger,
       });
 

package/src/db/workflow-db.ts

@@ -342,6 +342,21 @@ export class WorkflowDb {
     });
   }
 
+  /**
+   * Get the most recent run for a specific workflow
+   */
+  async getLastRun(workflowName: string): Promise<WorkflowRun | null> {
+    return this.withRetry(() => {
+      const stmt = this.db.prepare(`
+        SELECT * FROM workflow_runs
+        WHERE workflow_name = ?
+        ORDER BY started_at DESC
+        LIMIT 1
+      `);
+      return stmt.get(workflowName) as WorkflowRun | null;
+    });
+  }
+
   close(): void {
     this.db.close();
   }

package/src/runner/standard-tools-execution.test.ts

@@ -0,0 +1,39 @@
+import { describe, expect, it } from 'bun:test';
+import * as vm from 'node:vm';
+import { STANDARD_TOOLS } from './standard-tools';
+
+describe('Standard Tools Execution Verification', () => {
+  const scriptTools = STANDARD_TOOLS.filter(
+    (t) => t.execution && t.execution.type === 'script' && typeof t.execution.run === 'string'
+  );
+
+  for (const tool of scriptTools) {
+    it(`should compile and execute ${tool.name} without SyntaxError`, () => {
+      const script = tool.execution.run as string;
+      const sandbox = {
+        args: { path: '.', pattern: '*', query: 'test' },
+        require: (mod: string) => {
+          if (mod === 'node:fs' || mod === 'fs') {
+            return {
+              existsSync: () => true,
+              readdirSync: () => [],
+              statSync: () => ({ size: 0 }),
+              readFileSync: () => '',
+            };
+          }
+          if (mod === 'node:path' || mod === 'path') {
+            return { join: (...args: string[]) => args.join('/') };
+          }
+          if (mod === 'glob') {
+            return { globSync: () => [] };
+          }
+          return {};
+        },
+      };
+
+      expect(() => {
+        vm.runInNewContext(script, sandbox);
+      }).not.toThrow();
+    });
+  }
+});

package/src/runner/standard-tools.ts

@@ -38,19 +38,21 @@ export const STANDARD_TOOLS: AgentTool[] = [
       id: 'std_read_file_lines',
       type: 'script',
       run: `
-        const fs = require('node:fs');
-        const path = require('node:path');
-        const filePath = args.path;
-        const start = args.start || 1;
-        const count = args.count || 100;
-
-        if (!fs.existsSync(filePath)) {
-          throw new Error('File not found: ' + filePath);
-        }
-
-        const content = fs.readFileSync(filePath, 'utf8');
-        const lines = content.split('\\n');
-        return lines.slice(start - 1, start - 1 + count).join('\\n');
+        (function() {
+          const fs = require('node:fs');
+          const path = require('node:path');
+          const filePath = args.path;
+          const start = args.start || 1;
+          const count = args.count || 100;
+  
+          if (!fs.existsSync(filePath)) {
+            throw new Error('File not found: ' + filePath);
+          }
+  
+          const content = fs.readFileSync(filePath, 'utf8');
+          const lines = content.split('\\n');
+          return lines.slice(start - 1, start - 1 + count).join('\\n');
+        })();
       `,
       allowInsecure: true,
     },
@@ -91,18 +93,20 @@ export const STANDARD_TOOLS: AgentTool[] = [
       id: 'std_list_files',
       type: 'script',
       run: `
-        const fs = require('node:fs');
-        const path = require('node:path');
-        const dir = args.path || '.';
-        if (fs.existsSync(dir)) {
-          const files = fs.readdirSync(dir, { withFileTypes: true });
-          return files.map(f => ({
-            name: f.name,
-            type: f.isDirectory() ? 'directory' : 'file',
-            size: f.isFile() ? fs.statSync(path.join(dir, f.name)).size : undefined
-          }));
-        }
-        throw new Error('Directory not found: ' + dir);
+        (function() {
+          const fs = require('node:fs');
+          const path = require('node:path');
+          const dir = args.path || '.';
+          if (fs.existsSync(dir)) {
+            const files = fs.readdirSync(dir, { withFileTypes: true });
+            return files.map(f => ({
+              name: f.name,
+              type: f.isDirectory() ? 'directory' : 'file',
+              size: f.isFile() ? fs.statSync(path.join(dir, f.name)).size : undefined
+            }));
+          }
+          throw new Error('Directory not found: ' + dir);
+        })();
       `,
       allowInsecure: true,
     },
@@ -122,16 +126,18 @@ export const STANDARD_TOOLS: AgentTool[] = [
       id: 'std_search_files',
       type: 'script',
       run: `
-        const fs = require('node:fs');
-        const path = require('node:path');
-        const { globSync } = require('glob');
-        const dir = args.dir || '.';
-        const pattern = args.pattern;
-        try {
-          return globSync(pattern, { cwd: dir, nodir: true });
-        } catch (e) {
-          throw new Error('Search failed: ' + e.message);
-        }
+        (function() {
+          const fs = require('node:fs');
+          const path = require('node:path');
+          const { globSync } = require('glob');
+          const dir = args.dir || '.';
+          const pattern = args.pattern;
+          try {
+            return globSync(pattern, { cwd: dir, nodir: true });
+          } catch (e) {
+            throw new Error('Search failed: ' + e.message);
+          }
+        })();
       `,
       allowInsecure: true,
     },
@@ -156,42 +162,44 @@ export const STANDARD_TOOLS: AgentTool[] = [
       id: 'std_search_content',
       type: 'script',
       run: `
-        const fs = require('node:fs');
-        const path = require('node:path');
-        const { globSync } = require('glob');
-        const dir = args.dir || '.';
-        const pattern = args.pattern || '**/*';
-        const query = args.query;
-        if (query.length > 500) {
-          throw new Error('Search query exceeds maximum length of 500 characters');
-        }
-        const isRegex = query.startsWith('/') && query.endsWith('/');
-        let regex;
-        try {
-          regex = isRegex ? new RegExp(query.slice(1, -1)) : new RegExp(query.replace(/[.*+?^$\\{}()|[\\]\\\\]/g, '\\\\$&'), 'i');
-        } catch (e) {
-          throw new Error('Invalid regular expression: ' + e.message);
-        }
-
-        const files = globSync(pattern, { cwd: dir, nodir: true });
-        const results = [];
-        for (const file of files) {
-          const fullPath = path.join(dir, file);
-          const content = fs.readFileSync(fullPath, 'utf8');
-          const lines = content.split('\\n');
-          for (let i = 0; i < lines.length; i++) {
-            if (regex.test(lines[i])) {
-              results.push({
-                file,
-                line: i + 1,
-                content: lines[i].trim()
-              });
+        (function() {
+          const fs = require('node:fs');
+          const path = require('node:path');
+          const { globSync } = require('glob');
+          const dir = args.dir || '.';
+          const pattern = args.pattern || '**/*';
+          const query = args.query;
+          if (query.length > 500) {
+            throw new Error('Search query exceeds maximum length of 500 characters');
+          }
+          const isRegex = query.startsWith('/') && query.endsWith('/');
+          let regex;
+          try {
+            regex = isRegex ? new RegExp(query.slice(1, -1)) : new RegExp(query.replace(/[.*+?^$\\{}()|[\\]\\\\]/g, '\\\\$&'), 'i');
+          } catch (e) {
+            throw new Error('Invalid regular expression: ' + e.message);
+          }
+  
+          const files = globSync(pattern, { cwd: dir, nodir: true });
+          const results = [];
+          for (const file of files) {
+            const fullPath = path.join(dir, file);
+            const content = fs.readFileSync(fullPath, 'utf8');
+            const lines = content.split('\\n');
+            for (let i = 0; i < lines.length; i++) {
+              if (regex.test(lines[i])) {
+                results.push({
+                  file,
+                  line: i + 1,
+                  content: lines[i].trim()
+                });
+              }
+              if (results.length > 100) break; // Limit results
             }
-            if (results.length > 100) break; // Limit results
+            if (results.length > 100) break;
           }
-          if (results.length > 100) break;
-        }
-        return results;
+          return results;
+        })();
       `,
       allowInsecure: true,
     },

package/src/runner/step-executor.ts

@@ -18,6 +18,7 @@ import { getAdapter } from './llm-adapter.ts';
 import { detectShellInjectionRisk, executeShell } from './shell-executor.ts';
 
 import * as fs from 'node:fs';
+import { createRequire } from 'node:module';
 import * as os from 'node:os';
 import * as path from 'node:path';
 import * as readline from 'node:readline/promises';
@@ -543,6 +544,8 @@ async function executeScriptStep(
       );
     }
 
+    const requireFn = createRequire(import.meta.url);
+
     const result = await sandbox.execute(
       step.run,
       {
@@ -550,6 +553,10 @@ async function executeScriptStep(
         secrets: context.secrets,
         steps: context.steps,
         env: context.env,
+        // biome-ignore lint/suspicious/noExplicitAny: args is dynamic
+        args: (context as any).args,
+        require: requireFn,
+        console,
       },
       {
         timeout: step.timeout,

package/src/runner/workflow-runner.test.ts

@@ -115,6 +115,8 @@ describe('WorkflowRunner', () => {
       },
       error: (msg: string) => console.error(msg),
       warn: (msg: string) => console.warn(msg),
+      info: (msg: string) => {},
+      debug: (msg: string) => {},
     };
 
     const finallyWorkflow: Workflow = {
@@ -487,4 +489,59 @@ describe('WorkflowRunner', () => {
 
     if (existsSync(resumeDbPath)) rmSync(resumeDbPath);
   });
+
+  it('should resume a workflow marked as running (crashed process)', async () => {
+    const resumeDbPath = 'test-running-resume.db';
+    if (existsSync(resumeDbPath)) rmSync(resumeDbPath);
+
+    const workflow: Workflow = {
+      name: 'running-wf',
+      steps: [
+        { id: 's1', type: 'shell', run: 'echo "one"', needs: [] },
+        { id: 's2', type: 'shell', run: 'echo "two"', needs: ['s1'] },
+      ],
+      outputs: {
+        out: '${{ steps.s1.output.stdout.trim() }}-${{ steps.s2.output.stdout.trim() }}',
+      },
+    } as unknown as Workflow;
+
+    // Manually create a "running" state in the DB
+    const db = new WorkflowDb(resumeDbPath);
+    const runId = crypto.randomUUID();
+    await db.createRun(runId, workflow.name, {});
+    await db.updateRunStatus(runId, 'running');
+
+    // Create a completed step 1
+    const step1Id = crypto.randomUUID();
+    await db.createStep(step1Id, runId, 's1');
+    await db.completeStep(step1Id, 'success', { stdout: 'one\n', stderr: '', exitCode: 0 });
+    db.close();
+
+    // Verify warnings
+    let warningLogged = false;
+    const logger = {
+      log: () => {},
+      error: () => {},
+      warn: (msg: string) => {
+        if (msg.includes("Resuming a run marked as 'running'")) {
+          warningLogged = true;
+        }
+      },
+      info: () => {},
+      debug: () => {},
+    };
+
+    const runner = new WorkflowRunner(workflow, {
+      dbPath: resumeDbPath,
+      resumeRunId: runId,
+      // @ts-ignore
+      logger: logger,
+    });
+
+    const outputs = await runner.run();
+    expect(outputs.out).toBe('one-two');
+    expect(warningLogged).toBe(true);
+
+    if (existsSync(resumeDbPath)) rmSync(resumeDbPath);
+  });
 });

package/src/runner/workflow-runner.ts

@@ -161,10 +161,20 @@ export class WorkflowRunner {
       throw new Error(`Run ${this.runId} not found`);
     }
 
-    // Only allow resuming failed or paused runs
-    if (run.status !== WorkflowStatus.FAILED && run.status !== WorkflowStatus.PAUSED) {
+    // Only allow resuming failed, paused, or running (crash recovery) runs
+    if (
+      run.status !== WorkflowStatus.FAILED &&
+      run.status !== WorkflowStatus.PAUSED &&
+      run.status !== WorkflowStatus.RUNNING
+    ) {
       throw new Error(
-        `Cannot resume run with status '${run.status}'. Only 'failed' or 'paused' runs can be resumed.`
+        `Cannot resume run with status '${run.status}'. Only 'failed', 'paused', or 'running' runs can be resumed.`
+      );
+    }
+
+    if (run.status === WorkflowStatus.RUNNING) {
+      this.logger.warn(
+        `⚠️  Resuming a run marked as 'running'. This usually means the previous process crashed or was killed forcefully. Ensure no other instances are running.`
       );
     }