keystone-cli 1.1.2 → 1.3.0

package/src/runner/executors/dynamic-types.ts

@@ -0,0 +1,69 @@
+/**
+ * Shared types for Dynamic Step feature
+ */
+import type { StepResult } from './types.ts';
+
+export interface GeneratedStep {
+  id: string;
+  name: string;
+  type: 'llm' | 'shell' | 'workflow' | 'file' | 'request';
+  agent?: string;
+  prompt?: string;
+  run?: string;
+  path?: string;
+  op?: 'read' | 'write' | 'append';
+  content?: string;
+  needs?: string[];
+  inputs?: Record<string, unknown>;
+  allowStepFailure?: boolean;
+}
+
+export interface DynamicPlan {
+  workflow_id?: string;
+  steps: GeneratedStep[];
+  notes?: string;
+}
+
+/**
+ * State for tracking dynamic step execution
+ */
+export interface DynamicStepState {
+  // Identity
+  id?: string; // Database ID (optional for in-memory)
+  workflowId: string; // The workflow instance ID
+  runId?: string; // The higher-level run ID
+  stepId?: string; // The step ID within the workflow
+
+  // State
+  status: 'planning' | 'awaiting_confirmation' | 'executing' | 'completed' | 'failed';
+  generatedPlan: DynamicPlan;
+
+  // Execution tracking
+  stepResults: Map<string, StepResult>;
+  currentStepIndex: number;
+  replanCount: number;
+
+  // Timing & Metadata
+  startedAt: string;
+  completedAt?: string;
+  error?: string;
+  metadata?: Record<string, unknown>;
+}
+
+/**
+ * Individual generated step execution record
+ */
+export interface DynamicStepExecution {
+  id: string;
+  stateId: string;
+  stepId: string;
+  stepName: string;
+  stepType: string;
+  stepDefinition: GeneratedStep;
+  status: 'pending' | 'running' | 'success' | 'failed' | 'skipped';
+  output?: unknown;
+  error?: string;
+  startedAt?: string;
+  completedAt?: string;
+  executionOrder: number;
+}

package/src/runner/executors/file-executor.test.ts

@@ -36,13 +36,15 @@ describe('file-executor', () => {
       const blocks: SearchReplaceBlock[] = [
         { search: ' indent', replace: 'dedent' }, // one space vs two
       ];
-      // Should not find ' indent' because actual is '  indent' (double space) if strict?
-      // Actually '  indent'.split(' indent') -> [' ', ''] -> found once!
-      // Wait '  indent'.includes(' indent') is true.
-      // '  indent'.split(' indent') -> [' ', ''] (length 2).
-      // So it works.
       const result = applySearchReplaceBlocks(content, blocks);
       expect(result).toBe(' dedent');
     });
+
+    it('should handle CRLF line endings by treating them as LF', () => {
+      const content = 'line1\r\nline2\r\nline3';
+      const blocks: SearchReplaceBlock[] = [{ search: 'line2', replace: 'modified' }];
+      const result = applySearchReplaceBlocks(content, blocks);
+      expect(result).toBe('line1\nmodified\nline3');
+    });
   });
 });

package/src/runner/executors/file-executor.ts

@@ -116,7 +116,7 @@ export function applyUnifiedDiff(content: string, patch: string, targetPath: str
   const diff = parseUnifiedDiff(patch);
   assertDiffMatchesTarget(diff.newFile || diff.originalFile, targetPath);
 
-  const lines = content.split('\n');
+  const lines = content.replace(/\r\n/g, '\n').split('\n');
   const resultLines = [...lines];
 
   // Apply hunks in reverse order to keep line numbers valid
@@ -188,7 +188,7 @@ export function parseSearchReplaceBlocks(patch: string): SearchReplaceBlock[] {
 }
 
 export function applySearchReplaceBlocks(content: string, blocks: SearchReplaceBlock[]): string {
-  let result = content;
+  let result = content.replace(/\r\n/g, '\n');
 
   for (const block of blocks) {
     const parts = result.split(block.search);

package/src/runner/executors/git-executor.test.ts

@@ -0,0 +1,278 @@
+import { afterEach, beforeEach, describe, expect, it, jest, spyOn } from 'bun:test';
+import type { GitStep } from '../../parser/schema.ts';
+import { ConsoleLogger } from '../../utils/logger.ts';
+import { executeGitStep } from './git-executor.ts';
+import * as shellExecutor from './shell-executor.ts';
+
+describe('git-executor', () => {
+  const logger = new ConsoleLogger();
+  const context = {
+    env: {},
+    inputs: {},
+    steps: {},
+  };
+
+  let executeShellSpy: any;
+
+  beforeEach(() => {
+    executeShellSpy = spyOn(shellExecutor, 'executeShell').mockResolvedValue({
+      stdout: 'git output',
+      stderr: '',
+      exitCode: 0,
+    });
+  });
+
+  afterEach(() => {
+    executeShellSpy.mockRestore();
+  });
+
+  it('should execute clone operation', async () => {
+    const step: GitStep = {
+      id: 'git-clone',
+      type: 'git',
+      op: 'clone',
+      url: 'https://github.com/mhingston/keystone-cli.git',
+      needs: [],
+    };
+
+    const result = await executeGitStep(step, context, logger);
+    expect(result.status).toBe('success');
+    expect(executeShellSpy).toHaveBeenCalledWith(
+      expect.objectContaining({
+        run: 'git clone https://github.com/mhingston/keystone-cli.git ',
+      }),
+      context,
+      logger,
+      undefined,
+      'git clone https://github.com/mhingston/keystone-cli.git '
+    );
+  });
+
+  it('should execute clone with branch', async () => {
+    const step: GitStep = {
+      id: 'git-clone-branch',
+      type: 'git',
+      op: 'clone',
+      url: 'https://github.com/mhingston/keystone-cli.git',
+      branch: 'main',
+      needs: [],
+    };
+
+    const result = await executeGitStep(step, context, logger);
+    expect(result.status).toBe('success');
+    expect(executeShellSpy).toHaveBeenCalledWith(
+      expect.objectContaining({
+        run: 'git clone -b main https://github.com/mhingston/keystone-cli.git ',
+      }),
+      context,
+      logger,
+      undefined,
+      'git clone -b main https://github.com/mhingston/keystone-cli.git '
+    );
+  });
+
+  it('should execute worktree_add operation', async () => {
+    const step: GitStep = {
+      id: 'git-worktree-add',
+      type: 'git',
+      op: 'worktree_add',
+      path: './tmp-worktree',
+      branch: 'feature-branch',
+      needs: [],
+    };
+
+    const result = await executeGitStep(step, context, logger);
+    expect(result.status).toBe('success');
+    expect((result.output as any).worktreePath).toBe('./tmp-worktree');
+    expect(executeShellSpy).toHaveBeenCalledWith(
+      expect.objectContaining({
+        run: 'git worktree add ./tmp-worktree feature-branch',
+      }),
+      context,
+      logger,
+      undefined,
+      'git worktree add ./tmp-worktree feature-branch'
+    );
+  });
+
+  it('should execute worktree_remove operation', async () => {
+    const step: GitStep = {
+      id: 'git-worktree-remove',
+      type: 'git',
+      op: 'worktree_remove',
+      path: './tmp-worktree',
+      needs: [],
+    };
+
+    const result = await executeGitStep(step, context, logger);
+    expect(result.status).toBe('success');
+    expect(executeShellSpy).toHaveBeenCalledWith(
+      expect.objectContaining({
+        run: 'git worktree remove ./tmp-worktree',
+      }),
+      context,
+      logger,
+      undefined,
+      'git worktree remove ./tmp-worktree'
+    );
+  });
+
+  it('should execute checkout operation', async () => {
+    const step: GitStep = {
+      id: 'git-checkout',
+      type: 'git',
+      op: 'checkout',
+      branch: 'main',
+      needs: [],
+    };
+
+    const result = await executeGitStep(step, context, logger);
+    expect(result.status).toBe('success');
+    expect(executeShellSpy).toHaveBeenCalledWith(
+      expect.objectContaining({
+        run: 'git checkout main',
+      }),
+      context,
+      logger,
+      undefined,
+      'git checkout main'
+    );
+  });
+
+  it('should execute pull operation', async () => {
+    const step: GitStep = {
+      id: 'git-pull',
+      type: 'git',
+      op: 'pull',
+      branch: 'main',
+      needs: [],
+    };
+
+    const result = await executeGitStep(step, context, logger);
+    expect(result.status).toBe('success');
+    expect(executeShellSpy).toHaveBeenCalledWith(
+      expect.objectContaining({
+        run: 'git pull origin main',
+      }),
+      context,
+      logger,
+      undefined,
+      'git pull origin main'
+    );
+  });
+
+  it('should execute push operation', async () => {
+    const step: GitStep = {
+      id: 'git-push',
+      type: 'git',
+      op: 'push',
+      branch: 'main',
+      needs: [],
+    };
+
+    const result = await executeGitStep(step, context, logger);
+    expect(result.status).toBe('success');
+    expect(executeShellSpy).toHaveBeenCalledWith(
+      expect.objectContaining({
+        run: 'git push origin main',
+      }),
+      context,
+      logger,
+      undefined,
+      'git push origin main'
+    );
+  });
+
+  it('should execute commit operation', async () => {
+    const step: GitStep = {
+      id: 'git-commit',
+      type: 'git',
+      op: 'commit',
+      message: 'test commit',
+      needs: [],
+    };
+
+    const result = await executeGitStep(step, context, logger);
+    expect(result.status).toBe('success');
+    expect(executeShellSpy).toHaveBeenCalledWith(
+      expect.objectContaining({
+        run: 'git add . && git commit -m "test commit"',
+      }),
+      context,
+      logger,
+      undefined,
+      'git add . && git commit -m "test commit"'
+    );
+  });
+
+  it('should handle failed operations', async () => {
+    executeShellSpy.mockResolvedValue({
+      stdout: '',
+      stderr: 'git error',
+      exitCode: 1,
+    });
+
+    const step: GitStep = {
+      id: 'git-fail',
+      type: 'git',
+      op: 'pull',
+      needs: [],
+    };
+
+    const result = await executeGitStep(step, context, logger);
+    expect(result.status).toBe('failed');
+    expect(result.error).toContain('Git pull failed with code 1: git error');
+  });
+
+  it('should throw error for missing url in clone', async () => {
+    const step: GitStep = {
+      id: 'git-clone-no-url',
+      type: 'git',
+      op: 'clone',
+      needs: [],
+    };
+
+    await expect(executeGitStep(step, context, logger)).rejects.toThrow(
+      'Git clone requires a "url"'
+    );
+  });
+
+  it('should throw error for missing path in worktree_add', async () => {
+    const step: GitStep = {
+      id: 'git-worktree-no-path',
+      type: 'git',
+      op: 'worktree_add',
+      needs: [],
+    };
+
+    await expect(executeGitStep(step, context, logger)).rejects.toThrow(
+      'Git worktree_add requires a "path"'
+    );
+  });
+
+  it('should throw error for missing branch in checkout', async () => {
+    const step: GitStep = {
+      id: 'git-checkout-no-branch',
+      type: 'git',
+      op: 'checkout',
+      needs: [],
+    };
+
+    await expect(executeGitStep(step, context, logger)).rejects.toThrow(
+      'Git checkout requires a "branch"'
+    );
+  });
+
+  it('should throw error for missing message in commit', async () => {
+    const step: GitStep = {
+      id: 'git-commit-no-msg',
+      type: 'git',
+      op: 'commit',
+      needs: [],
+    };
+
+    await expect(executeGitStep(step, context, logger)).rejects.toThrow(
+      'Git commit requires a "message"'
+    );
+  });
+});

package/src/runner/executors/git-executor.ts

@@ -0,0 +1,100 @@
+import { ExpressionEvaluator } from '../../expression/evaluator.ts';
+import type { ExpressionContext } from '../../expression/evaluator.ts';
+import type { GitStep } from '../../parser/schema.ts';
+import { ConsoleLogger, type Logger } from '../../utils/logger.ts';
+import { PathResolver } from '../../utils/paths.ts';
+import { type ShellResult, executeShell } from './shell-executor.ts';
+import type { StepResult } from './types.ts';
+
+/**
+ * Git command executor
+ */
+export async function executeGitStep(
+  step: GitStep,
+  context: ExpressionContext,
+  logger: Logger = new ConsoleLogger(),
+  abortSignal?: AbortSignal
+): Promise<StepResult> {
+  const op = step.op;
+  const cwd = step.cwd ? ExpressionEvaluator.evaluateString(step.cwd, context) : undefined;
+
+  if (cwd) {
+    PathResolver.assertWithinCwd(cwd, step.allowOutsideCwd, 'CWD');
+  }
+
+  // Pre-process common inputs
+  const path = step.path ? ExpressionEvaluator.evaluateString(step.path, context) : undefined;
+  const url = step.url ? ExpressionEvaluator.evaluateString(step.url, context) : undefined;
+  const branch = step.branch ? ExpressionEvaluator.evaluateString(step.branch, context) : undefined;
+  const message = step.message
+    ? ExpressionEvaluator.evaluateString(step.message, context)
+    : undefined;
+
+  let command = '';
+  switch (op) {
+    case 'clone':
+      if (!url) throw new Error('Git clone requires a "url"');
+      command = `git clone ${branch ? `-b ${branch} ` : ''}${url} ${path || ''}`;
+      break;
+    case 'worktree_add':
+      if (!path) throw new Error('Git worktree_add requires a "path"');
+      command = `git worktree add ${path} ${branch || ''}`;
+      break;
+    case 'worktree_remove':
+      if (!path) throw new Error('Git worktree_remove requires a "path"');
+      command = `git worktree remove ${path}`;
+      break;
+    case 'checkout':
+      if (!branch) throw new Error('Git checkout requires a "branch"');
+      command = `git checkout ${branch}`;
+      break;
+    case 'pull':
+      command = `git pull origin ${branch || ''}`;
+      break;
+    case 'push':
+      command = `git push origin ${branch || ''}`;
+      break;
+    case 'commit':
+      if (!message) throw new Error('Git commit requires a "message"');
+      command = `git add . && git commit -m "${message.replace(/"/g, '\\"')}"`;
+      break;
+    default:
+      throw new Error(`Unsupported git operation: ${op}`);
+  }
+
+  // Use executeShell to leverage its security checks and output handling
+  // We explicitly allow insecure if the user set it, but by default executeShell
+  // will check the command against the whitelist.
+  // Note: Git commands often contain spaces/slashes which are in the whitelist.
+  const result: ShellResult = await executeShell(
+    {
+      ...step,
+      type: 'shell',
+      run: command,
+      dir: cwd,
+    },
+    context,
+    logger,
+    abortSignal,
+    command
+  );
+
+  if (result.exitCode !== 0) {
+    return {
+      output: result,
+      status: 'failed',
+      error: `Git ${op} failed with code ${result.exitCode}: ${result.stderr}`,
+    };
+  }
+
+  // Provide some structured output based on the operation
+  const output: any = { ...result };
+  if (op === 'worktree_add') {
+    output.worktreePath = path;
+  }
+
+  return {
+    output,
+    status: 'success',
+  };
+}

package/src/runner/executors/security.test.ts

@@ -0,0 +1,69 @@
+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
+import { AUTO_LOAD_SECRET_PREFIXES } from '../../utils/env-constants';
+import { SecretManager } from '../services/secret-manager';
+import { detectShellInjectionRisk } from './shell-executor';
+
+describe('Security Fixes', () => {
+  describe('ShellExecutor Command Injection', () => {
+    test('detectShellInjectionRisk should block newlines', () => {
+      // Regular space should be allowed
+      expect(detectShellInjectionRisk('echo hello')).toBe(false);
+
+      // Newline characters should be detected as risk
+      expect(detectShellInjectionRisk('echo hello\n')).toBe(true);
+      expect(detectShellInjectionRisk('echo hello\r')).toBe(true);
+      expect(detectShellInjectionRisk('echo hello\nrm -rf /')).toBe(true);
+
+      // Standard allowed characters should still pass
+      expect(detectShellInjectionRisk('curl -X POST https://example.com/api')).toBe(false);
+      expect(detectShellInjectionRisk('file_name-v1.0+beta.txt')).toBe(false);
+    });
+
+    test('detectShellInjectionRisk should correctly handle quotes', () => {
+      // Content inside single quotes is considered safe (literal)
+      expect(detectShellInjectionRisk("echo 'safe string with ; and |'")).toBe(false);
+
+      // But unsafe chars outside quotes must be caught
+      expect(detectShellInjectionRisk("echo 'safe'; rm -rf /")).toBe(true);
+    });
+  });
+
+  describe('SecretManager Auto-loading', () => {
+    // Save original env to restore later
+    const originalEnv = { ...Bun.env };
+
+    afterAll(() => {
+      // Restore env
+      for (const key of Object.keys(Bun.env)) {
+        delete Bun.env[key];
+      }
+      for (const [key, value] of Object.entries(originalEnv)) {
+        if (value) Bun.env[key] = value;
+      }
+    });
+
+    test('should only load secrets with allowed prefixes', () => {
+      // Setup test env vars
+      Bun.env.KEYSTONE_TEST_SECRET = 'secret-value-1';
+      Bun.env.GITHUB_TOKEN = 'gh-token-123';
+      Bun.env.MY_RANDOM_TOKEN = 'unsafe-token-should-not-load';
+      Bun.env.SOME_API_KEY = 'unsafe-api-key';
+
+      const secretManager = new SecretManager();
+      const secrets = secretManager.loadSecrets();
+
+      expect(secrets.KEYSTONE_TEST_SECRET).toBe('secret-value-1');
+      expect(secrets.GITHUB_TOKEN).toBe('gh-token-123');
+
+      // Should NOT load these even though they contain "TOKEN" or "KEY"
+      expect(secrets.MY_RANDOM_TOKEN).toBeUndefined();
+      expect(secrets.SOME_API_KEY).toBeUndefined();
+    });
+
+    test('env constants should match implementation', () => {
+      // Quick sanity check that our test logic matches the constants
+      expect(AUTO_LOAD_SECRET_PREFIXES).toContain('KEYSTONE_');
+      expect(AUTO_LOAD_SECRET_PREFIXES).toContain('GITHUB_');
+    });
+  });
+});

package/src/runner/executors/shell-executor.ts

@@ -43,6 +43,9 @@ export async function executeShellStep(
   abortSignal?: AbortSignal
 ): Promise<StepResult> {
   if (step.args) {
+    // args are inherently safe from shell injection as they skip the shell
+    // and pass the array directly to the OS via Bun.spawn.
+
     const command = step.args.map((a) => ExpressionEvaluator.evaluateString(a, context)).join(' ');
     if (dryRun) {
       logger.log(`[DRY RUN] Would execute: ${command}`);
@@ -200,13 +203,35 @@ async function readStreamWithLimit(
 }
 
 // Whitelist of allowed characters for secure shell command execution
-// Allows: Alphanumeric, whitespace, and common safe punctuation (_ . / : @ , + - = ' " !)
-// Blocks: Backslashes, pipes, redirects, subshells, variables ($), etc.
-const SAFE_SHELL_CHARS = /^[a-zA-Z0-9\s_./:@,+=~'"!-]+$/;
+// Allows: Alphanumeric, space, and common safe punctuation (_ . / : @ , + - = ' " ! ~)
+// Blocks: Newlines (\n, \r), Pipes, redirects, subshells, variables ($), etc.
+const SAFE_SHELL_CHARS = /^[a-zA-Z0-9 _./:@,+=~'"!-]+$/;
 
 export function detectShellInjectionRisk(rawCommand: string): boolean {
-  // If the command contains any character NOT in the whitelist, it's considered risky
-  return !SAFE_SHELL_CHARS.test(rawCommand);
+  // We scan the command to handle single quotes correctly.
+  // Characters inside single quotes are considered escaped/literal and safe from shell injection.
+  let inSingleQuote = false;
+
+  for (let i = 0; i < rawCommand.length; i++) {
+    const char = rawCommand[i];
+
+    if (char === "'") {
+      inSingleQuote = !inSingleQuote;
+      continue;
+    }
+
+    // Outside single quotes, we enforce the strict whitelist
+    if (!inSingleQuote) {
+      if (!SAFE_SHELL_CHARS.test(char)) {
+        return true;
+      }
+    }
+    // Inside single quotes, everything is treated as a literal string by the shell,
+    // so we don't need to block special characters.
+  }
+
+  // If we ended with an unclosed single quote, it's a syntax risk
+  return inSingleQuote;
 }
 
 /**

package/src/runner/memoization-leak.test.ts

@@ -0,0 +1,83 @@
+import { afterEach, describe, expect, it } from 'bun:test';
+import { existsSync, rmSync } from 'node:fs';
+import { MemoryDb } from '../db/memory-db';
+import { WorkflowDb } from '../db/workflow-db';
+import type { Workflow } from '../parser/schema';
+import { container } from '../utils/container';
+import { ConsoleLogger } from '../utils/logger';
+import { WorkflowRunner } from './workflow-runner';
+
+describe('Workflow Memoization Leak (Args Check)', () => {
+  const dbPath = 'test-memoization-leak.db';
+
+  container.register('logger', new ConsoleLogger());
+  container.register('db', new WorkflowDb(dbPath));
+  container.register('memoryDb', new MemoryDb());
+
+  afterEach(() => {
+    if (existsSync(dbPath)) {
+      rmSync(dbPath);
+    }
+  });
+
+  it('should NOT collide for shell steps with same command but different args', async () => {
+    const workflow: Workflow = {
+      name: 'memoize-args-wf',
+      inputs: {
+        arg: { type: 'string' },
+      },
+      steps: [
+        {
+          id: 's1',
+          type: 'shell',
+          args: ['echo', '${{ inputs.arg }}'],
+          allowInsecure: true,
+          memoize: true,
+          needs: [],
+        },
+      ],
+      outputs: {
+        out: '${{ steps.s1.output.stdout.trim() }}',
+      },
+    } as unknown as Workflow;
+
+    let executeCount = 0;
+    const trackedExecuteStep = async (step: any, context: any, logger: any, options: any) => {
+      if (step.id === 's1') executeCount++;
+      const { executeStep } = await import('./step-executor');
+      return executeStep(step, context, logger, options);
+    };
+
+    // Run 1: arg=A
+    const runner1 = new WorkflowRunner(workflow, {
+      dbPath,
+      inputs: { arg: 'A' },
+      executeStep: trackedExecuteStep,
+    });
+    const out1 = await runner1.run();
+    expect(out1.out).toBe('A');
+    expect(executeCount).toBe(1);
+
+    // Run 2: arg=A -> Cache Hit
+    const runner2 = new WorkflowRunner(workflow, {
+      dbPath,
+      inputs: { arg: 'A' },
+      executeStep: trackedExecuteStep,
+    });
+    executeCount = 0;
+    const out2 = await runner2.run();
+    expect(out2.out).toBe('A');
+    expect(executeCount).toBe(0); // Memoized
+
+    // Run 3: arg=B -> Execute (Must not collide with A)
+    const runner3 = new WorkflowRunner(workflow, {
+      dbPath,
+      inputs: { arg: 'B' },
+      executeStep: trackedExecuteStep,
+    });
+    executeCount = 0;
+    const out3 = await runner3.run();
+    expect(out3.out).toBe('B');
+    expect(executeCount).toBe(1); // Should execute because args are different!
+  });
+});