keystone-cli 2.0.0 → 2.1.0

package/src/runner/executors/dynamic-executor.ts

@@ -208,7 +208,7 @@ function convertToExecutableStep(
 export async function executeDynamicStep(
   step: DynamicStep,
   context: ExpressionContext,
-  executeStepFn: (step: Step, context: ExpressionContext) => Promise<StepResult>,
+  executeStepFn: (step: Step, context: ExpressionContext, depth?: number) => Promise<StepResult>,
   logger: Logger,
   options: StepExecutorOptions & {
     stateManager?: DynamicStateManager;
@@ -216,9 +216,18 @@ export async function executeDynamicStep(
     saveState?: (stepId: string, state: DynamicStepState) => Promise<void>;
     executeLlmStep?: typeof executeLlmStep;
     executeHumanStep?: typeof executeHumanStep;
+    depth?: number;
   }
 ): Promise<StepResult> {
   const { runId, db, abortSignal } = options;
+  const depth = options.depth || 0;
+
+  // Prevent infinite recursion in dynamic steps
+  // 10 is matching WorkflowRunner.MAX_RECURSION_DEPTH
+  // We should ideally import this constant, but for now we hardcode or add to LIMITS
+  if (depth > 10) {
+    throw new Error('Maximum workflow recursion depth (10) exceeded in dynamic step.');
+  }
   const stateManager = options.stateManager || (db ? new DynamicStateManager(db) : null);
 
   const { state, dbState } = await initializeState(step, runId, stateManager, options.loadState);
@@ -242,7 +251,7 @@ export async function executeDynamicStep(
           stateManager,
           executeStepFn,
           logger,
-          options
+          { ...options, depth }
         );
       }
 
@@ -259,7 +268,7 @@ export async function executeDynamicStep(
           stateManager,
           executeStepFn,
           logger,
-          options
+          { ...options, depth }
         );
       }
     }
@@ -329,12 +338,13 @@ async function handlePlanningPhase(
   state: DynamicStepState,
   dbState: DynamicStepState | null,
   stateManager: DynamicStateManager | null,
-  executeStepFn: (step: Step, context: ExpressionContext) => Promise<StepResult>,
+  executeStepFn: (step: Step, context: ExpressionContext, depth?: number) => Promise<StepResult>,
   logger: Logger,
   options: StepExecutorOptions & {
     stateManager?: DynamicStateManager;
     saveState?: (stepId: string, state: DynamicStepState) => Promise<void>;
     executeLlmStep?: typeof executeLlmStep;
+    depth?: number;
   }
 ) {
   const { runId, emitEvent, workflowName, abortSignal, mcpManager, workflowDir } = options;
@@ -392,6 +402,17 @@ async function handlePlanningPhase(
   }
 
   state.generatedPlan = planResult.output as DynamicPlan;
+  // Early validation of plan structure and dependencies
+  try {
+    topologicalSort(state.generatedPlan.steps);
+  } catch (err) {
+    state.status = 'failed';
+    state.error = err instanceof Error ? err.message : String(err);
+    if (stateManager && dbState && dbState.id)
+      await stateManager.finish(dbState.id, 'failed', state.error);
+    throw new Error(state.error);
+  }
+
   state.status = step.confirmPlan ? 'awaiting_confirmation' : 'executing';
 
   logger.log(`  📋 Plan generated with ${state.generatedPlan.steps.length} steps:`);
@@ -449,6 +470,8 @@ async function handleConfirmationPhase(
       try {
         const modifiedPlan = JSON.parse(response) as DynamicPlan;
         if (modifiedPlan.steps && Array.isArray(modifiedPlan.steps)) {
+          // Validate modified plan
+          topologicalSort(modifiedPlan.steps);
           state.generatedPlan = modifiedPlan;
           logger.log('  ✓ Using modified plan');
         }
@@ -480,14 +503,15 @@ async function handleExecutionPhase(
   state: DynamicStepState,
   dbState: DynamicStepState | null,
   stateManager: DynamicStateManager | null,
-  executeStepFn: (step: Step, context: ExpressionContext) => Promise<StepResult>,
+  executeStepFn: (step: Step, context: ExpressionContext, depth?: number) => Promise<StepResult>,
   logger: Logger,
   options: StepExecutorOptions & {
     stateManager?: DynamicStateManager;
     saveState?: (stepId: string, state: DynamicStepState) => Promise<void>;
+    depth?: number;
   }
 ) {
-  const { abortSignal, runId, workflowName, emitEvent, saveState } = options;
+  const { abortSignal, runId, workflowName, emitEvent, saveState, depth = 0 } = options;
 
   // Detect circular dependencies and validate plan
   topologicalSort(state.generatedPlan.steps);
@@ -522,6 +546,9 @@ async function handleExecutionPhase(
         1;
   logger.log(`  🚀 Starting parallel execution (concurrency: ${maxConcurrency})`);
 
+  // Track running promises to avoid busy wait
+  const executionPromises = new Set<Promise<void>>();
+
   while (completed.size + failed.size < state.generatedPlan.steps.length) {
     if (abortSignal?.aborted) throw new Error('Dynamic step execution canceled');
 
@@ -551,94 +578,104 @@ async function handleExecutionPhase(
       break;
     }
 
+    // Helper to wrap execution with cleanup
+    const executeWrapper = async (genStep: GeneratedStep, i: number) => {
+      try {
+        logger.log(
+          `  ⚡ [${i + 1}/${state.generatedPlan.steps.length}] Executing step: ${genStep.name}`
+        );
+
+        const executableStep = convertToExecutableStep(genStep, step.id, step.allowInsecure);
+        const stepContext = {
+          ...dynamicContext,
+          steps: {
+            ...(dynamicContext.steps || {}),
+            ...Object.fromEntries(
+              Array.from(resultsMap.entries()).map(([id, res]) => [
+                `${step.id}_${id}`,
+                { output: res.output },
+              ])
+            ),
+          },
+        } as ExpressionContext;
+
+        if (emitEvent && runId && workflowName) {
+          emitEvent({
+            type: 'step.start',
+            timestamp: new Date().toISOString(),
+            runId,
+            workflow: workflowName,
+            stepId: executableStep.id,
+            stepType: executableStep.type,
+            phase: 'main',
+            stepIndex: i + 1,
+            totalSteps: state.generatedPlan.steps.length,
+          });
+        }
+
+        const res = await executeStepFn(executableStep, stepContext, depth + 1);
+        resultsMap.set(genStep.id, res);
+        state.stepResults.set(genStep.id, res);
+
+        if (stateManager && dbState && dbState.id) {
+          await stateManager.completeStep(dbState.id, genStep.id, res);
+          await stateManager.updateProgress(dbState.id, completed.size + failed.size + 1);
+        } else if (saveState) {
+          await saveState(step.id, state);
+        }
+
+        if (emitEvent && runId && workflowName) {
+          emitEvent({
+            type: 'step.end',
+            timestamp: new Date().toISOString(),
+            runId,
+            workflow: workflowName,
+            stepId: executableStep.id,
+            stepType: executableStep.type,
+            phase: 'main',
+            status: res.status as any,
+            stepIndex: i + 1,
+            totalSteps: state.generatedPlan.steps.length,
+          });
+        }
+
+        if (res.status === 'success') {
+          completed.add(genStep.id);
+        } else {
+          failed.add(genStep.id);
+          if (!(genStep.allowStepFailure ?? step.allowStepFailure ?? false)) {
+            state.status = 'failed';
+            state.error = `Step "${genStep.name}" failed: ${res.error}`;
+          }
+        }
+      } catch (err) {
+        const failRes: StepResult = { status: 'failed', error: String(err), output: {} };
+        resultsMap.set(genStep.id, failRes);
+        state.stepResults.set(genStep.id, failRes);
+        failed.add(genStep.id);
+        state.status = 'failed';
+        state.error = `Step "${genStep.name}" crashed: ${err}`;
+      } finally {
+        running.delete(genStep.id);
+      }
+    };
+
     if (ready.length > 0 && running.size < maxConcurrency) {
       const toStart = ready.slice(0, maxConcurrency - running.size);
       for (const genStep of toStart) {
         running.add(genStep.id);
-        (async () => {
-          try {
-            const i = state.generatedPlan.steps.indexOf(genStep);
-            logger.log(
-              `  ⚡ [${i + 1}/${state.generatedPlan.steps.length}] Executing step: ${genStep.name}`
-            );
-
-            const executableStep = convertToExecutableStep(genStep, step.id, step.allowInsecure);
-            const stepContext = {
-              ...dynamicContext,
-              steps: {
-                ...(dynamicContext.steps || {}),
-                ...Object.fromEntries(
-                  Array.from(resultsMap.entries()).map(([id, res]) => [
-                    `${step.id}_${id}`,
-                    { output: res.output },
-                  ])
-                ),
-              },
-            } as ExpressionContext;
-
-            if (emitEvent && runId && workflowName) {
-              emitEvent({
-                type: 'step.start',
-                timestamp: new Date().toISOString(),
-                runId,
-                workflow: workflowName,
-                stepId: executableStep.id,
-                stepType: executableStep.type,
-                phase: 'main',
-                stepIndex: i + 1,
-                totalSteps: state.generatedPlan.steps.length,
-              });
-            }
-
-            const res = await executeStepFn(executableStep, stepContext);
-            resultsMap.set(genStep.id, res);
-            state.stepResults.set(genStep.id, res);
-
-            if (stateManager && dbState && dbState.id) {
-              await stateManager.completeStep(dbState.id, genStep.id, res);
-              await stateManager.updateProgress(dbState.id, completed.size + failed.size + 1);
-            } else if (saveState) {
-              await saveState(step.id, state);
-            }
-
-            if (emitEvent && runId && workflowName) {
-              emitEvent({
-                type: 'step.end',
-                timestamp: new Date().toISOString(),
-                runId,
-                workflow: workflowName,
-                stepId: executableStep.id,
-                stepType: executableStep.type,
-                phase: 'main',
-                status: res.status as any,
-                stepIndex: i + 1,
-                totalSteps: state.generatedPlan.steps.length,
-              });
-            }
-
-            if (res.status === 'success') {
-              completed.add(genStep.id);
-            } else {
-              failed.add(genStep.id);
-              if (!(genStep.allowStepFailure ?? step.allowStepFailure ?? false)) {
-                state.status = 'failed';
-                state.error = `Step "${genStep.name}" failed: ${res.error}`;
-              }
-            }
-          } catch (err) {
-            const failRes: StepResult = { status: 'failed', error: String(err), output: {} };
-            resultsMap.set(genStep.id, failRes);
-            state.stepResults.set(genStep.id, failRes);
-            failed.add(genStep.id);
-            state.status = 'failed';
-            state.error = `Step "${genStep.name}" crashed: ${err}`;
-          } finally {
-            running.delete(genStep.id);
-          }
-        })();
+        const i = state.generatedPlan.steps.indexOf(genStep);
+        const promise = executeWrapper(genStep, i).then(() => {
+          executionPromises.delete(promise);
+        });
+        executionPromises.add(promise);
       }
     }
-    await new Promise((resolve) => setTimeout(resolve, 100));
+
+    // Wait for at least one task to complete before re-evaluating loop
+    if (executionPromises.size > 0) {
+      await Promise.race(executionPromises);
+    }
   }
 
   const allSatisfied = Array.from(resultsMap.entries()).every(

package/src/runner/executors/engine-executor.ts

@@ -2,6 +2,7 @@ import { spawn } from 'node:child_process';
 import { randomUUID } from 'node:crypto';
 import { existsSync, mkdirSync } from 'node:fs';
 import * as path from 'node:path';
+import { StringDecoder } from 'node:string_decoder';
 import yaml from 'js-yaml';
 import type { ExpressionContext } from '../../expression/evaluator';
 import { ExpressionEvaluator } from '../../expression/evaluator';
@@ -52,44 +53,7 @@ class LRUCache<K, V> {
 }
 
 const VERSION_CACHE = new LRUCache<string, string>(LIMITS.VERSION_CACHE_MAX_SIZE);
-const TRUNCATED_SUFFIX = '... [truncated output]';
-
-function createOutputLimiter(maxBytes: number) {
-  let bytes = 0;
-  let text = '';
-  let truncated = false;
-
-  const append = (chunk: Buffer | string) => {
-    if (truncated || maxBytes <= 0) {
-      truncated = true;
-      return;
-    }
-    const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
-    const remaining = maxBytes - bytes;
-    if (remaining <= 0) {
-      truncated = true;
-      return;
-    }
-    if (buffer.length <= remaining) {
-      text += buffer.toString();
-      bytes += buffer.length;
-      return;
-    }
-    text += buffer.subarray(0, remaining).toString();
-    bytes = maxBytes;
-    truncated = true;
-  };
-
-  const finalize = () => (truncated ? `${text}${TRUNCATED_SUFFIX}` : text);
-
-  return {
-    append,
-    finalize,
-    get truncated() {
-      return truncated;
-    },
-  };
-}
+import { createOutputLimiter } from '../../utils/stream-utils';
 
 export interface EngineExecutionResult {
   stdout: string;
@@ -163,11 +127,17 @@ async function runCommand(
     if (child.stdout) {
       child.stdout.on('data', (chunk: Buffer) => {
         stdoutLimiter.append(chunk);
+        if (stdoutLimiter.truncated) {
+          child.kill();
+        }
       });
     }
     if (child.stderr) {
       child.stderr.on('data', (chunk: Buffer) => {
         stderrLimiter.append(chunk);
+        if (stderrLimiter.truncated) {
+          child.kill();
+        }
       });
     }
 
@@ -208,7 +178,8 @@ async function checkEngineVersion(
   cwd: string,
   abortSignal?: AbortSignal
 ): Promise<string> {
-  const cacheKey = `${command}::${versionArgs.join(' ')}`;
+  const pathEnv = env.PATH || env.Path || env.path || '';
+  const cacheKey = `${command}::${versionArgs.join(' ')}::${cwd}::${pathEnv}`;
   const cached = VERSION_CACHE.get(cacheKey);
   if (cached) return cached;
 

package/src/runner/executors/file-executor.ts

@@ -113,6 +113,73 @@ export function parseUnifiedDiff(patch: string): UnifiedDiff {
 }
 
 export function applyUnifiedDiff(content: string, patch: string, targetPath: string): string {
+  // Try using system `patch` command first as it's more robust
+  try {
+    const { spawnSync } = require('node:child_process');
+
+    // Check if patch is available (quick check)
+    // We assume standard unix `patch` or compatible.
+    // writing content to temp file and patch to temp file?
+    // actually, we can pipe to stdin.
+    // echo content | patch -o output
+    // But patch usually works on files.
+
+    // Since we are operating on in-memory strings (content), using `patch` binary requires tmp files.
+    // This might be slow for many small files.
+    // BUT the robustness is worth it.
+
+    const fs = require('node:fs');
+    const os = require('node:os');
+    const path = require('node:path');
+
+    // Create temp dir
+    const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'keystone-patch-'));
+    const tmpSrc = path.join(tmpDir, 'source');
+    const tmpPatch = path.join(tmpDir, 'changes.patch');
+
+    try {
+      fs.writeFileSync(tmpSrc, content);
+      fs.writeFileSync(tmpPatch, patch);
+
+      // Run patch: patch -p1 -i changes.patch -o output (if headers have paths)
+      // Or just patch tmpSrc < changes.patch?
+      // Unified diffs usually expect paths.
+      // If we force it...
+      // `patch` utility is tricky with paths.
+      // LLM generated diffs might have /dev/null or a/b paths.
+
+      // Let's try `git apply` if inside a git repo?
+      // No, we might not be in a git repo.
+
+      // Let's stick to the JS Custom Parser BUT make it more lenient/robust as per user request?
+      // User said: "rely on the system's patch or git apply"
+
+      // Let's try `patch -u -l --fuzz=2 -i patchfile srcfile -o outfile`
+      const result = spawnSync(
+        'patch',
+        ['-u', '-l', '--fuzz=2', '-i', tmpPatch, tmpSrc, '-o', path.join(tmpDir, 'result')],
+        {
+          encoding: 'utf-8',
+          stdio: 'pipe',
+        }
+      );
+
+      if (result.status === 0 && fs.existsSync(path.join(tmpDir, 'result'))) {
+        return fs.readFileSync(path.join(tmpDir, 'result'), 'utf-8');
+      }
+    } catch (e) {
+      // ignore
+    } finally {
+      // cleanup
+      try {
+        fs.rmSync(tmpDir, { recursive: true, force: true });
+      } catch {}
+    }
+  } catch (e) {
+    // ignore
+  }
+
+  // Fallback to JS implementation
   const diff = parseUnifiedDiff(patch);
   assertDiffMatchesTarget(diff.newFile || diff.originalFile, targetPath);