keystone-cli 1.1.2 → 1.3.0

package/src/parser/workflow-parser.ts

@@ -4,6 +4,7 @@ import { z } from 'zod';
 import { ExpressionEvaluator } from '../expression/evaluator.ts';
 import { ResourceLoader } from '../utils/resource-loader.ts';
 import { validateJsonSchemaDefinition } from '../utils/schema-validator.ts';
+import { topologicalSort } from '../utils/topo-sort.ts';
 import { resolveAgentPath } from './agent-parser.ts';
 import { type Workflow, WorkflowSchema } from './schema.ts';
 
@@ -298,68 +299,8 @@ export class WorkflowParser {
    * Returns steps in execution order
    */
   static topologicalSort(workflow: Workflow): string[] {
-    const stepMap = new Map(workflow.steps.map((step) => [step.id, step.needs]));
-    const inDegree = new Map<string, number>();
-
-    // Validate all dependencies exist before sorting
-    for (const step of workflow.steps) {
-      const needs = step.needs || [];
-      for (const dep of needs) {
-        if (!stepMap.has(dep)) {
-          throw new Error(`Step "${step.id}" depends on non-existent step "${dep}"`);
-        }
-      }
-    }
-
-    // Calculate in-degree
-    // In-degree = number of dependencies a step has
-    for (const step of workflow.steps) {
-      const needs = step.needs || [];
-      inDegree.set(step.id, needs.length);
-    }
-
-    // Build reverse dependency map for O(1) lookups instead of O(n)
-    const dependents = new Map<string, string[]>();
-    for (const step of workflow.steps) {
-      const needs = step.needs || [];
-      for (const dep of needs) {
-        if (!dependents.has(dep)) dependents.set(dep, []);
-        dependents.get(dep)?.push(step.id);
-      }
-    }
-
-    // Kahn's algorithm
-    const queue: string[] = [];
-    const result: string[] = [];
-
-    // Add all nodes with in-degree 0
-    for (const [stepId, degree] of inDegree.entries()) {
-      if (degree === 0) {
-        queue.push(stepId);
-      }
-    }
-
-    let queueIndex = 0;
-    while (queueIndex < queue.length) {
-      const stepId = queue[queueIndex];
-      queueIndex += 1;
-      result.push(stepId);
-
-      // Find all steps that depend on this step (O(1) lookup)
-      for (const dependentId of dependents.get(stepId) || []) {
-        const newDegree = (inDegree.get(dependentId) || 0) - 1;
-        inDegree.set(dependentId, newDegree);
-        if (newDegree === 0) {
-          queue.push(dependentId);
-        }
-      }
-    }
-
-    if (result.length !== workflow.steps.length) {
-      throw new Error('Topological sort failed - circular dependency detected');
-    }
-
-    return result;
+    const sorted = topologicalSort(workflow.steps);
+    return sorted.map((s) => s.id);
   }
 
   /**

package/src/runner/executors/dynamic-executor.test.ts

@@ -0,0 +1,613 @@
+/**
+ * Tests for the Dynamic Step Executor
+ */
+import { describe, expect, it, mock } from 'bun:test';
+import type { ExpressionContext } from '../../expression/evaluator.ts';
+import type { DynamicStep, Step } from '../../parser/schema.ts';
+import { SilentLogger } from '../../utils/logger.ts';
+import { DYNAMIC_STEP_OUTPUT_SCHEMA, executeDynamicStep } from './dynamic-executor.ts';
+import type { DynamicPlan, DynamicStepState } from './dynamic-types.ts';
+import type { StepResult } from './types.ts';
+
+describe('DynamicStepExecutor', () => {
+  const logger = new SilentLogger();
+  const baseContext: ExpressionContext = {
+    inputs: {},
+    env: {},
+    steps: {},
+  };
+
+  // Mock step executor
+  const mockExecuteStepFn = async (
+    step: Step,
+    _context: ExpressionContext
+  ): Promise<StepResult> => {
+    return {
+      output: { executed: step.id, type: step.type },
+      status: 'success',
+    };
+  };
+
+  // Mock LLM step that returns a plan
+  const createMockLlmExecutor = (plan: DynamicPlan) => {
+    return async () => ({
+      output: plan,
+      status: 'success' as const,
+    });
+  };
+
+  describe('DYNAMIC_STEP_OUTPUT_SCHEMA', () => {
+    it('should define the expected schema structure', () => {
+      expect(DYNAMIC_STEP_OUTPUT_SCHEMA.type).toBe('object');
+      expect(DYNAMIC_STEP_OUTPUT_SCHEMA.properties.steps).toBeDefined();
+      expect(DYNAMIC_STEP_OUTPUT_SCHEMA.required).toContain('steps');
+    });
+  });
+
+  describe('executeDynamicStep', () => {
+    it('should generate and execute a plan', async () => {
+      const step: DynamicStep = {
+        id: 'test-dynamic',
+        type: 'dynamic',
+        goal: 'Create a simple test file',
+        agent: 'keystone-architect',
+        needs: [],
+        maxSteps: 10,
+        maxIterations: 5,
+        allowStepFailure: false,
+        concurrency: 1,
+        confirmPlan: false,
+        maxReplans: 0,
+      };
+
+      const mockPlan: DynamicPlan = {
+        steps: [
+          { id: 'step1', name: 'Create file', type: 'shell', run: 'touch test.txt' },
+          {
+            id: 'step2',
+            name: 'Write content',
+            type: 'shell',
+            run: 'echo hello > test.txt',
+            needs: ['step1'],
+          },
+        ],
+        notes: 'Simple two-step plan',
+      };
+
+      const result = await executeDynamicStep(step, baseContext, mockExecuteStepFn, logger, {
+        executeLlmStep: createMockLlmExecutor(mockPlan),
+      });
+
+      expect(result.status).toBe('success');
+      expect(result.output).toBeDefined();
+
+      const output = result.output as { plan: DynamicPlan; summary: { total: number } };
+      expect(output.plan.steps.length).toBe(2);
+      expect(output.summary.total).toBe(2);
+    });
+
+    it('should handle planning failure', async () => {
+      const step: DynamicStep = {
+        id: 'test-fail',
+        type: 'dynamic',
+        goal: 'This will fail',
+        agent: 'keystone-architect',
+        needs: [],
+        maxSteps: 10,
+        maxIterations: 5,
+        allowStepFailure: false,
+        concurrency: 1,
+        confirmPlan: false,
+        maxReplans: 0,
+      };
+
+      const mockFailingLlm = async () => ({
+        output: null,
+        status: 'failed' as const,
+        error: 'LLM failed',
+      });
+
+      const result = await executeDynamicStep(step, baseContext, mockExecuteStepFn, logger, {
+        executeLlmStep: mockFailingLlm,
+      });
+
+      expect(result.status).toBe('failed');
+      expect(result.error).toBe('LLM failed');
+    });
+
+    it('should stop on step failure when allowStepFailure is false', async () => {
+      const step: DynamicStep = {
+        id: 'test-step-fail',
+        type: 'dynamic',
+        goal: 'Run steps that fail',
+        agent: 'keystone-architect',
+        needs: [],
+        maxSteps: 10,
+        maxIterations: 5,
+        allowStepFailure: false,
+        concurrency: 1,
+        confirmPlan: false,
+        maxReplans: 0,
+      };
+
+      const mockPlan: DynamicPlan = {
+        steps: [
+          { id: 'step1', name: 'First step', type: 'shell', run: 'echo ok' },
+          { id: 'step2', name: 'Failing step', type: 'shell', run: 'exit 1', needs: ['step1'] },
+          { id: 'step3', name: 'Never reached', type: 'shell', run: 'echo done', needs: ['step2'] },
+        ],
+      };
+
+      const failingExecutor = async (
+        step: Step,
+        _context: ExpressionContext
+      ): Promise<StepResult> => {
+        if (step.id.includes('step2')) {
+          return { output: null, status: 'failed', error: 'Command failed' };
+        }
+        return { output: { done: true }, status: 'success' };
+      };
+
+      const result = await executeDynamicStep(step, baseContext, failingExecutor, logger, {
+        executeLlmStep: createMockLlmExecutor(mockPlan),
+      });
+
+      expect(result.status).toBe('failed');
+      expect(result.error).toContain('Failing step');
+    });
+
+    it('should continue on step failure when allowStepFailure is true', async () => {
+      const step: DynamicStep = {
+        id: 'test-allow-fail',
+        type: 'dynamic',
+        goal: 'Run steps that fail but continue',
+        agent: 'keystone-architect',
+        needs: [],
+        maxSteps: 10,
+        maxIterations: 5,
+        allowStepFailure: true,
+        concurrency: 1,
+        confirmPlan: false,
+        maxReplans: 0,
+      };
+
+      const mockPlan: DynamicPlan = {
+        steps: [
+          { id: 'step1', name: 'First step', type: 'shell', run: 'echo ok' },
+          { id: 'step2', name: 'Failing step', type: 'shell', run: 'exit 1' }, // No dependency
+          { id: 'step3', name: 'Still runs', type: 'shell', run: 'echo done' }, // No dependency on step2
+        ],
+      };
+
+      const failingExecutor = async (
+        step: Step,
+        _context: ExpressionContext
+      ): Promise<StepResult> => {
+        if (step.id.includes('step2')) {
+          return { output: null, status: 'failed', error: 'Command failed' };
+        }
+        return { output: { done: true }, status: 'success' };
+      };
+
+      const result = await executeDynamicStep(step, baseContext, failingExecutor, logger, {
+        executeLlmStep: createMockLlmExecutor(mockPlan),
+      });
+
+      // Should succeed because allowStepFailure is true
+      expect(result.status).toBe('success');
+
+      // But all steps should have been attempted
+      const output = result.output as {
+        summary: { total: number; succeeded: number; failed: number };
+      };
+      expect(output.summary.total).toBe(3);
+      expect(output.summary.succeeded).toBe(2);
+      expect(output.summary.failed).toBe(1);
+    });
+
+    it('should support state persistence for resumability', async () => {
+      const step: DynamicStep = {
+        id: 'test-resume',
+        type: 'dynamic',
+        goal: 'Resumable workflow',
+        agent: 'keystone-architect',
+        needs: [],
+        maxSteps: 10,
+        maxIterations: 5,
+        allowStepFailure: false,
+        concurrency: 1,
+        confirmPlan: false,
+        maxReplans: 0,
+      };
+
+      const mockPlan: DynamicPlan = {
+        steps: [
+          { id: 'step1', name: 'First', type: 'shell', run: 'echo 1' },
+          { id: 'step2', name: 'Second', type: 'shell', run: 'echo 2', needs: ['step1'] },
+        ],
+      };
+
+      // Track saved states
+      const savedStates: DynamicStepState[] = [];
+      const saveState = async (_stepId: string, state: DynamicStepState) => {
+        savedStates.push({ ...state, stepResults: new Map(state.stepResults) });
+      };
+
+      await executeDynamicStep(step, baseContext, mockExecuteStepFn, logger, {
+        executeLlmStep: createMockLlmExecutor(mockPlan),
+        saveState,
+      });
+
+      // Should have saved state multiple times:
+      // 1. After planning
+      // 2. After each step
+      // 3. After completion
+      expect(savedStates.length).toBeGreaterThanOrEqual(3);
+      expect(savedStates[savedStates.length - 1].status).toBe('completed');
+    });
+
+    it('should detect circular dependencies', async () => {
+      const step: DynamicStep = {
+        id: 'test-circular',
+        type: 'dynamic',
+        goal: 'Circular deps test',
+        agent: 'keystone-architect',
+        needs: [],
+        maxSteps: 10,
+        maxIterations: 5,
+        allowStepFailure: false,
+        concurrency: 1,
+        confirmPlan: false,
+        maxReplans: 0,
+      };
+
+      const mockPlan: DynamicPlan = {
+        steps: [
+          { id: 'step1', name: 'First', type: 'shell', run: 'echo 1', needs: ['step2'] },
+          { id: 'step2', name: 'Second', type: 'shell', run: 'echo 2', needs: ['step1'] }, // Circular!
+        ],
+      };
+
+      const result = await executeDynamicStep(step, baseContext, mockExecuteStepFn, logger, {
+        executeLlmStep: createMockLlmExecutor(mockPlan),
+      });
+
+      expect(result.status).toBe('failed');
+      expect(result.error).toContain('Circular dependency');
+    });
+
+    it('should execute independent steps in parallel', async () => {
+      const step: DynamicStep = {
+        id: 'test-parallel',
+        type: 'dynamic',
+        goal: 'Parallel execution test',
+        agent: 'keystone-architect',
+        needs: [],
+        concurrency: 2,
+        maxSteps: 10,
+        maxIterations: 5,
+        allowStepFailure: false,
+        confirmPlan: false,
+        maxReplans: 0,
+      };
+
+      const mockPlan: DynamicPlan = {
+        steps: [
+          { id: 'step1', name: 'Task 1', type: 'shell', run: 'sleep 0.1' },
+          { id: 'step2', name: 'Task 2', type: 'shell', run: 'sleep 0.1' },
+        ],
+      };
+
+      // Track execution timing
+      const startTimes: Map<string, number> = new Map();
+      const delayedExecutor = async (s: Step, _context: ExpressionContext): Promise<StepResult> => {
+        startTimes.set(s.id, Date.now());
+        await new Promise((resolve) => setTimeout(resolve, 200));
+        return { output: { id: s.id }, status: 'success' };
+      };
+
+      const startTime = Date.now();
+      const result = await executeDynamicStep(step, baseContext, delayedExecutor, logger, {
+        executeLlmStep: createMockLlmExecutor(mockPlan),
+      });
+      const endTime = Date.now();
+
+      expect(result.status).toBe('success');
+
+      // Both steps should have started around the same time
+      const t1 = startTimes.get('test-parallel_step1');
+      const t2 = startTimes.get('test-parallel_step2');
+      expect(t1).toBeDefined();
+      expect(t2).toBeDefined();
+      expect(Math.abs((t1 ?? 0) - (t2 ?? 0))).toBeLessThan(150);
+
+      // Total time should be significantly less than serial (400ms+)
+      expect(endTime - startTime).toBeLessThan(450);
+    });
+
+    it('should respect dependencies during parallel execution', async () => {
+      const step: DynamicStep = {
+        id: 'test-parallel-deps',
+        type: 'dynamic',
+        goal: 'Parallel dependencies test',
+        agent: 'keystone-architect',
+        needs: [],
+        concurrency: 2,
+        maxSteps: 10,
+        maxIterations: 5,
+        allowStepFailure: false,
+        confirmPlan: false,
+        maxReplans: 0,
+      };
+
+      const mockPlan: DynamicPlan = {
+        steps: [
+          { id: 'step1', name: 'Dependency', type: 'shell', run: 'sleep 0.1' },
+          { id: 'step2', name: 'Dependent', type: 'shell', run: 'sleep 0.1', needs: ['step1'] },
+        ],
+      };
+
+      const startTimes: Map<string, number> = new Map();
+      const finishTimes: Map<string, number> = new Map();
+
+      const delayedExecutor = async (s: Step, _context: ExpressionContext): Promise<StepResult> => {
+        startTimes.set(s.id, Date.now());
+        await new Promise((resolve) => setTimeout(resolve, 200));
+        finishTimes.set(s.id, Date.now());
+        return { output: { id: s.id }, status: 'success' };
+      };
+
+      await executeDynamicStep(step, baseContext, delayedExecutor, logger, {
+        executeLlmStep: createMockLlmExecutor(mockPlan),
+      });
+
+      const t1Start = startTimes.get('test-parallel-deps_step1') ?? 0;
+      const t1End = finishTimes.get('test-parallel-deps_step1') ?? 0;
+      const t2Start = startTimes.get('test-parallel-deps_step2') ?? 0;
+
+      // Step 2 must start AFTER Step 1 finishes
+      expect(t2Start).toBeGreaterThanOrEqual(t1End);
+    });
+    it('should support planning gate and confirmation', async () => {
+      const step: DynamicStep = {
+        id: 'test-gate',
+        type: 'dynamic',
+        goal: 'Test gate',
+        agent: 'keystone-architect',
+        needs: [],
+        confirmPlan: true,
+        maxSteps: 10,
+        maxIterations: 5,
+        allowStepFailure: false,
+        concurrency: 1,
+        maxReplans: 0,
+      };
+
+      const mockPlan: DynamicPlan = {
+        steps: [{ id: 'step1', name: 'Original Step', type: 'shell', run: 'echo original' }],
+      };
+
+      let confirmed = false;
+      const mockExecuteHumanStep = async () => {
+        confirmed = true;
+        return { status: 'success' as const, output: 'yes' };
+      };
+
+      const result = await executeDynamicStep(step, baseContext, mockExecuteStepFn, logger, {
+        executeLlmStep: createMockLlmExecutor(mockPlan),
+        executeHumanStep: mockExecuteHumanStep,
+      });
+
+      expect(result.status).toBe('success');
+      expect(confirmed).toBe(true);
+    });
+
+    it('should support plan modification via planning gate', async () => {
+      const step: DynamicStep = {
+        id: 'test-modify',
+        type: 'dynamic',
+        goal: 'Test modify',
+        agent: 'keystone-architect',
+        needs: [],
+        confirmPlan: true,
+        maxSteps: 10,
+        maxIterations: 5,
+        allowStepFailure: false,
+        concurrency: 1,
+        maxReplans: 0,
+      };
+
+      const mockPlan: DynamicPlan = {
+        steps: [{ id: 'step1', name: 'Original Step', type: 'shell', run: 'echo original' }],
+      };
+
+      const modifiedPlan: DynamicPlan = {
+        steps: [
+          { id: 'modified-step', name: 'Modified Step', type: 'shell', run: 'echo modified' },
+        ],
+      };
+
+      const mockExecuteHumanStep = async () => {
+        return { status: 'success' as const, output: JSON.stringify(modifiedPlan) };
+      };
+
+      const executedSteps: string[] = [];
+      const customExecutor = async (s: Step) => {
+        executedSteps.push(s.id);
+        return { status: 'success' as const, output: {} };
+      };
+
+      const result = await executeDynamicStep(step, baseContext, customExecutor, logger, {
+        executeLlmStep: createMockLlmExecutor(mockPlan),
+        executeHumanStep: mockExecuteHumanStep,
+      });
+
+      expect(result.status).toBe('success');
+      expect(executedSteps).toContain('test-modify_modified-step');
+      expect(executedSteps).not.toContain('test-modify_step1');
+    });
+
+    it('should attempt re-planning when execution fails (Self-Correction)', async () => {
+      const step: DynamicStep = {
+        id: 'test-replan',
+        type: 'dynamic',
+        goal: 'Try something that fails first',
+        needs: [],
+        agent: 'keystone-architect',
+        maxSteps: 10,
+        maxIterations: 5,
+        allowStepFailure: false,
+        concurrency: 1,
+        confirmPlan: false,
+        maxReplans: 1,
+      };
+
+      const mockPlan1: DynamicPlan = {
+        steps: [{ id: 'fail-step', name: 'Step that fails', type: 'llm', prompt: 'fail' }],
+      };
+
+      const mockPlan2: DynamicPlan = {
+        steps: [{ id: 'fix-step', name: 'Step that fixes', type: 'llm', prompt: 'fix' }],
+      };
+
+      let planAttempt = 0;
+      const mockLlmExecutor = async (s: Step) => {
+        if (s.id.includes('supervisor')) {
+          planAttempt++;
+          return { status: 'success' as const, output: planAttempt === 1 ? mockPlan1 : mockPlan2 };
+        }
+        if (s.id.includes('fail-step')) {
+          return { status: 'failed' as const, error: 'First attempt failed', output: {} };
+        }
+        return { status: 'success' as const, output: { fixed: true } };
+      };
+
+      const result = (await executeDynamicStep(
+        step,
+        baseContext,
+        async (s) => mockLlmExecutor(s),
+        logger,
+        {
+          executeLlmStep: mockLlmExecutor,
+        }
+      )) as any;
+
+      expect(result.status).toBe('success');
+      expect(planAttempt).toBe(2);
+      expect(result.output.summary.replans).toBe(1);
+      expect(result.output.results['fix-step'].status).toBe('success');
+    });
+  });
+
+  it('should resume execution from a partially completed state using dbState', async () => {
+    const step: DynamicStep = {
+      id: 'test-resume-db',
+      type: 'dynamic',
+      goal: 'Resumable workflow',
+      agent: 'keystone-architect',
+      needs: [],
+      maxSteps: 10,
+      maxIterations: 5,
+      allowStepFailure: false,
+      concurrency: 1,
+      confirmPlan: false,
+      maxReplans: 0,
+    };
+
+    const mockPlan: DynamicPlan = {
+      steps: [
+        { id: 'step1', name: 'First', type: 'shell', run: 'echo 1' },
+        { id: 'step2', name: 'Second', type: 'shell', run: 'echo 2', needs: ['step1'] },
+      ],
+    };
+
+    // Mock DB state where step1 is already completed
+    const mockDbState: DynamicStepState = {
+      id: 'db-state-id',
+      workflowId: 'test-run-id',
+      runId: 'test-run-id',
+      stepId: step.id,
+      status: 'executing',
+      generatedPlan: mockPlan,
+      currentStepIndex: 0,
+      stepResults: new Map([
+        [
+          'step1',
+          { status: 'success', output: { executed: 'test-resume-db_step1', type: 'shell' } },
+        ],
+      ]),
+      startedAt: new Date().toISOString(),
+      replanCount: 0,
+    };
+
+    const mockStateManager = {
+      load: mock()
+        .mockResolvedValueOnce(mockDbState) // First call returns state
+        .mockResolvedValueOnce(mockDbState), // Subsequent calls
+      getStepResultsMap: mock().mockResolvedValue(mockDbState.stepResults),
+      completeStep: mock().mockResolvedValue(undefined),
+      updateProgress: mock().mockResolvedValue(undefined),
+      finish: mock().mockResolvedValue(undefined),
+      updateStatus: mock().mockResolvedValue(undefined),
+      setPlan: mock().mockResolvedValue(undefined),
+    };
+
+    const executedSteps: string[] = [];
+    const customExecutor = async (s: Step) => {
+      executedSteps.push(s.id);
+      return { status: 'success' as const, output: { executed: s.id } };
+    };
+
+    const result = await executeDynamicStep(step, baseContext, customExecutor, logger, {
+      executeLlmStep: createMockLlmExecutor(mockPlan),
+      stateManager: mockStateManager as any,
+      runId: 'test-run-id',
+    });
+
+    expect(result.status).toBe('success');
+    // Should ONLY execute step 2, since step 1 was already in state
+    expect(executedSteps).toContain('test-resume-db_step2');
+    expect(executedSteps).not.toContain('test-resume-db_step1');
+    expect(mockStateManager.load).toHaveBeenCalled();
+  });
+
+  it('should handle invalid plan structure gracefully', async () => {
+    const step: DynamicStep = {
+      id: 'test-invalid-plan',
+      type: 'dynamic',
+      goal: 'Invalid plan',
+      agent: 'keystone-architect',
+      needs: [],
+      maxSteps: 10,
+      maxIterations: 5,
+      allowStepFailure: false,
+      concurrency: 1,
+      confirmPlan: false,
+      maxReplans: 0,
+    };
+
+    // Plan missing required fields or invalid types
+    const invalidPlan: any = {
+      steps: [
+        { id: 'step1' }, // Missing type, name
+      ],
+    };
+
+    const result = await executeDynamicStep(step, baseContext, mockExecuteStepFn, logger, {
+      executeLlmStep: createMockLlmExecutor(invalidPlan),
+    });
+
+    // Should fail because convertToExecutableStep will default to shell/echo error,
+    // but the system should not crash.
+    // Actually, currently it defaults to an echo command "Unknown step type: undefined"
+    // So it might return success if that echo command "succeeds" (is mocked).
+
+    // Wait, generated.type is checked. If missing, it goes to default 'shell'.
+    // { id: 'step1' } -> type undefined -> default case.
+
+    expect(result.status).toBe('success');
+    const plan = (result.output as any).plan;
+    expect(plan.steps).toBeDefined();
+    // Verify that it didn't crash
+  });
+});