keystone-cli 1.2.0 → 1.3.0

package/src/db/dynamic-state-manager.ts

@@ -0,0 +1,411 @@
+/**
+ * Dynamic State Manager
+ *
+ * Manages persistence for dynamic workflow steps, enabling resumability
+ * of LLM-orchestrated workflows.
+ */
+
+import type { Database } from 'bun:sqlite';
+import { randomUUID } from 'node:crypto';
+import type {
+  DynamicPlan,
+  DynamicStepExecution,
+  DynamicStepState,
+  GeneratedStep,
+} from '../runner/executors/dynamic-types.ts';
+
+export type { DynamicPlan, DynamicStepExecution, DynamicStepState, GeneratedStep };
+import type { WorkflowDb } from './workflow-db.ts';
+
+/**
+ * Raw row from dynamic_workflow_state table
+ */
+interface DynamicStateRow {
+  id: string;
+  run_id: string;
+  step_id: string;
+  workflow_id: string | null;
+  status: string;
+  generated_plan: string;
+  current_step_index: number;
+  started_at: string;
+  completed_at: string | null;
+  error: string | null;
+  metadata: string | null;
+  replan_count: number;
+}
+
+/**
+ * Raw row from dynamic_step_executions table
+ */
+interface DynamicExecRow {
+  id: string;
+  state_id: string;
+  step_id: string;
+  step_name: string;
+  step_type: string;
+  step_definition: string;
+  status: string;
+  output: string | null;
+  error: string | null;
+  started_at: string | null;
+  completed_at: string | null;
+  execution_order: number;
+}
+
+/**
+ * Manages persistence for dynamic workflow steps
+ */
+export class DynamicStateManager {
+  constructor(private db: WorkflowDb) {}
+
+  /**
+   * Access the underlying database for direct queries
+   */
+  private getDatabase(): Database {
+    return this.db.getDatabase();
+  }
+
+  /**
+   * Create a new dynamic state record
+   */
+  async create(params: {
+    runId: string;
+    stepId: string;
+    workflowId?: string;
+  }): Promise<DynamicStepState> {
+    const id = randomUUID();
+    const now = new Date().toISOString();
+
+    const state: DynamicStepState = {
+      id,
+      runId: params.runId,
+      stepId: params.stepId,
+      workflowId: params.workflowId || params.runId, // Fallback to runId if workflowId not provided
+      status: 'planning',
+      generatedPlan: { steps: [] },
+      currentStepIndex: 0,
+      stepResults: new Map(),
+      startedAt: now,
+      replanCount: 0,
+    };
+
+    const db = this.getDatabase();
+    db.prepare(`
+      INSERT INTO dynamic_workflow_state 
+      (id, run_id, step_id, workflow_id, status, generated_plan, current_step_index, started_at, replan_count)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `).run(
+      id,
+      params.runId,
+      params.stepId,
+      params.workflowId || null,
+      'planning',
+      JSON.stringify(state.generatedPlan),
+      0,
+      now,
+      0
+    );
+
+    return state;
+  }
+
+  /**
+   * Load existing state for resume
+   */
+  async load(runId: string, stepId: string): Promise<DynamicStepState | null> {
+    const db = this.getDatabase();
+    const row = db
+      .prepare(`
+      SELECT * FROM dynamic_workflow_state
+      WHERE run_id = ? AND step_id = ?
+    `)
+      .get(runId, stepId) as DynamicStateRow | null;
+
+    if (!row) return null;
+
+    return this.rowToState(row);
+  }
+
+  /**
+   * Load state by ID
+   */
+  async loadById(id: string): Promise<DynamicStepState | null> {
+    const db = this.getDatabase();
+    const row = db
+      .prepare(`
+      SELECT * FROM dynamic_workflow_state WHERE id = ?
+    `)
+      .get(id) as DynamicStateRow | null;
+
+    if (!row) return null;
+
+    return this.rowToState(row);
+  }
+
+  /**
+   * Convert a database row to DynamicStepState
+   */
+  private rowToState(row: DynamicStateRow): DynamicStepState {
+    return {
+      id: row.id,
+      runId: row.run_id,
+      stepId: row.step_id,
+      workflowId: row.workflow_id || row.run_id,
+      status: row.status as DynamicStepState['status'],
+      generatedPlan: JSON.parse(row.generated_plan),
+      currentStepIndex: row.current_step_index,
+      stepResults: new Map(),
+      startedAt: row.started_at,
+      completedAt: row.completed_at || undefined,
+      error: row.error || undefined,
+      metadata: row.metadata ? JSON.parse(row.metadata) : undefined,
+      replanCount: row.replan_count || 0,
+    };
+  }
+
+  /**
+   * Update state after plan generation
+   */
+  async setPlan(
+    stateId: string,
+    plan: DynamicPlan,
+    status: DynamicStepState['status'] = 'executing'
+  ): Promise<void> {
+    const db = this.getDatabase();
+    const now = new Date().toISOString();
+
+    // Load current state to get replanCount
+    const current = db
+      .prepare('SELECT replan_count FROM dynamic_workflow_state WHERE id = ?')
+      .get(stateId) as { replan_count: number };
+    const replanCount = current?.replan_count || 0;
+
+    db.prepare(`
+      UPDATE dynamic_workflow_state
+      SET generated_plan = ?, status = ?, updated_at = ?, replan_count = ?
+      WHERE id = ?
+    `).run(JSON.stringify(plan), status, now, replanCount, stateId);
+
+    // Delete previous execution records IF this is a re-plan (optional, but cleaner)
+    if (replanCount > 0) {
+      db.prepare('DELETE FROM dynamic_step_executions WHERE state_id = ?').run(stateId);
+    }
+
+    // Create execution records for each step
+    for (let i = 0; i < plan.steps.length; i++) {
+      const step = plan.steps[i];
+      db.prepare(`
+        INSERT INTO dynamic_step_executions
+        (id, state_id, step_id, step_name, step_type, step_definition, status, execution_order)
+        VALUES (?, ?, ?, ?, ?, ?, 'pending', ?)
+      `).run(randomUUID(), stateId, step.id, step.name, step.type, JSON.stringify(step), i);
+    }
+  }
+
+  /**
+   * Update status and optionally replan count
+   */
+  async updateStatus(
+    stateId: string,
+    status: DynamicStepState['status'],
+    replanCount?: number
+  ): Promise<void> {
+    const db = this.getDatabase();
+    const now = new Date().toISOString();
+
+    if (replanCount !== undefined) {
+      db.prepare(`
+        UPDATE dynamic_workflow_state
+        SET status = ?, replan_count = ?, updated_at = ?
+        WHERE id = ?
+      `).run(status, replanCount, now, stateId);
+    } else {
+      db.prepare(`
+        UPDATE dynamic_workflow_state
+        SET status = ?, updated_at = ?
+        WHERE id = ?
+      `).run(status, now, stateId);
+    }
+  }
+
+  /**
+   * Mark plan as confirmed and ready for execution
+   */
+  async confirmPlan(stateId: string): Promise<void> {
+    const db = this.getDatabase();
+    const now = new Date().toISOString();
+
+    db.prepare(`
+      UPDATE dynamic_workflow_state
+      SET status = 'executing', updated_at = ?
+      WHERE id = ?
+    `).run(now, stateId);
+  }
+
+  /**
+   * Update current step index (for resume)
+   */
+  async updateProgress(stateId: string, stepIndex: number): Promise<void> {
+    const db = this.getDatabase();
+    const now = new Date().toISOString();
+
+    db.prepare(`
+      UPDATE dynamic_workflow_state
+      SET current_step_index = ?, updated_at = ?
+      WHERE id = ?
+    `).run(stepIndex, now, stateId);
+  }
+
+  /**
+   * Record step execution start
+   */
+  async startStep(stateId: string, stepId: string): Promise<void> {
+    const db = this.getDatabase();
+    const now = new Date().toISOString();
+
+    db.prepare(`
+      UPDATE dynamic_step_executions
+      SET status = 'running', started_at = ?
+      WHERE state_id = ? AND step_id = ?
+    `).run(now, stateId, stepId);
+  }
+
+  /**
+   * Record step completion
+   */
+  async completeStep(
+    stateId: string,
+    stepId: string,
+    result: { status: string; output?: unknown; error?: string }
+  ): Promise<void> {
+    const db = this.getDatabase();
+    const now = new Date().toISOString();
+
+    // Map result.status to our status enum
+    let status: string;
+    switch (result.status) {
+      case 'success':
+        status = 'success';
+        break;
+      case 'failed':
+        status = 'failed';
+        break;
+      case 'skipped':
+        status = 'skipped';
+        break;
+      default:
+        status = 'failed';
+    }
+
+    db.prepare(`
+      UPDATE dynamic_step_executions
+      SET status = ?, output = ?, error = ?, completed_at = ?
+      WHERE state_id = ? AND step_id = ?
+    `).run(
+      status,
+      result.output !== undefined ? JSON.stringify(result.output) : null,
+      result.error || null,
+      now,
+      stateId,
+      stepId
+    );
+  }
+
+  /**
+   * Mark workflow as completed or failed
+   */
+  async finish(stateId: string, status: 'completed' | 'failed', error?: string): Promise<void> {
+    const db = this.getDatabase();
+    const now = new Date().toISOString();
+
+    db.prepare(`
+      UPDATE dynamic_workflow_state
+      SET status = ?, error = ?, completed_at = ?, updated_at = ?
+      WHERE id = ?
+    `).run(status, error || null, now, now, stateId);
+  }
+
+  /**
+   * Get all step executions for a dynamic state
+   */
+  async getStepExecutions(stateId: string): Promise<DynamicStepExecution[]> {
+    const db = this.getDatabase();
+    const rows = db
+      .prepare(`
+      SELECT * FROM dynamic_step_executions
+      WHERE state_id = ?
+      ORDER BY execution_order ASC
+    `)
+      .all(stateId) as DynamicExecRow[];
+
+    return rows.map((row) => ({
+      id: row.id,
+      stateId: row.state_id,
+      stepId: row.step_id,
+      stepName: row.step_name,
+      stepType: row.step_type,
+      stepDefinition: JSON.parse(row.step_definition),
+      status: row.status as DynamicStepExecution['status'],
+      output: row.output ? JSON.parse(row.output) : undefined,
+      error: row.error || undefined,
+      startedAt: row.started_at || undefined,
+      completedAt: row.completed_at || undefined,
+      executionOrder: row.execution_order,
+    }));
+  }
+
+  /**
+   * Get step results as a Map (for executor integration)
+   */
+  async getStepResultsMap(
+    stateId: string
+  ): Promise<Map<string, { output: unknown; status: string; error?: string }>> {
+    const executions = await this.getStepExecutions(stateId);
+    const map = new Map<string, { output: unknown; status: string; error?: string }>();
+
+    for (const exec of executions) {
+      if (exec.status !== 'pending') {
+        map.set(exec.stepId, {
+          output: exec.output,
+          status: exec.status,
+          error: exec.error,
+        });
+      }
+    }
+
+    return map;
+  }
+
+  /**
+   * List all active (non-completed) dynamic states
+   */
+  async listActive(): Promise<DynamicStepState[]> {
+    const db = this.getDatabase();
+    const rows = db
+      .prepare(`
+      SELECT * FROM dynamic_workflow_state
+      WHERE status IN ('planning', 'executing')
+      ORDER BY started_at DESC
+    `)
+      .all() as DynamicStateRow[];
+
+    return rows.map((row) => this.rowToState(row));
+  }
+
+  /**
+   * List dynamic states by run ID
+   */
+  async listByRun(runId: string): Promise<DynamicStepState[]> {
+    const db = this.getDatabase();
+    const rows = db
+      .prepare(`
+      SELECT * FROM dynamic_workflow_state
+      WHERE run_id = ?
+      ORDER BY started_at ASC
+    `)
+      .all(runId) as DynamicStateRow[];
+
+    return rows.map((row) => this.rowToState(row));
+  }
+}

package/src/db/workflow-db.ts

@@ -101,6 +101,14 @@ export class DatabaseError extends Error {
 export class WorkflowDb {
   private db: Database;
 
+  /**
+   * Access the underlying database instance.
+   * This is intended for internal extensions like DynamicStateManager.
+   */
+  public getDatabase(): Database {
+    return this.db;
+  }
+
   private createRunStmt!: Statement;
   private updateRunStatusStmt!: Statement;
   private getRunStmt!: Statement;
@@ -549,6 +557,62 @@ export class WorkflowDb {
         PRAGMA user_version = 4;
       `);
     }
+
+    // Version 5: Add dynamic workflow state tables
+    if (version < 5) {
+      this.db.exec(`
+        -- Table for tracking dynamic step execution state
+        CREATE TABLE IF NOT EXISTS dynamic_workflow_state (
+          id TEXT PRIMARY KEY,
+          run_id TEXT NOT NULL,
+          step_id TEXT NOT NULL,
+          workflow_id TEXT,
+          status TEXT NOT NULL DEFAULT 'planning',
+          generated_plan TEXT NOT NULL DEFAULT '{"steps":[]}',
+          current_step_index INTEGER DEFAULT 0,
+          started_at TEXT NOT NULL,
+          completed_at TEXT,
+          error TEXT,
+          metadata TEXT,
+          created_at TEXT DEFAULT CURRENT_TIMESTAMP,
+          updated_at TEXT DEFAULT CURRENT_TIMESTAMP,
+          FOREIGN KEY (run_id) REFERENCES workflow_runs(id) ON DELETE CASCADE,
+          UNIQUE (run_id, step_id)
+        );
+        CREATE INDEX IF NOT EXISTS idx_dynamic_state_run ON dynamic_workflow_state(run_id);
+        CREATE INDEX IF NOT EXISTS idx_dynamic_state_status ON dynamic_workflow_state(status);
+
+        -- Table for individual generated step executions
+        CREATE TABLE IF NOT EXISTS dynamic_step_executions (
+          id TEXT PRIMARY KEY,
+          state_id TEXT NOT NULL,
+          step_id TEXT NOT NULL,
+          step_name TEXT NOT NULL,
+          step_type TEXT NOT NULL,
+          step_definition TEXT NOT NULL,
+          status TEXT NOT NULL DEFAULT 'pending',
+          output TEXT,
+          error TEXT,
+          started_at TEXT,
+          completed_at TEXT,
+          execution_order INTEGER NOT NULL,
+          FOREIGN KEY (state_id) REFERENCES dynamic_workflow_state(id) ON DELETE CASCADE,
+          UNIQUE (state_id, step_id)
+        );
+        CREATE INDEX IF NOT EXISTS idx_dynamic_exec_state ON dynamic_step_executions(state_id);
+        CREATE INDEX IF NOT EXISTS idx_dynamic_exec_order ON dynamic_step_executions(state_id, execution_order);
+        
+        PRAGMA user_version = 5;
+      `);
+    }
+
+    // Version 6: Add replan_count to dynamic_workflow_state
+    if (version < 6) {
+      this.db.exec(`
+        ALTER TABLE dynamic_workflow_state ADD COLUMN replan_count INTEGER DEFAULT 0;
+        PRAGMA user_version = 6;
+      `);
+    }
   }
 
   private initSchema(): void {

package/src/parser/schema.ts

@@ -415,6 +415,35 @@ const WaitStepSchema = BaseStepSchema.extend({
   // timeout is already in BaseStepSchema, but let's make it explicit here if needed
 });
 
+const DynamicStepSchema = BaseStepSchema.extend({
+  type: z.literal('dynamic'),
+  goal: z.string(), // The high-level goal to accomplish
+  context: z.string().optional(), // Additional context for the supervisor
+  prompt: z.string().optional(), // Custom supervisor prompt (overrides default)
+  supervisor: z.string().optional(), // Supervisor agent (defaults to agent or keystone-architect)
+  agent: z.string().optional().default('keystone-architect'), // Default agent for generated steps
+  provider: z.string().optional(),
+  model: z.string().optional(),
+  templates: z.record(z.string()).optional(), // Role -> Agent mapping, e.g., { "planner": "@org-planner" }
+  maxSteps: z.number().int().positive().default(20), // Max steps the supervisor can generate
+  maxIterations: z.number().int().positive().default(5), // Max LLM iterations for planning
+  allowStepFailure: z.boolean().optional().default(false), // Continue on individual step failure
+  stateFile: z.string().optional(), // Path to persist workflow state (for external tools)
+  concurrency: z.union([z.number().int().positive(), z.string()]).optional().default(1), // Max parallel steps
+  library: z
+    .array(
+      z.object({
+        name: z.string(),
+        description: z.string(),
+        steps: z.array(z.any()), // Pre-defined steps in this pattern
+      })
+    )
+    .optional(), // Library of pre-defined step patterns
+  confirmPlan: z.boolean().optional().default(false), // Review and approve plan before execution
+  maxReplans: z.number().int().nonnegative().default(3), // Max automatic recovery attempts
+  allowInsecure: z.boolean().optional(), // Allow generated steps to use insecure commands (e.g. shell redirects)
+});
+
 // ===== Discriminated Union for Steps =====
 
 export const StepSchema: z.ZodType<any> = z.lazy(() =>
@@ -435,6 +464,7 @@ export const StepSchema: z.ZodType<any> = z.lazy(() =>
     ArtifactStepSchema as any,
     WaitStepSchema as any,
     GitStepSchema as any,
+    DynamicStepSchema as any,
   ])
 );
 
@@ -518,7 +548,8 @@ export type Step =
   | z.infer<typeof BlueprintStepSchema>
   | z.infer<typeof ArtifactStepSchema>
   | z.infer<typeof WaitStepSchema>
-  | z.infer<typeof GitStepSchema>;
+  | z.infer<typeof GitStepSchema>
+  | z.infer<typeof DynamicStepSchema>;
 
 export type ShellStep = z.infer<typeof ShellStepSchema>;
 export type LlmStep = z.infer<typeof LlmStepSchema>;
@@ -539,6 +570,7 @@ export type Blueprint = z.infer<typeof BlueprintSchema>;
 export type Workflow = z.infer<typeof WorkflowSchema>;
 export type AgentTool = z.infer<typeof AgentToolSchema>;
 export type WaitStep = z.infer<typeof WaitStepSchema>;
+export type DynamicStep = z.infer<typeof DynamicStepSchema>;
 
 // ===== Helper Schemas =====
 export {
@@ -565,5 +597,6 @@ export {
   MemoryStepSchema,
   ArtifactStepSchema,
   GitStepSchema,
+  DynamicStepSchema,
 };
 export type Agent = z.infer<typeof AgentSchema>;

package/src/parser/workflow-parser.test.ts

@@ -56,13 +56,12 @@ describe('WorkflowParser', () => {
       expect(() => WorkflowParser.topologicalSort(workflow)).toThrow(/circular dependency/i);
     });
 
-    test('should throw on missing dependencies', () => {
+    test('should NOT throw on missing dependencies (leniency for partial execution)', () => {
       const workflow = {
         steps: [{ id: 'step1', type: 'shell', run: 'echo 1', needs: ['non-existent'] }],
       } as unknown as Workflow;
-      expect(() => WorkflowParser.topologicalSort(workflow)).toThrow(
-        /depends on non-existent step/
-      );
+      expect(() => WorkflowParser.topologicalSort(workflow)).not.toThrow();
+      expect(WorkflowParser.topologicalSort(workflow)).toEqual(['step1']);
     });
   });
 

package/src/parser/workflow-parser.ts

@@ -4,6 +4,7 @@ import { z } from 'zod';
 import { ExpressionEvaluator } from '../expression/evaluator.ts';
 import { ResourceLoader } from '../utils/resource-loader.ts';
 import { validateJsonSchemaDefinition } from '../utils/schema-validator.ts';
+import { topologicalSort } from '../utils/topo-sort.ts';
 import { resolveAgentPath } from './agent-parser.ts';
 import { type Workflow, WorkflowSchema } from './schema.ts';
 
@@ -298,68 +299,8 @@ export class WorkflowParser {
    * Returns steps in execution order
    */
   static topologicalSort(workflow: Workflow): string[] {
-    const stepMap = new Map(workflow.steps.map((step) => [step.id, step.needs]));
-    const inDegree = new Map<string, number>();
-
-    // Validate all dependencies exist before sorting
-    for (const step of workflow.steps) {
-      const needs = step.needs || [];
-      for (const dep of needs) {
-        if (!stepMap.has(dep)) {
-          throw new Error(`Step "${step.id}" depends on non-existent step "${dep}"`);
-        }
-      }
-    }
-
-    // Calculate in-degree
-    // In-degree = number of dependencies a step has
-    for (const step of workflow.steps) {
-      const needs = step.needs || [];
-      inDegree.set(step.id, needs.length);
-    }
-
-    // Build reverse dependency map for O(1) lookups instead of O(n)
-    const dependents = new Map<string, string[]>();
-    for (const step of workflow.steps) {
-      const needs = step.needs || [];
-      for (const dep of needs) {
-        if (!dependents.has(dep)) dependents.set(dep, []);
-        dependents.get(dep)?.push(step.id);
-      }
-    }
-
-    // Kahn's algorithm
-    const queue: string[] = [];
-    const result: string[] = [];
-
-    // Add all nodes with in-degree 0
-    for (const [stepId, degree] of inDegree.entries()) {
-      if (degree === 0) {
-        queue.push(stepId);
-      }
-    }
-
-    let queueIndex = 0;
-    while (queueIndex < queue.length) {
-      const stepId = queue[queueIndex];
-      queueIndex += 1;
-      result.push(stepId);
-
-      // Find all steps that depend on this step (O(1) lookup)
-      for (const dependentId of dependents.get(stepId) || []) {
-        const newDegree = (inDegree.get(dependentId) || 0) - 1;
-        inDegree.set(dependentId, newDegree);
-        if (newDegree === 0) {
-          queue.push(dependentId);
-        }
-      }
-    }
-
-    if (result.length !== workflow.steps.length) {
-      throw new Error('Topological sort failed - circular dependency detected');
-    }
-
-    return result;
+    const sorted = topologicalSort(workflow.steps);
+    return sorted.map((s) => s.id);
   }
 
   /**