keystone-cli 0.1.0

package/src/parser/config-schema.ts

@@ -0,0 +1,54 @@
+import { z } from 'zod';
+
+export const ConfigSchema = z.object({
+  default_provider: z.string().default('openai'),
+  default_model: z.string().optional(),
+  providers: z
+    .record(
+      z.object({
+        type: z.enum(['openai', 'anthropic', 'copilot']).default('openai'),
+        base_url: z.string().optional(),
+        api_key_env: z.string().optional(),
+        default_model: z.string().optional(),
+      })
+    )
+    .default({
+      openai: {
+        type: 'openai',
+        base_url: 'https://api.openai.com/v1',
+        api_key_env: 'OPENAI_API_KEY',
+        default_model: 'gpt-4o',
+      },
+      anthropic: {
+        type: 'anthropic',
+        base_url: 'https://api.anthropic.com/v1',
+        api_key_env: 'ANTHROPIC_API_KEY',
+        default_model: 'claude-3-5-sonnet-20240620',
+      },
+      copilot: {
+        type: 'copilot',
+        base_url: 'https://api.githubcopilot.com',
+        default_model: 'gpt-4o',
+      },
+    }),
+  model_mappings: z.record(z.string()).default({
+    'claude-*': 'anthropic',
+  }),
+  storage: z
+    .object({
+      retention_days: z.number().default(30),
+    })
+    .default({}),
+  workflows_directory: z.string().default('workflows'),
+  mcp_servers: z
+    .record(
+      z.object({
+        command: z.string(),
+        args: z.array(z.string()).optional(),
+        env: z.record(z.string()).optional(),
+      })
+    )
+    .default({}),
+});
+
+export type Config = z.infer<typeof ConfigSchema>;

package/src/parser/schema.ts

@@ -0,0 +1,157 @@
+import { z } from 'zod';
+
+// ===== Input/Output Schema =====
+
+const InputSchema = z.object({
+  type: z.string(),
+  default: z.any().optional(),
+  description: z.string().optional(),
+});
+
+// ===== Retry Schema =====
+
+const RetrySchema = z.object({
+  count: z.number().int().min(0).default(0),
+  backoff: z.enum(['linear', 'exponential']).default('linear'),
+  baseDelay: z.number().int().min(0).default(1000),
+});
+
+// ===== Base Step Schema =====
+
+const BaseStepSchema = z.object({
+  id: z.string(),
+  type: z.string(),
+  needs: z.array(z.string()).default([]),
+  if: z.string().optional(),
+  timeout: z.number().int().positive().optional(),
+  retry: RetrySchema.optional(),
+  foreach: z.string().optional(),
+  // Accept both number and string (for expressions or YAML number-as-string)
+  concurrency: z.union([z.number().int().positive(), z.string()]).optional(),
+  transform: z.string().optional(),
+});
+
+// ===== Step Type Schemas =====
+
+const ShellStepSchema = BaseStepSchema.extend({
+  type: z.literal('shell'),
+  run: z.string(),
+  dir: z.string().optional(),
+  env: z.record(z.string()).optional(),
+});
+
+// Forward declaration for AgentToolSchema which depends on StepSchema
+const AgentToolSchema = z.object({
+  name: z.string(),
+  description: z.string().optional(),
+  parameters: z.any().optional(), // JSON Schema for tool arguments
+  execution: z.lazy(() => StepSchema), // Tools are essentially steps
+});
+
+const LlmStepSchema = BaseStepSchema.extend({
+  type: z.literal('llm'),
+  agent: z.string(),
+  provider: z.string().optional(),
+  model: z.string().optional(),
+  prompt: z.string(),
+  schema: z.any().optional(),
+  tools: z.array(AgentToolSchema).optional(),
+  maxIterations: z.number().int().positive().default(10),
+  useGlobalMcp: z.boolean().optional(),
+  mcpServers: z
+    .array(
+      z.union([
+        z.string(),
+        z.object({
+          name: z.string(),
+          command: z.string(),
+          args: z.array(z.string()).optional(),
+          env: z.record(z.string()).optional(),
+        }),
+      ])
+    )
+    .optional(),
+});
+
+const WorkflowStepSchema = BaseStepSchema.extend({
+  type: z.literal('workflow'),
+  path: z.string(),
+  inputs: z.record(z.string()).optional(),
+});
+
+const FileStepSchema = BaseStepSchema.extend({
+  type: z.literal('file'),
+  path: z.string(),
+  content: z.string().optional(),
+  op: z.enum(['read', 'write', 'append']),
+});
+
+const RequestStepSchema = BaseStepSchema.extend({
+  type: z.literal('request'),
+  url: z.string(),
+  method: z.enum(['GET', 'POST', 'PUT', 'PATCH', 'DELETE']).default('GET'),
+  body: z.any().optional(),
+  headers: z.record(z.string()).optional(),
+});
+
+const HumanStepSchema = BaseStepSchema.extend({
+  type: z.literal('human'),
+  message: z.string(),
+  inputType: z.enum(['confirm', 'text']).default('confirm'),
+});
+
+const SleepStepSchema = BaseStepSchema.extend({
+  type: z.literal('sleep'),
+  duration: z.union([z.number().int().positive(), z.string()]),
+});
+
+// ===== Discriminated Union for Steps =====
+
+export const StepSchema = z.discriminatedUnion('type', [
+  ShellStepSchema,
+  LlmStepSchema,
+  WorkflowStepSchema,
+  FileStepSchema,
+  RequestStepSchema,
+  HumanStepSchema,
+  SleepStepSchema,
+]);
+
+// ===== Workflow Schema =====
+
+export const WorkflowSchema = z.object({
+  name: z.string(),
+  description: z.string().optional(),
+  inputs: z.record(InputSchema).optional(),
+  outputs: z.record(z.string()).optional(),
+  env: z.record(z.string()).optional(),
+  steps: z.array(StepSchema),
+  finally: z.array(StepSchema).optional(),
+});
+
+// ===== Agent Schema =====
+
+export const AgentSchema = z.object({
+  name: z.string(),
+  description: z.string().optional(),
+  provider: z.string().optional(),
+  model: z.string().optional(),
+  tools: z.array(AgentToolSchema).default([]),
+  systemPrompt: z.string(),
+});
+
+// ===== Types =====
+
+export type WorkflowInput = z.infer<typeof InputSchema>;
+export type RetryConfig = z.infer<typeof RetrySchema>;
+export type Step = z.infer<typeof StepSchema>;
+export type ShellStep = z.infer<typeof ShellStepSchema>;
+export type LlmStep = z.infer<typeof LlmStepSchema>;
+export type WorkflowStep = z.infer<typeof WorkflowStepSchema>;
+export type FileStep = z.infer<typeof FileStepSchema>;
+export type RequestStep = z.infer<typeof RequestStepSchema>;
+export type HumanStep = z.infer<typeof HumanStepSchema>;
+export type SleepStep = z.infer<typeof SleepStepSchema>;
+export type Workflow = z.infer<typeof WorkflowSchema>;
+export type AgentTool = z.infer<typeof AgentToolSchema>;
+export type Agent = z.infer<typeof AgentSchema>;

package/src/parser/workflow-parser.test.ts

@@ -0,0 +1,212 @@
+import { afterAll, describe, expect, test } from 'bun:test';
+import { mkdirSync, rmSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import type { Workflow } from './schema';
+import { WorkflowParser } from './workflow-parser';
+
+describe('WorkflowParser', () => {
+  const tempDir = join(process.cwd(), 'temp-test-workflows');
+  try {
+    mkdirSync(tempDir, { recursive: true });
+  } catch (e) {}
+
+  afterAll(() => {
+    try {
+      rmSync(tempDir, { recursive: true, force: true });
+    } catch (e) {}
+  });
+  describe('topologicalSort', () => {
+    test('should sort simple dependencies', () => {
+      const workflow = {
+        steps: [
+          { id: 'step1', type: 'shell', run: 'echo 1', needs: [] },
+          { id: 'step2', type: 'shell', run: 'echo 2', needs: ['step1'] },
+          { id: 'step3', type: 'shell', run: 'echo 3', needs: ['step2'] },
+        ],
+      } as unknown as Workflow;
+      expect(WorkflowParser.topologicalSort(workflow)).toEqual(['step1', 'step2', 'step3']);
+    });
+
+    test('should handle parallel branches', () => {
+      const workflow = {
+        steps: [
+          { id: 'step1', type: 'shell', run: 'echo 1', needs: [] },
+          { id: 'step2a', type: 'shell', run: 'echo 2a', needs: ['step1'] },
+          { id: 'step2b', type: 'shell', run: 'echo 2b', needs: ['step1'] },
+          { id: 'step3', type: 'shell', run: 'echo 3', needs: ['step2a', 'step2b'] },
+        ],
+      } as unknown as Workflow;
+      const result = WorkflowParser.topologicalSort(workflow);
+      expect(result[0]).toBe('step1');
+      expect(new Set([result[1], result[2]])).toEqual(new Set(['step2a', 'step2b']));
+      expect(result[3]).toBe('step3');
+    });
+
+    test('should throw on circular dependencies', () => {
+      const workflow = {
+        steps: [
+          { id: 'step1', type: 'shell', run: 'echo 1', needs: ['step2'] },
+          { id: 'step2', type: 'shell', run: 'echo 2', needs: ['step1'] },
+        ],
+      } as unknown as Workflow;
+      expect(() => WorkflowParser.topologicalSort(workflow)).toThrow(/circular dependency/i);
+    });
+
+    test('should throw on missing dependencies', () => {
+      const workflow = {
+        steps: [{ id: 'step1', type: 'shell', run: 'echo 1', needs: ['non-existent'] }],
+      } as unknown as Workflow;
+      expect(() => WorkflowParser.topologicalSort(workflow)).toThrow(
+        /depends on non-existent step/
+      );
+    });
+  });
+
+  describe('loadWorkflow', () => {
+    test('should load valid workflow', () => {
+      const content = `
+name: example-workflow
+steps:
+  - id: step1
+    type: shell
+    run: echo hello
+`;
+      const filePath = join(tempDir, 'valid.yaml');
+      writeFileSync(filePath, content);
+      const workflow = WorkflowParser.loadWorkflow(filePath);
+      expect(workflow.name).toBe('example-workflow');
+      expect(workflow.steps.length).toBeGreaterThan(0);
+    });
+
+    test('should throw on invalid schema', () => {
+      const content = `
+name: invalid
+steps:
+  - id: step1
+    type: invalid-type
+`;
+      const filePath = join(tempDir, 'invalid.yaml');
+      writeFileSync(filePath, content);
+      expect(() => WorkflowParser.loadWorkflow(filePath)).toThrow(/Invalid workflow schema/);
+    });
+
+    test('should throw on non-existent file', () => {
+      expect(() => WorkflowParser.loadWorkflow('non-existent.yaml')).toThrow(
+        /Failed to parse workflow/
+      );
+    });
+
+    test('should validate DAG during load', () => {
+      const content = `
+name: circular
+steps:
+  - id: step1
+    type: shell
+    run: echo 1
+    needs: [step2]
+  - id: step2
+    type: shell
+    run: echo 2
+    needs: [step1]
+`;
+      const filePath = join(tempDir, 'circular.yaml');
+      writeFileSync(filePath, content);
+      expect(() => WorkflowParser.loadWorkflow(filePath)).toThrow(/Circular dependency detected/);
+    });
+
+    test('should validate finally block Step ID conflicts', () => {
+      const content = `
+name: conflict
+steps:
+  - id: step1
+    type: shell
+    run: echo 1
+finally:
+  - id: step1
+    type: shell
+    run: echo finally
+`;
+      const filePath = join(tempDir, 'conflict.yaml');
+      writeFileSync(filePath, content);
+      expect(() => WorkflowParser.loadWorkflow(filePath)).toThrow(/conflicts with main steps/);
+    });
+
+    test('should validate duplicate finally block Step IDs', () => {
+      const content = `
+name: finally-dup
+steps:
+  - id: step1
+    type: shell
+    run: echo 1
+finally:
+  - id: fin1
+    type: shell
+    run: echo f1
+  - id: fin1
+    type: shell
+    run: echo f2
+`;
+      const filePath = join(tempDir, 'finally-dup.yaml');
+      writeFileSync(filePath, content);
+      expect(() => WorkflowParser.loadWorkflow(filePath)).toThrow(
+        /Duplicate Step ID "fin1" in finally block/
+      );
+    });
+
+    test('should throw on non-existent agents in LLM steps', () => {
+      const content = `
+name: llm-agent
+steps:
+  - id: step1
+    type: llm
+    agent: non-existent-agent
+    prompt: hello
+`;
+      const filePath = join(tempDir, 'llm-agent.yaml');
+      writeFileSync(filePath, content);
+
+      expect(() => WorkflowParser.loadWorkflow(filePath)).toThrow(
+        /Agent "non-existent-agent" referenced in step "step1" not found/
+      );
+    });
+
+    test('should resolve implicit dependencies from expressions', () => {
+      const content = `
+name: implicit-deps
+steps:
+  - id: ask_name
+    type: human
+    message: "What is your name?"
+  - id: greet
+    type: shell
+    run: echo "Hello, \${{ steps.ask_name.output }}!"
+`;
+      const filePath = join(tempDir, 'implicit-deps.yaml');
+      writeFileSync(filePath, content);
+
+      const workflow = WorkflowParser.loadWorkflow(filePath);
+      const greetStep = workflow.steps.find((s) => s.id === 'greet');
+      expect(greetStep?.needs).toContain('ask_name');
+    });
+
+    test('should resolve implicit dependencies in finally block', () => {
+      const content = `
+name: finally-implicit
+steps:
+  - id: step1
+    type: shell
+    run: echo 1
+finally:
+  - id: cleanup
+    type: shell
+    run: echo "Cleaning up after \${{ steps.step1.output }}"
+`;
+      const filePath = join(tempDir, 'finally-implicit.yaml');
+      writeFileSync(filePath, content);
+
+      const workflow = WorkflowParser.loadWorkflow(filePath);
+      const cleanupStep = workflow.finally?.find((s) => s.id === 'cleanup');
+      expect(cleanupStep?.needs).toContain('step1');
+    });
+  });
+});

package/src/parser/workflow-parser.ts

@@ -0,0 +1,228 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import * as yaml from 'js-yaml';
+import { z } from 'zod';
+import { ExpressionEvaluator } from '../expression/evaluator.ts';
+import { resolveAgentPath } from './agent-parser.ts';
+import { type Workflow, WorkflowSchema } from './schema.ts';
+
+export class WorkflowParser {
+  /**
+   * Load and validate a workflow from a YAML file
+   */
+  static loadWorkflow(path: string): Workflow {
+    try {
+      const content = readFileSync(path, 'utf-8');
+      const raw = yaml.load(content);
+      const workflow = WorkflowSchema.parse(raw);
+
+      // Resolve implicit dependencies from expressions
+      WorkflowParser.resolveImplicitDependencies(workflow);
+
+      // Validate DAG (no circular dependencies)
+      WorkflowParser.validateDAG(workflow);
+
+      // Validate agents exist
+      WorkflowParser.validateAgents(workflow);
+
+      // Validate finally block
+      WorkflowParser.validateFinally(workflow);
+
+      return workflow;
+    } catch (error) {
+      if (error instanceof z.ZodError) {
+        const issues = error.issues
+          .map((issue) => `  - ${issue.path.join('.')}: ${issue.message}`)
+          .join('\n');
+        throw new Error(`Invalid workflow schema at ${path}:\n${issues}`);
+      }
+      if (error instanceof Error) {
+        throw new Error(`Failed to parse workflow at ${path}: ${error.message}`);
+      }
+      throw error;
+    }
+  }
+
+  /**
+   * Automatically detect step dependencies from expressions
+   */
+  private static resolveImplicitDependencies(workflow: Workflow): void {
+    const allSteps = [...workflow.steps, ...(workflow.finally || [])];
+    for (const step of allSteps) {
+      const detected = new Set<string>();
+
+      // Helper to scan any value for dependencies
+      const scan = (value: unknown) => {
+        if (typeof value === 'string') {
+          for (const dep of ExpressionEvaluator.findStepDependencies(value)) {
+            detected.add(dep);
+          }
+        } else if (Array.isArray(value)) {
+          for (const item of value) {
+            scan(item);
+          }
+        } else if (value && typeof value === 'object') {
+          for (const val of Object.values(value)) {
+            scan(val);
+          }
+        }
+      };
+
+      // Scan all step properties
+      scan(step);
+
+      // Add detected dependencies to step.needs
+      for (const depId of detected) {
+        // Step cannot depend on itself
+        if (depId !== step.id && !step.needs.includes(depId)) {
+          step.needs.push(depId);
+        }
+      }
+    }
+  }
+
+  /**
+   * Validate that the workflow forms a valid DAG (no cycles)
+   */
+  private static validateDAG(workflow: Workflow): void {
+    const stepMap = new Map(workflow.steps.map((step) => [step.id, step.needs]));
+    const visited = new Set<string>();
+    const recursionStack = new Set<string>();
+
+    const hasCycle = (stepId: string): boolean => {
+      if (!visited.has(stepId)) {
+        visited.add(stepId);
+        recursionStack.add(stepId);
+
+        const dependencies = stepMap.get(stepId) || [];
+        for (const dep of dependencies) {
+          if (!stepMap.has(dep)) {
+            throw new Error(`Step "${stepId}" depends on non-existent step "${dep}"`);
+          }
+          if (!visited.has(dep) && hasCycle(dep)) {
+            return true;
+          }
+          if (recursionStack.has(dep)) {
+            return true;
+          }
+        }
+      }
+      recursionStack.delete(stepId);
+      return false;
+    };
+
+    for (const step of workflow.steps) {
+      if (hasCycle(step.id)) {
+        throw new Error(`Circular dependency detected involving step "${step.id}"`);
+      }
+    }
+  }
+
+  /**
+   * Validate that all agents referenced in LLM steps exist
+   */
+  private static validateAgents(workflow: Workflow): void {
+    const allSteps = [...workflow.steps, ...(workflow.finally || [])];
+    for (const step of allSteps) {
+      if (step.type === 'llm') {
+        try {
+          resolveAgentPath(step.agent);
+        } catch (error) {
+          throw new Error(`Agent "${step.agent}" referenced in step "${step.id}" not found.`);
+        }
+      }
+    }
+  }
+
+  /**
+   * Validate finally block
+   */
+  private static validateFinally(workflow: Workflow): void {
+    if (!workflow.finally) return;
+
+    const mainStepIds = new Set(workflow.steps.map((s) => s.id));
+    const finallyStepIds = new Set<string>();
+
+    for (const step of workflow.finally) {
+      if (mainStepIds.has(step.id)) {
+        throw new Error(`Step ID "${step.id}" in finally block conflicts with main steps`);
+      }
+      if (finallyStepIds.has(step.id)) {
+        throw new Error(`Duplicate Step ID "${step.id}" in finally block`);
+      }
+      finallyStepIds.add(step.id);
+
+      // Finally steps can only depend on main steps or previous finally steps
+      for (const dep of step.needs) {
+        if (!mainStepIds.has(dep) && !finallyStepIds.has(dep)) {
+          throw new Error(
+            `Finally step "${step.id}" depends on non-existent step "${dep}". Finally steps can only depend on main steps or previous finally steps.`
+          );
+        }
+      }
+    }
+  }
+
+  /**
+   * Perform topological sort on steps
+   * Returns steps in execution order
+   */
+  static topologicalSort(workflow: Workflow): string[] {
+    const stepMap = new Map(workflow.steps.map((step) => [step.id, step.needs]));
+    const inDegree = new Map<string, number>();
+
+    // Validate all dependencies exist before sorting
+    for (const step of workflow.steps) {
+      for (const dep of step.needs) {
+        if (!stepMap.has(dep)) {
+          throw new Error(`Step "${step.id}" depends on non-existent step "${dep}"`);
+        }
+      }
+    }
+
+    // Initialize in-degree
+    for (const step of workflow.steps) {
+      inDegree.set(step.id, 0);
+    }
+
+    // Calculate in-degree
+    // In-degree = number of dependencies a step has
+    for (const step of workflow.steps) {
+      inDegree.set(step.id, step.needs.length);
+    }
+
+    // Kahn's algorithm
+    const queue: string[] = [];
+    const result: string[] = [];
+
+    // Add all nodes with in-degree 0
+    for (const [stepId, degree] of inDegree.entries()) {
+      if (degree === 0) {
+        queue.push(stepId);
+      }
+    }
+
+    while (queue.length > 0) {
+      const stepId = queue.shift();
+      if (!stepId) continue;
+      result.push(stepId);
+
+      // Find all steps that depend on this step
+      for (const step of workflow.steps) {
+        if (step.needs.includes(stepId)) {
+          const newDegree = (inDegree.get(step.id) || 0) - 1;
+          inDegree.set(step.id, newDegree);
+          if (newDegree === 0) {
+            queue.push(step.id);
+          }
+        }
+      }
+    }
+
+    if (result.length !== workflow.steps.length) {
+      throw new Error('Topological sort failed - circular dependency detected');
+    }
+
+    return result;
+  }
+}