keystone-cli 0.8.0 → 1.0.1

package/src/expression/evaluator.ts

@@ -27,7 +27,11 @@ jsep.plugins.register(jsepObject);
 export interface ExpressionContext {
   inputs?: Record<string, unknown>;
   secrets?: Record<string, string>;
-  steps?: Record<string, { output?: unknown; outputs?: Record<string, unknown>; status?: string }>;
+  secretValues?: string[];
+  steps?: Record<
+    string,
+    { output?: unknown; outputs?: Record<string, unknown>; status?: string; error?: string }
+  >;
   item?: unknown;
   args?: unknown;
   index?: number;
@@ -35,6 +39,8 @@ export interface ExpressionContext {
   output?: unknown;
   autoHealAttempts?: number;
   reflexionAttempts?: number;
+  outputRepairAttempts?: number;
+  last_failed_step?: { id: string; error: string };
 }
 
 type ASTNode = jsep.Expression;
@@ -76,6 +82,14 @@ export class ExpressionEvaluator {
   private static readonly MAX_TEMPLATE_LENGTH = 10_000;
   // Maximum length for plain strings without expressions (1MB)
   private static readonly MAX_PLAIN_STRING_LENGTH = 1_000_000;
+  // Maximum nesting depth for expressions (prevents stack overflow)
+  private static readonly MAX_NESTING_DEPTH = 50;
+  // Maximum array literal size (prevents memory exhaustion)
+  private static readonly MAX_ARRAY_SIZE = 1000;
+  // Maximum total nodes to evaluate (prevents DoS via complex expressions)
+  private static readonly MAX_TOTAL_NODES = 10000;
+  // Maximum arrow function nesting depth
+  private static readonly MAX_ARROW_DEPTH = 3;
 
   /**
    * Helper to scan string for matches of ${{ ... }} handling nested braces manually
@@ -237,7 +251,9 @@ export class ExpressionEvaluator {
   static evaluateExpression(expr: string, context: ExpressionContext): unknown {
     try {
       const ast = jsep(expr);
-      return ExpressionEvaluator.evaluateNode(ast, context);
+      // Track total nodes evaluated to prevent DoS
+      const nodeCounter = { count: 0 };
+      return ExpressionEvaluator.evaluateNode(ast, context, 0, nodeCounter);
     } catch (error) {
       throw new Error(
         `Failed to evaluate expression "${expr}": ${error instanceof Error ? error.message : String(error)}`
@@ -248,7 +264,25 @@ export class ExpressionEvaluator {
   /**
    * Evaluate an AST node recursively
    */
-  private static evaluateNode(node: ASTNode, context: ExpressionContext): unknown {
+  private static evaluateNode(
+    node: ASTNode,
+    context: ExpressionContext,
+    depth = 0,
+    nodeCounter: { count: number } = { count: 0 },
+    arrowDepth = 0
+  ): unknown {
+    // Increment node counter for DoS protection
+    nodeCounter.count++;
+    if (nodeCounter.count > ExpressionEvaluator.MAX_TOTAL_NODES) {
+      throw new Error(
+        `Expression exceeds maximum complexity of ${ExpressionEvaluator.MAX_TOTAL_NODES} nodes`
+      );
+    }
+    if (depth > ExpressionEvaluator.MAX_NESTING_DEPTH) {
+      throw new Error(
+        `Expression nesting exceeds maximum depth of ${ExpressionEvaluator.MAX_NESTING_DEPTH}`
+      );
+    }
     switch (node.type) {
       case 'Literal':
         return (node as jsep.Literal).value;
@@ -256,6 +290,30 @@ export class ExpressionEvaluator {
       case 'Identifier': {
         const name = (node as jsep.Identifier).name;
 
+        // Security: Block dangerous global identifiers that could enable code execution
+        const FORBIDDEN_IDENTIFIERS = new Set([
+          'eval',
+          'Function',
+          'AsyncFunction',
+          'GeneratorFunction',
+          'globalThis',
+          'global',
+          'self',
+          'window',
+          'top',
+          'parent',
+          'frames',
+          'Reflect',
+          'Proxy',
+          'require',
+          'import',
+          'module',
+          'exports',
+        ]);
+        if (FORBIDDEN_IDENTIFIERS.has(name)) {
+          throw new Error(`Access to "${name}" is forbidden for security reasons`);
+        }
+
         // Safe global functions and values
         const safeGlobals: Record<string, unknown> = {
           Boolean: Boolean,
@@ -300,6 +358,7 @@ export class ExpressionEvaluator {
           index: context.index,
           env: context.env || {},
           stdout: contextAsRecord.stdout, // For transform expressions
+          last_failed_step: context.last_failed_step,
         };
 
         if (name in rootContext && rootContext[name] !== undefined) {
@@ -441,8 +500,13 @@ export class ExpressionEvaluator {
 
       case 'ArrayExpression': {
         const arrayNode = node as jsep.ArrayExpression;
+        if (arrayNode.elements.length > ExpressionEvaluator.MAX_ARRAY_SIZE) {
+          throw new Error(
+            `Array literal exceeds maximum size of ${ExpressionEvaluator.MAX_ARRAY_SIZE} elements`
+          );
+        }
         return arrayNode.elements.map((elem) =>
-          elem ? ExpressionEvaluator.evaluateNode(elem, context) : null
+          elem ? ExpressionEvaluator.evaluateNode(elem, context, depth + 1) : null
         );
       }
 

package/src/parser/agent-parser.ts

@@ -1,11 +1,14 @@
-import { existsSync, readFileSync } from 'node:fs';
 import { homedir } from 'node:os';
 import { join } from 'node:path';
 import yaml from 'js-yaml';
+import { ResourceLoader } from '../utils/resource-loader';
 import { type Agent, AgentSchema } from './schema';
 
 export function parseAgent(filePath: string): Agent {
-  const content = readFileSync(filePath, 'utf8');
+  const content = ResourceLoader.readFile(filePath);
+  if (content === null) {
+    throw new Error(`Agent file not found at ${filePath}`);
+  }
   // Flexible regex to handle both standard and single-line frontmatter
   const match = content.match(/^---[\r\n]*([\s\S]*?)[\r\n]*---(?:\r?\n?([\s\S]*))?$/);
 
@@ -58,7 +61,7 @@ export function resolveAgentPath(agentName: string, baseDir?: string): string {
   );
 
   for (const path of possiblePaths) {
-    if (existsSync(path)) {
+    if (ResourceLoader.exists(path)) {
       return path;
     }
   }

package/src/parser/config-schema.ts

@@ -6,10 +6,20 @@ export const ConfigSchema = z.object({
   providers: z
     .record(
       z.object({
-        type: z.enum(['openai', 'anthropic', 'copilot']).default('openai'),
+        type: z
+          .enum([
+            'openai',
+            'anthropic',
+            'anthropic-claude',
+            'copilot',
+            'openai-chatgpt',
+            'google-gemini',
+          ])
+          .default('openai'),
         base_url: z.string().optional(),
         api_key_env: z.string().optional(),
         default_model: z.string().optional(),
+        project_id: z.string().optional(),
       })
     )
     .default({
@@ -37,9 +47,9 @@ export const ConfigSchema = z.object({
   storage: z
     .object({
       retention_days: z.number().default(30),
+      redact_secrets_at_rest: z.boolean().default(true),
     })
     .default({}),
-  workflows_directory: z.string().default('workflows'),
   mcp_servers: z
     .record(
       z.union([
@@ -64,6 +74,32 @@ export const ConfigSchema = z.object({
       ])
     )
     .default({}),
+  engines: z
+    .object({
+      allowlist: z
+        .record(
+          z.object({
+            command: z.string(),
+            args: z.array(z.string()).optional(),
+            version: z.string(),
+            versionArgs: z.array(z.string()).optional().default(['--version']),
+          })
+        )
+        .default({}),
+      denylist: z.array(z.string()).default([]),
+    })
+    .default({}),
+  concurrency: z
+    .object({
+      default: z.number().int().positive().default(10),
+      pools: z.record(z.number().int().positive()).default({
+        llm: 2,
+        shell: 5,
+        http: 10,
+        engine: 2,
+      }),
+    })
+    .default({}),
 });
 
 export type Config = z.infer<typeof ConfigSchema>;

package/src/parser/schema.ts

@@ -2,11 +2,84 @@ import { z } from 'zod';
 
 // ===== Input/Output Schema =====
 
-const InputSchema = z.object({
-  type: z.enum(['string', 'number', 'boolean', 'array', 'object']),
-  default: z.any().optional(),
-  description: z.string().optional(),
-});
+const InputSchema = z
+  .object({
+    type: z.enum(['string', 'number', 'boolean', 'array', 'object']),
+    default: z.any().optional(),
+    values: z.array(z.union([z.string(), z.number(), z.boolean()])).optional(),
+    secret: z.boolean().optional(),
+    description: z.string().optional(),
+  })
+  .superRefine((value, ctx) => {
+    const type = value.type;
+    const defaultValue = value.default;
+
+    if (defaultValue !== undefined) {
+      if (type === 'string' && typeof defaultValue !== 'string') {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `default must be a string for type "${type}"`,
+        });
+      }
+      if (type === 'number' && typeof defaultValue !== 'number') {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `default must be a number for type "${type}"`,
+        });
+      }
+      if (type === 'boolean' && typeof defaultValue !== 'boolean') {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `default must be a boolean for type "${type}"`,
+        });
+      }
+      if (type === 'array' && !Array.isArray(defaultValue)) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `default must be an array for type "${type}"`,
+        });
+      }
+      if (
+        type === 'object' &&
+        (typeof defaultValue !== 'object' || defaultValue === null || Array.isArray(defaultValue))
+      ) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `default must be an object for type "${type}"`,
+        });
+      }
+    }
+
+    if (value.values) {
+      if (type !== 'string' && type !== 'number' && type !== 'boolean') {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `values cannot be used with type "${type}"`,
+        });
+        return;
+      }
+
+      for (const allowed of value.values) {
+        const matchesType =
+          (type === 'string' && typeof allowed === 'string') ||
+          (type === 'number' && typeof allowed === 'number') ||
+          (type === 'boolean' && typeof allowed === 'boolean');
+        if (!matchesType) {
+          ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: `enum value ${JSON.stringify(allowed)} must be a ${type}`,
+          });
+        }
+      }
+
+      if (defaultValue !== undefined && !value.values.includes(defaultValue as never)) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `default must be one of: ${value.values.map((v) => JSON.stringify(v)).join(', ')}`,
+        });
+      }
+    }
+  });
 
 // ===== Retry Schema =====
 
@@ -42,11 +115,21 @@ const BaseStepSchema = z.object({
   retry: RetrySchema.optional(),
   auto_heal: AutoHealSchema.optional(),
   reflexion: ReflexionSchema.optional(),
+  allowFailure: z.boolean().optional(),
+  idempotencyKey: z.string().optional(), // Expression for dedup key (evaluated at runtime)
+  idempotencyScope: z.enum(['run', 'global']).optional(), // Default: run
+  idempotencyTtlSeconds: z.number().int().positive().optional(),
   foreach: z.string().optional(),
   // Accept both number and string (for expressions or YAML number-as-string)
   concurrency: z.union([z.number().int().positive(), z.string()]).optional(),
+  pool: z.string().optional(), // Resource pool to use for this step
   transform: z.string().optional(),
   learn: z.boolean().optional(),
+  inputSchema: z.any().optional(),
+  outputSchema: z.any().optional(),
+  outputRetries: z.number().int().min(0).optional(), // Max retries for output validation failures
+  repairStrategy: z.enum(['reask', 'repair', 'hybrid']).optional(), // Strategy for output repair
+  compensate: z.lazy(() => StepSchema).optional(), // Compensation step to run on rollback
 });
 
 // ===== Step Type Schemas =====
@@ -67,15 +150,41 @@ const AgentToolSchema = z.object({
   execution: z.lazy(() => StepSchema), // Tools are essentially steps
 });
 
+const JoinStepSchema = BaseStepSchema.extend({
+  type: z.literal('join'),
+  target: z.enum(['steps', 'branches']).optional().default('steps'),
+  condition: z
+    .union([z.literal('all'), z.literal('any'), z.number().int().positive()])
+    .default('all'),
+});
+
+const EngineConfigSchema = z.object({
+  command: z.string(),
+  args: z.array(z.string()).optional(),
+  input: z.any().optional(),
+  env: z.record(z.string()),
+  cwd: z.string(),
+});
+
+const EngineHandoffSchema = z.object({
+  name: z.string().optional(),
+  description: z.string().optional(),
+  inputSchema: z.any().optional(),
+  engine: EngineConfigSchema.extend({
+    timeout: z.number().int().positive().optional(),
+    outputSchema: z.any().optional(),
+  }),
+});
+
 const LlmStepSchema = BaseStepSchema.extend({
   type: z.literal('llm'),
   agent: z.string(),
   provider: z.string().optional(),
   model: z.string().optional(),
   prompt: z.string(),
-  schema: z.any().optional(),
   tools: z.array(AgentToolSchema).optional(),
   maxIterations: z.number().int().positive().default(10),
+  maxMessageHistory: z.number().int().positive().optional(), // Max messages to keep in conversation history
   useGlobalMcp: z.boolean().optional(),
   allowClarification: z.boolean().optional(),
   mcpServers: z
@@ -98,12 +207,22 @@ const LlmStepSchema = BaseStepSchema.extend({
   useStandardTools: z.boolean().optional(),
   allowOutsideCwd: z.boolean().optional(),
   allowInsecure: z.boolean().optional(),
+  handoff: EngineHandoffSchema.optional(),
 });
 
+const OutputMappingItemSchema = z.union([
+  z.string(), // Rename: alias -> originalKey
+  z.object({
+    from: z.string(),
+    default: z.any().optional(),
+  }),
+]);
+
 const WorkflowStepSchema = BaseStepSchema.extend({
   type: z.literal('workflow'),
   path: z.string(),
   inputs: z.record(z.string()).optional(),
+  outputMapping: z.record(OutputMappingItemSchema).optional(),
 });
 
 const FileStepSchema = BaseStepSchema.extend({
@@ -117,9 +236,10 @@ const FileStepSchema = BaseStepSchema.extend({
 const RequestStepSchema = BaseStepSchema.extend({
   type: z.literal('request'),
   url: z.string(),
-  method: z.enum(['GET', 'POST', 'PUT', 'PATCH', 'DELETE']).default('GET'),
+  method: z.enum(['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD']).default('GET'),
   body: z.any().optional(),
   headers: z.record(z.string()).optional(),
+  allowInsecure: z.boolean().optional(),
 });
 
 const HumanStepSchema = BaseStepSchema.extend({
@@ -131,6 +251,7 @@ const HumanStepSchema = BaseStepSchema.extend({
 const SleepStepSchema = BaseStepSchema.extend({
   type: z.literal('sleep'),
   duration: z.union([z.number().int().positive(), z.string()]),
+  durable: z.boolean().optional(), // Persist across restarts for long sleeps
 });
 
 const ScriptStepSchema = BaseStepSchema.extend({
@@ -139,6 +260,57 @@ const ScriptStepSchema = BaseStepSchema.extend({
   allowInsecure: z.boolean().optional().default(false),
 });
 
+const EngineStepSchema = BaseStepSchema.extend({
+  type: z.literal('engine'),
+}).merge(EngineConfigSchema);
+
+const BlueprintSchema = z.object({
+  architecture: z.object({
+    description: z.string(),
+    patterns: z.array(z.string()).optional(),
+  }),
+  apis: z
+    .array(
+      z.object({
+        name: z.string(),
+        description: z.string(),
+        endpoints: z
+          .array(
+            z.object({
+              path: z.string(),
+              method: z.string(),
+              purpose: z.string(),
+            })
+          )
+          .optional(),
+      })
+    )
+    .optional(),
+  files: z.array(
+    z.object({
+      path: z.string(),
+      purpose: z.string(),
+      constraints: z.array(z.string()).optional(),
+    })
+  ),
+  dependencies: z
+    .array(
+      z.object({
+        name: z.string(),
+        version: z.string().optional(),
+        purpose: z.string(),
+      })
+    )
+    .optional(),
+  constraints: z.array(z.string()).optional(),
+});
+
+const BlueprintStepSchema = BaseStepSchema.extend({
+  type: z.literal('blueprint'),
+  prompt: z.string(),
+  agent: z.string().optional().default('keystone-architect'),
+});
+
 const MemoryStepSchema = BaseStepSchema.extend({
   type: z.literal('memory'),
   op: z.enum(['search', 'store']),
@@ -162,7 +334,10 @@ export const StepSchema: z.ZodType<any> = z.lazy(() =>
     HumanStepSchema,
     SleepStepSchema,
     ScriptStepSchema,
+    EngineStepSchema,
     MemoryStepSchema,
+    JoinStepSchema,
+    BlueprintStepSchema,
   ])
 );
 
@@ -173,6 +348,8 @@ const EvalSchema = z.object({
   agent: z.string().optional(),
   prompt: z.string().optional(),
   run: z.string().optional(), // for script scorer
+  allowInsecure: z.boolean().optional(),
+  allowSecrets: z.boolean().optional(),
 });
 
 // ===== Workflow Schema =====
@@ -182,10 +359,14 @@ export const WorkflowSchema = z.object({
   description: z.string().optional(),
   inputs: z.record(InputSchema).optional(),
   outputs: z.record(z.string()).optional(),
+  outputSchema: z.any().optional(), // JSON Schema for final workflow outputs
   env: z.record(z.string()).optional(),
   concurrency: z.union([z.number().int().positive(), z.string()]).optional(),
+  pools: z.record(z.union([z.number().int().positive(), z.string()])).optional(), // Resource pool overrides
   steps: z.array(StepSchema),
+  errors: z.array(StepSchema).optional(),
   finally: z.array(StepSchema).optional(),
+  compensate: z.lazy(() => StepSchema).optional(), // Top-level compensation for the entire workflow
   eval: EvalSchema.optional(),
 });
 
@@ -214,6 +395,10 @@ export type HumanStep = z.infer<typeof HumanStepSchema>;
 export type SleepStep = z.infer<typeof SleepStepSchema>;
 export type ScriptStep = z.infer<typeof ScriptStepSchema>;
 export type MemoryStep = z.infer<typeof MemoryStepSchema>;
+export type EngineStep = z.infer<typeof EngineStepSchema>;
+export type JoinStep = z.infer<typeof JoinStepSchema>;
+export type BlueprintStep = z.infer<typeof BlueprintStepSchema>;
+export type Blueprint = z.infer<typeof BlueprintSchema>;
 export type Workflow = z.infer<typeof WorkflowSchema>;
 export type AgentTool = z.infer<typeof AgentToolSchema>;
 export type Agent = z.infer<typeof AgentSchema>;

package/src/parser/test-schema.ts

@@ -0,0 +1,29 @@
+export interface TestDefinition {
+  name: string;
+  workflow: string; // Name or path
+  fixture: {
+    inputs?: Record<string, unknown>;
+    env?: Record<string, string>;
+    secrets?: Record<string, string>;
+    mocks?: Array<{
+      step?: string;
+      type?: string;
+      prompt?: string;
+      // biome-ignore lint/suspicious/noExplicitAny: Mock responses can be any type
+      response: any;
+    }>;
+  };
+  snapshot?: {
+    steps: Record<
+      string,
+      {
+        status: string;
+        // biome-ignore lint/suspicious/noExplicitAny: Step outputs can be any type
+        output: any;
+        error?: string;
+      }
+    >;
+    // biome-ignore lint/suspicious/noExplicitAny: Workflow outputs can be any type
+    outputs: Record<string, any>;
+  };
+}

package/src/parser/workflow-parser.test.ts

@@ -94,6 +94,24 @@ steps:
       expect(() => WorkflowParser.loadWorkflow(filePath)).toThrow(/Invalid workflow schema/);
     });
 
+    test('should throw on invalid input enum defaults', () => {
+      const content = `
+name: invalid-enum
+inputs:
+  mode:
+    type: string
+    values: [fast, slow]
+    default: medium
+steps:
+  - id: step1
+    type: shell
+    run: echo test
+`;
+      const filePath = join(tempDir, 'invalid-enum.yaml');
+      writeFileSync(filePath, content);
+      expect(() => WorkflowParser.loadWorkflow(filePath)).toThrow(/Invalid workflow schema/);
+    });
+
     test('should throw on non-existent file', () => {
       expect(() => WorkflowParser.loadWorkflow('non-existent.yaml')).toThrow(
         /Failed to parse workflow/
@@ -213,4 +231,40 @@ finally:
       expect(cleanupStep?.needs).toContain('step1');
     });
   });
+
+  describe('validateStrict', () => {
+    test('should throw on invalid step schema definitions', () => {
+      const workflow = {
+        name: 'strict-invalid',
+        steps: [
+          {
+            id: 's1',
+            type: 'shell',
+            run: 'echo ok',
+            needs: [],
+            inputSchema: { type: 123 },
+          },
+        ],
+      } as unknown as Workflow;
+
+      expect(() => WorkflowParser.validateStrict(workflow)).toThrow(/Strict validation failed/);
+    });
+
+    test('should pass on valid step schema definitions', () => {
+      const workflow = {
+        name: 'strict-valid',
+        steps: [
+          {
+            id: 's1',
+            type: 'shell',
+            run: 'echo ok',
+            needs: [],
+            inputSchema: { type: 'object', properties: { run: { type: 'string' } } },
+          },
+        ],
+      } as unknown as Workflow;
+
+      expect(() => WorkflowParser.validateStrict(workflow)).not.toThrow();
+    });
+  });
 });