keystone-cli 1.3.0 → 2.0.0

package/src/runner/__test__/llm-test-setup.ts

@@ -0,0 +1,271 @@
+/**
+ * LLM Test Setup - Helper for setting up LLM mocks
+ *
+ * This file provides helpers to mock the LLM adapter using spyOn, allowing
+ * tests to opt-in to mocking rather than having it applied globally.
+ */
+import { mock, spyOn } from 'bun:test';
+import * as llmAdapter from '../llm-adapter';
+
+// Create singleton mock functions that all test files share
+export const mockGetModel = mock();
+export const mockGetEmbeddingModel = mock();
+export const mockResetProviderRegistry = mock();
+export const mockDynamicProviderRegistry = { getProvider: mock() };
+
+// Shared types for test responses
+export interface MockLLMResponse {
+  message: {
+    role: string;
+    content?: string | null;
+    tool_calls?: Array<{
+      id: string;
+      type: 'function';
+      function: { name: string; arguments: string };
+    }>;
+  };
+  usage?: { prompt_tokens: number; completion_tokens: number; total_tokens: number };
+}
+
+// Global current chat function that tests can set
+let _currentChatFn: (messages: any[], options?: any) => Promise<MockLLMResponse> = async () => ({
+  message: { role: 'assistant', content: 'Default mock response' },
+});
+
+export function setCurrentChatFn(fn: typeof _currentChatFn) {
+  _currentChatFn = fn;
+}
+
+export function getCurrentChatFn() {
+  return _currentChatFn;
+}
+
+/**
+ * Creates a unified mock model that simulates AI SDK LanguageModel behavior.
+ */
+export function createUnifiedMockModel() {
+  return {
+    specificationVersion: 'v2',
+    provider: 'mock',
+    modelId: 'mock-model',
+    doGenerate: async (options: any) => {
+      const mapMessages = (prompt: any[]) =>
+        prompt.flatMap((m: any) => {
+          let content = m.content || '';
+          if (Array.isArray(m.content)) {
+            const toolResults = m.content.filter((p: any) => p && p.type === 'tool-result');
+            if (toolResults.length > 0) {
+              return toolResults.map((tr: any) => ({
+                role: 'tool',
+                tool_call_id: tr.toolCallId,
+                content: JSON.stringify(tr.result),
+              }));
+            }
+            const textParts = m.content
+              .filter((p: any) => p && p.type === 'text')
+              .map((p: any) => p.text)
+              .join('');
+            if (textParts) content = textParts;
+          }
+          return [
+            {
+              role: m.role,
+              content: typeof content === 'string' ? content : JSON.stringify(content),
+            },
+          ];
+        });
+
+      const messages = mapMessages(options.prompt || options.input || []);
+      const tools = (options.tools || options.mode?.tools)?.map((t: any) => ({
+        type: 'function',
+        function: {
+          name: t.name,
+          description: t.description,
+          parameters: t.parameters || t.inputSchema,
+        },
+      }));
+
+      const response = await _currentChatFn(messages, { tools });
+
+      const toolCalls = response.message.tool_calls?.map((tc: any) => ({
+        type: 'tool-call' as const,
+        toolCallId: tc.id,
+        toolName: tc.function.name,
+        args:
+          typeof tc.function.arguments === 'string'
+            ? JSON.parse(tc.function.arguments)
+            : tc.function.arguments,
+      }));
+
+      const finalToolCalls = toolCalls && toolCalls.length > 0 ? toolCalls : undefined;
+      const text = response.message.content || ' ';
+
+      // Internal AI SDK v6.0.3+ seems to expect 'content' on the result object
+      // during generateText processing, even if not in the official v2 spec.
+      const content: any[] = [];
+      if (text) {
+        content.push({ type: 'text', text });
+      }
+      if (finalToolCalls && finalToolCalls.length > 0) {
+        for (const tc of finalToolCalls) {
+          content.push({
+            type: 'tool-call',
+            toolCallId: tc.toolCallId,
+            toolName: tc.toolName,
+            args: tc.args,
+            input: JSON.stringify(tc.args), // Add required input field
+          });
+        }
+      }
+
+      return {
+        text,
+        content,
+        toolCalls: finalToolCalls,
+        finishReason: finalToolCalls ? 'tool-calls' : 'stop',
+        usage: { promptTokens: 10, completionTokens: 5 },
+        rawResponse: { headers: {} },
+        responseMessages: [
+          {
+            role: 'assistant',
+            content,
+          },
+        ],
+      } as any;
+    },
+    doStream: async (options: any) => {
+      const mapMessages = (prompt: any[]) =>
+        prompt.flatMap((m: any) => {
+          let content = m.content || '';
+          if (Array.isArray(m.content)) {
+            const toolResults = m.content.filter((p: any) => p && p.type === 'tool-result');
+            if (toolResults.length > 0) {
+              return toolResults.map((tr: any) => ({
+                role: 'tool',
+                tool_call_id: tr.toolCallId,
+                content: JSON.stringify(tr.result),
+              }));
+            }
+            const textParts = m.content
+              .filter((p: any) => p && p.type === 'text')
+              .map((p: any) => p.text)
+              .join('');
+            if (textParts) content = textParts;
+          }
+          return [
+            {
+              role: m.role,
+              content: typeof content === 'string' ? content : JSON.stringify(content),
+            },
+          ];
+        });
+
+      const messages = mapMessages(options.prompt || options.input || []);
+      const tools = (options.tools || options.mode?.tools)?.map((t: any) => ({
+        type: 'function',
+        function: {
+          name: t.name,
+          description: t.description,
+          parameters: t.parameters || t.inputSchema,
+        },
+      }));
+
+      const response = await _currentChatFn(messages, { tools });
+
+      const stream = new ReadableStream({
+        async start(controller) {
+          if (response.message.content) {
+            controller.enqueue({
+              type: 'text-delta',
+              delta: response.message.content,
+              text: response.message.content,
+            });
+          }
+
+          const toolCalls = response.message.tool_calls?.map((tc: any) => ({
+            type: 'tool-call' as const,
+            toolCallId: tc.id,
+            toolName: tc.function.name,
+            args:
+              typeof tc.function.arguments === 'string'
+                ? JSON.parse(tc.function.arguments)
+                : tc.function.arguments,
+            id: tc.id,
+            name: tc.function.name,
+            input:
+              typeof tc.function.arguments === 'string'
+                ? tc.function.arguments
+                : JSON.stringify(tc.function.arguments),
+          }));
+
+          if (toolCalls?.length) {
+            for (const tc of toolCalls) {
+              controller.enqueue(tc);
+            }
+          }
+
+          controller.enqueue({
+            type: 'finish',
+            finishReason: toolCalls?.length ? 'tool-calls' : 'stop',
+            usage: { promptTokens: 10, completionTokens: 5 },
+          });
+
+          controller.close();
+        },
+      });
+
+      return { stream, rawResponse: { headers: {} } };
+    },
+    doEmbed: async (options: any) => {
+      return {
+        embeddings: options.values.map(() => [0.1, 0.2, 0.3]),
+        usage: { tokens: 5 },
+      };
+    },
+  };
+}
+
+/**
+ * Sets up the LLM mocks by mocking the provider packages.
+ * This allows llm-adapter to run its real logic but return mock models.
+ */
+
+import { resetProviderRegistry } from '../llm-adapter';
+
+export function setupLlmMocks() {
+  resetProviderRegistry(); // Clear cache to ensure new mock is used
+
+  // Provider factory (e.g. createOpenAI) returns a Provider Instance function
+  const mockProviderInstance = (modelId: string) => createUnifiedMockModel();
+  const mockProviderFactory = (options?: any) => mockProviderInstance;
+
+  // Add properties that some providers might export (like 'openai' object)
+  const mockProviderModule = {
+    openai: mockProviderFactory,
+    createOpenAI: mockProviderFactory,
+    anthropic: mockProviderFactory,
+    createAnthropic: mockProviderFactory,
+    google: mockProviderFactory,
+    createGoogleGenerativeAI: mockProviderFactory,
+    default: mockProviderFactory,
+  };
+
+  // Mock the provider packages
+  mock.module('@ai-sdk/openai', () => mockProviderModule);
+  mock.module('@ai-sdk/anthropic', () => mockProviderModule);
+  mock.module('@ai-sdk/google', () => mockProviderModule);
+
+  _currentChatFn = async () => ({
+    message: { role: 'assistant', content: 'Default mock response' },
+  });
+}
+
+/**
+ * Resets all mocks to default state. Call in afterEach.
+ */
+export function resetLlmMocks() {
+  resetProviderRegistry();
+  _currentChatFn = async () => ({
+    message: { role: 'assistant', content: 'Default mock response' },
+  });
+}

package/src/runner/engine-executor.test.ts

@@ -73,14 +73,21 @@ describe('engine-executor', () => {
     });
 
     it('should reject if PATH is not in env for non-absolute command', async () => {
-      const step = createStep({
-        cwd: '/tmp',
-        env: { HOME: '/home' },
-      });
+      const originalPath = process.env.PATH;
+      process.env.PATH = undefined;
 
-      await expect(
-        executeEngineStep(step, { inputs: {}, secrets: {}, env: {}, steps: {} })
-      ).rejects.toThrow('requires env.PATH');
+      try {
+        const step = createStep({
+          cwd: '/tmp',
+          env: { HOME: '/home' },
+        });
+
+        await expect(
+          executeEngineStep(step, { inputs: {}, secrets: {}, env: {}, steps: {} })
+        ).rejects.toThrow('requires env.PATH');
+      } finally {
+        process.env.PATH = originalPath;
+      }
     });
 
     it('should reject if command is denied', async () => {
@@ -90,7 +97,7 @@ describe('engine-executor', () => {
           denylist: ['rm', 'dd'],
           allowlist: {},
         },
-      } as ReturnType<typeof ConfigLoader.ConfigLoader.load>);
+      } as unknown as ReturnType<typeof ConfigLoader.ConfigLoader.load>);
 
       try {
         const step = createStep({
@@ -117,7 +124,7 @@ describe('engine-executor', () => {
             python: { command: 'python3', version: '3.11', versionArgs: [], args: [] },
           },
         },
-      } as ReturnType<typeof ConfigLoader.ConfigLoader.load>);
+      } as unknown as ReturnType<typeof ConfigLoader.ConfigLoader.load>);
 
       try {
         const step = createStep({
@@ -143,7 +150,7 @@ describe('engine-executor', () => {
             echo: { command: 'echo', version: '', versionArgs: [], args: [] },
           },
         },
-      } as ReturnType<typeof ConfigLoader.ConfigLoader.load>);
+      } as unknown as ReturnType<typeof ConfigLoader.ConfigLoader.load>);
 
       try {
         mkdirSync(tempDir, { recursive: true });
@@ -175,7 +182,7 @@ describe('engine-executor', () => {
             echo: { command: 'echo', version: '999.0.0', versionArgs: [], args: [] },
           },
         },
-      } as ReturnType<typeof ConfigLoader.ConfigLoader.load>);
+      } as unknown as ReturnType<typeof ConfigLoader.ConfigLoader.load>);
 
       try {
         mkdirSync(tempDir, { recursive: true });
@@ -207,7 +214,7 @@ describe('engine-executor', () => {
             echo: { command: 'echo', version: '', versionArgs: [], args: [] },
           },
         },
-      } as ReturnType<typeof ConfigLoader.ConfigLoader.load>);
+      } as unknown as ReturnType<typeof ConfigLoader.ConfigLoader.load>);
 
       try {
         mkdirSync(tempDir, { recursive: true });
@@ -241,7 +248,7 @@ describe('engine-executor', () => {
             sh: { command: 'sh', version: '', versionArgs: [], args: [] },
           },
         },
-      } as ReturnType<typeof ConfigLoader.ConfigLoader.load>);
+      } as unknown as ReturnType<typeof ConfigLoader.ConfigLoader.load>);
 
       try {
         mkdirSync(tempDir, { recursive: true });
@@ -274,7 +281,7 @@ describe('engine-executor', () => {
             sh: { command: 'sh', version: '', versionArgs: [], args: [] },
           },
         },
-      } as ReturnType<typeof ConfigLoader.ConfigLoader.load>);
+      } as unknown as ReturnType<typeof ConfigLoader.ConfigLoader.load>);
 
       try {
         mkdirSync(tempDir, { recursive: true });
@@ -310,7 +317,7 @@ describe('engine-executor', () => {
             echo: { command: 'echo', version: '', versionArgs: [], args: [] },
           },
         },
-      } as ReturnType<typeof ConfigLoader.ConfigLoader.load>);
+      } as unknown as ReturnType<typeof ConfigLoader.ConfigLoader.load>);
 
       try {
         mkdirSync(tempDir, { recursive: true });
@@ -343,7 +350,7 @@ describe('engine-executor', () => {
             echo: { command: 'echo', version: '', versionArgs: [], args: [] },
           },
         },
-      } as ReturnType<typeof ConfigLoader.ConfigLoader.load>);
+      } as unknown as ReturnType<typeof ConfigLoader.ConfigLoader.load>);
 
       try {
         mkdirSync(tempDir, { recursive: true });
@@ -382,7 +389,7 @@ describe('engine-executor', () => {
             echo: { command: 'echo', version: '', versionArgs: [], args: [] },
           },
         },
-      } as ReturnType<typeof ConfigLoader.ConfigLoader.load>);
+      } as unknown as ReturnType<typeof ConfigLoader.ConfigLoader.load>);
 
       try {
         mkdirSync(tempDir, { recursive: true });
@@ -414,7 +421,7 @@ describe('engine-executor', () => {
             echo: { command: 'echo', version: 'test', versionArgs: ['test'], args: [] },
           },
         },
-      } as ReturnType<typeof ConfigLoader.ConfigLoader.load>);
+      } as unknown as ReturnType<typeof ConfigLoader.ConfigLoader.load>);
 
       try {
         mkdirSync(tempDir, { recursive: true });

package/src/runner/executors/blueprint-executor.ts

@@ -129,7 +129,6 @@ export async function executeBlueprintStep(
     mcpManager,
     workflowDir,
     abortSignal,
-    undefined,
     emitEvent,
     workflowName ? { runId, workflow: workflowName } : undefined
   );

package/src/runner/executors/dynamic-executor.ts

@@ -9,7 +9,7 @@ import { DynamicStateManager } from '../../db/dynamic-state-manager.ts';
 import type { WorkflowDb } from '../../db/workflow-db.ts';
 import type { ExpressionContext } from '../../expression/evaluator.ts';
 import { ExpressionEvaluator } from '../../expression/evaluator.ts';
-import type { DynamicStep, LlmStep, Step } from '../../parser/schema.ts';
+import type { DynamicStep, HumanStep, LlmStep, Step } from '../../parser/schema.ts';
 import type { Logger } from '../../utils/logger.ts';
 import { topologicalSort } from '../../utils/topo-sort.ts';
 import type { WorkflowEvent } from '../events.ts';
@@ -173,7 +173,7 @@ function convertToExecutableStep(
         ...baseProps,
         type: 'file' as const,
         path: generated.path || '',
-        op: (generated.op as any) || (generated.inputs?.op as any) || 'read',
+        op: generated.op || (generated.inputs?.op as 'read' | 'write' | 'append') || 'read',
         content: generated.content || (generated.inputs?.content as string),
       };
 
@@ -307,7 +307,7 @@ async function initializeState(
         startedAt: dbState.startedAt,
         completedAt: dbState.completedAt,
         error: dbState.error,
-        replanCount: (dbState as any).replanCount || 0,
+        replanCount: dbState.replanCount || 0,
       };
     } else {
       dbState = await stateManager.create({ runId, stepId: step.id, workflowId: state.workflowId });
@@ -379,7 +379,6 @@ async function handlePlanningPhase(
     mcpManager,
     workflowDir,
     abortSignal,
-    undefined,
     emitEvent,
     workflowName && runId ? { runId, workflow: workflowName } : undefined
   );
@@ -428,7 +427,13 @@ async function handleConfirmationPhase(
   const planJson = JSON.stringify(state.generatedPlan, null, 2);
   const message = `Please review and confirm the generated plan:\n\n${planJson}\n\nType 'yes' to confirm or provide a modified JSON plan:`;
 
-  const humanStep: any = { id: `${step.id}_confirm`, type: 'human', message, inputType: 'text' };
+  const humanStep: HumanStep = {
+    id: `${step.id}_confirm`,
+    type: 'human',
+    message,
+    inputType: 'text',
+    needs: [],
+  };
   const confirmResult = await (options.executeHumanStep || executeHumanStep)(
     humanStep,
     context,
@@ -677,7 +682,7 @@ async function handleExecutionError(
   dbState: DynamicStepState | null,
   stateManager: DynamicStateManager | null,
   saveState: ((stepId: string, state: DynamicStepState) => Promise<void>) | undefined,
-  error: any
+  error: unknown
 ): Promise<StepResult> {
   state.status = 'failed';
   state.error = error instanceof Error ? error.message : String(error);

package/src/runner/executors/engine-executor.ts

@@ -8,6 +8,7 @@ import { ExpressionEvaluator } from '../../expression/evaluator';
 import type { EngineStep } from '../../parser/schema';
 import { ConfigLoader } from '../../utils/config-loader';
 import { LIMITS } from '../../utils/constants';
+import { filterSensitiveEnv } from '../../utils/env-filter';
 import { extractJson } from '../../utils/json-parser';
 import { ConsoleLogger, type Logger } from '../../utils/logger';
 import type { StepResult } from './types.ts';
@@ -294,7 +295,10 @@ export async function executeEngineStep(
   // This means args are passed directly to the process without shell interpretation.
   // Combined with the allowlist and version check, this is secure against injection.
 
-  const env: Record<string, string> = {};
+  // Inherit safe host environment variables
+  const hostEnv = filterSensitiveEnv(process.env);
+  const env: Record<string, string> = { ...hostEnv };
+
   for (const [key, value] of Object.entries(step.env || {})) {
     env[key] = ExpressionEvaluator.evaluateString(value, context);
   }