keystone-cli 0.7.2 → 1.0.0

package/src/runner/stream-utils.test.ts

@@ -52,6 +52,20 @@ describe('processOpenAIStream', () => {
     expect(onStream).toHaveBeenCalledTimes(1);
   });
 
+  it('handles tool calls in a final line without a newline', async () => {
+    const response = responseFromChunks([
+      'data: {"choices":[{"delta":{"tool_calls":[{"index":0,"id":"call_1","function":{"name":"final_tool","arguments":"{\\"x\\":1}"}}]}}]}',
+    ]);
+
+    const result = await processOpenAIStream(response);
+
+    expect(result.message.tool_calls?.[0]).toEqual({
+      id: 'call_1',
+      type: 'function',
+      function: { name: 'final_tool', arguments: '{"x":1}' },
+    });
+  });
+
   it('logs malformed JSON and continues processing', async () => {
     const logger = {
       log: mock(() => {}),

package/src/runner/subflow-outputs.test.ts

@@ -0,0 +1,103 @@
+import { describe, expect, it, mock, spyOn } from 'bun:test';
+import type { Workflow } from '../parser/schema';
+import { WorkflowParser } from '../parser/workflow-parser';
+import { WorkflowRegistry } from '../utils/workflow-registry';
+import { WorkflowRunner } from './workflow-runner';
+
+describe('Sub-workflow Output Mapping and Contracts', () => {
+  const dbPath = ':memory:';
+
+  it('should support workflow output schema validation', async () => {
+    const workflow: Workflow = {
+      name: 'contract-wf',
+      steps: [{ id: 's1', type: 'shell', run: 'echo "hello"', needs: [] }],
+      outputs: {
+        val: '${{ steps.s1.output.stdout.trim() }}',
+      },
+      outputSchema: {
+        type: 'object',
+        properties: {
+          val: { type: 'number' }, // Should fail because it's a string
+        },
+        required: ['val'],
+      },
+    } as unknown as Workflow;
+
+    const runner = new WorkflowRunner(workflow, { dbPath });
+    await expect(runner.run()).rejects.toThrow(/Workflow output validation failed/);
+  });
+
+  it('should support namespacing and explicit mapping for sub-workflows', async () => {
+    const childWorkflow: Workflow = {
+      name: 'child-wf',
+      steps: [
+        { id: 'cs1', type: 'shell', run: 'echo "v1"', needs: [] },
+        { id: 'cs2', type: 'shell', run: 'echo "v2"', needs: [] },
+      ],
+      outputs: {
+        foo: '${{ steps.cs1.output.stdout.trim() }}',
+        bar: '${{ steps.cs2.output.stdout.trim() }}',
+      },
+    } as unknown as Workflow;
+
+    const parentWorkflow: Workflow = {
+      name: 'parent-wf',
+      steps: [
+        {
+          id: 'sub',
+          type: 'workflow',
+          path: 'child.yaml',
+          needs: [],
+          outputMapping: {
+            mappedFoo: 'foo',
+            withDefault: { from: 'missing', default: 'fallback' },
+          },
+        },
+      ],
+      outputs: {
+        foo: '${{ steps.sub.output.mappedFoo }}',
+        rawFoo: '${{ steps.sub.output.outputs.foo }}',
+        def: '${{ steps.sub.output.withDefault }}',
+      },
+    } as unknown as Workflow;
+
+    spyOn(WorkflowRegistry, 'resolvePath').mockReturnValue('child.yaml');
+    spyOn(WorkflowParser, 'loadWorkflow').mockReturnValue(childWorkflow);
+
+    const runner = new WorkflowRunner(parentWorkflow, { dbPath });
+    const outputs = await runner.run();
+
+    expect(outputs.foo).toBe('v1');
+    expect(outputs.rawFoo).toBe('v1');
+    expect(outputs.def).toBe('fallback');
+  });
+
+  it('should fail if mapped output is missing and no default is provided', async () => {
+    const childWorkflow: Workflow = {
+      name: 'child-wf',
+      steps: [{ id: 'cs1', type: 'shell', run: 'echo "ok"', needs: [] }],
+      outputs: { ok: 'true' },
+    } as unknown as Workflow;
+
+    const parentWorkflow: Workflow = {
+      name: 'parent-wf',
+      steps: [
+        {
+          id: 'sub',
+          type: 'workflow',
+          path: 'child.yaml',
+          needs: [],
+          outputMapping: {
+            missing: 'nonexistent',
+          },
+        },
+      ],
+    } as unknown as Workflow;
+
+    spyOn(WorkflowRegistry, 'resolvePath').mockReturnValue('child.yaml');
+    spyOn(WorkflowParser, 'loadWorkflow').mockReturnValue(childWorkflow);
+
+    const runner = new WorkflowRunner(parentWorkflow, { dbPath });
+    await expect(runner.run()).rejects.toThrow(/Sub-workflow output "nonexistent" not found/);
+  });
+});

package/src/runner/test-harness.ts

@@ -0,0 +1,161 @@
+import { randomUUID } from 'node:crypto';
+import { existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { type ExpressionContext, ExpressionEvaluator } from '../expression/evaluator';
+import type { Step, Workflow } from '../parser/schema';
+import { ConsoleLogger, type Logger } from '../utils/logger';
+import type { LLMAdapter, LLMMessage, LLMResponse } from './llm-adapter';
+import { type StepExecutorOptions, type StepResult, executeStep } from './step-executor';
+import { WorkflowRunner } from './workflow-runner';
+
+export interface TestFixture {
+  inputs?: Record<string, unknown>;
+  env?: Record<string, string>;
+  secrets?: Record<string, string>;
+  mocks?: Array<{
+    step?: string;
+    type?: string;
+    prompt?: string;
+    response: unknown;
+  }>;
+}
+
+export interface TestSnapshot {
+  steps: Record<
+    string,
+    {
+      status: string;
+      output: unknown;
+      error?: string;
+    }
+  >;
+  outputs: Record<string, unknown>;
+}
+
+export class TestHarness {
+  private stepResults: Map<string, { status: string; output: unknown; error?: string }> = new Map();
+  private mockResponses: Map<string, unknown> = new Map();
+  private llmMocks: Array<{ prompt: string; response: unknown }> = [];
+
+  constructor(
+    private workflow: Workflow,
+    private fixture: TestFixture = {}
+  ) {
+    if (fixture.mocks) {
+      for (const mock of fixture.mocks) {
+        if (mock.step) {
+          this.mockResponses.set(mock.step, mock.response);
+        } else if (mock.prompt) {
+          this.llmMocks.push({ prompt: mock.prompt, response: mock.response });
+        }
+      }
+    }
+  }
+
+  async run(): Promise<TestSnapshot> {
+    // Capture original environment for cleanup
+    const originalEnv = { ...process.env };
+    const addedKeys: string[] = [];
+
+    const runner = new WorkflowRunner(this.workflow, {
+      inputs: this.fixture.inputs,
+      secrets: this.fixture.secrets,
+      executeStep: this.mockExecuteStep.bind(this),
+      getAdapter: this.getMockAdapter.bind(this),
+      // Use memory DB for tests
+      dbPath: ':memory:',
+    });
+
+    try {
+      // Inject env
+      if (this.fixture.env) {
+        for (const [key, value] of Object.entries(this.fixture.env)) {
+          if (!(key in originalEnv)) {
+            addedKeys.push(key);
+          }
+          process.env[key] = value;
+        }
+      }
+
+      const outputs = await runner.run();
+
+      return {
+        steps: Object.fromEntries(this.stepResults.entries()),
+        outputs,
+      };
+    } finally {
+      // Restore original environment
+      for (const key of addedKeys) {
+        delete process.env[key];
+      }
+      for (const [key, value] of Object.entries(originalEnv)) {
+        if (value !== undefined) {
+          process.env[key] = value;
+        }
+      }
+    }
+  }
+
+  private async mockExecuteStep(
+    step: Step,
+    context: ExpressionContext,
+    logger: Logger,
+    options: StepExecutorOptions
+  ): Promise<StepResult> {
+    const mockResponse = this.mockResponses.get(step.id);
+    if (mockResponse !== undefined) {
+      const result: StepResult = {
+        output: mockResponse,
+        status: 'success',
+      };
+      this.stepResults.set(step.id, {
+        status: result.status,
+        output: result.output,
+        error: result.error,
+      });
+      return result;
+    }
+
+    // Default to real execution but capture snapshot
+    const result = await executeStep(step, context, logger, {
+      ...options,
+      executeStep: this.mockExecuteStep.bind(this),
+      getAdapter: this.getMockAdapter.bind(this),
+    });
+
+    this.stepResults.set(step.id, {
+      status: result.status,
+      output: result.output,
+      error: result.error,
+    });
+
+    return result;
+  }
+
+  private getMockAdapter(model: string): { adapter: LLMAdapter; resolvedModel: string } {
+    return {
+      resolvedModel: model,
+      adapter: {
+        chat: async (messages: LLMMessage[]) => {
+          const userMessage = messages.find((m) => m.role === 'user')?.content || '';
+
+          for (const mock of this.llmMocks) {
+            if (userMessage.includes(mock.prompt)) {
+              return {
+                message: {
+                  role: 'assistant',
+                  content:
+                    typeof mock.response === 'string'
+                      ? mock.response
+                      : JSON.stringify(mock.response),
+                },
+              };
+            }
+          }
+
+          throw new Error(`No LLM mock found for prompt: ${userMessage.substring(0, 100)}...`);
+        },
+      },
+    };
+  }
+}

package/src/runner/tool-integration.test.ts

@@ -3,16 +3,8 @@ import { mkdirSync, unlinkSync, writeFileSync } from 'node:fs';
 import { join } from 'node:path';
 import type { ExpressionContext } from '../expression/evaluator';
 import type { LlmStep, Step } from '../parser/schema';
-import {
-  AnthropicAdapter,
-  CopilotAdapter,
-  type LLMMessage,
-  type LLMResponse,
-  type LLMTool,
-  OpenAIAdapter,
-} from './llm-adapter';
+import type { LLMAdapter } from './llm-adapter';
 import { executeLlmStep } from './llm-executor';
-import { MCPClient, type MCPResponse } from './mcp-client';
 import type { StepResult } from './step-executor';
 
 interface MockToolCall {
@@ -24,6 +16,37 @@ interface MockToolCall {
 describe('llm-executor with tools and MCP', () => {
   const agentsDir = join(process.cwd(), '.keystone', 'workflows', 'agents');
   const agentPath = join(agentsDir, 'tool-test-agent.md');
+  const createMockGetAdapter = (chatFn: LLMAdapter['chat']) => {
+    return (_modelString: string) => ({
+      adapter: { chat: chatFn } as LLMAdapter,
+      resolvedModel: 'gpt-4',
+    });
+  };
+  const createMockMcpClient = (
+    options: {
+      tools?: { name: string; description?: string; inputSchema: Record<string, unknown> }[];
+      callTool?: (name: string, args: Record<string, unknown>) => Promise<unknown>;
+    } = {}
+  ) => {
+    const listTools = mock(async () => options.tools ?? []);
+    const callTool =
+      options.callTool || (mock(async () => ({})) as unknown as typeof options.callTool);
+    return {
+      listTools,
+      callTool,
+    };
+  };
+  const createMockMcpManager = (
+    options: {
+      clients?: Record<string, ReturnType<typeof createMockMcpClient> | undefined>;
+    } = {}
+  ) => {
+    const getClient = mock(async (serverRef: string | { name: string }) => {
+      const name = typeof serverRef === 'string' ? serverRef : serverRef.name;
+      return options.clients?.[name];
+    });
+    return { getClient };
+  };
 
   beforeAll(() => {
     try {
@@ -53,9 +76,6 @@ Test system prompt`;
   });
 
   it('should merge tools from agent, step and MCP', async () => {
-    const originalOpenAIChat = OpenAIAdapter.prototype.chat;
-    const originalCopilotChat = CopilotAdapter.prototype.chat;
-    const originalAnthropicChat = AnthropicAdapter.prototype.chat;
     let capturedTools: MockToolCall[] = [];
 
     const mockChat = mock(async (_messages: unknown, options: unknown) => {
@@ -63,30 +83,21 @@ Test system prompt`;
       return {
         message: { role: 'assistant', content: 'Final response' },
       };
-    });
+    }) as unknown as LLMAdapter['chat'];
+    const getAdapter = createMockGetAdapter(mockChat);
 
-    OpenAIAdapter.prototype.chat = mockChat as unknown as typeof originalOpenAIChat;
-    CopilotAdapter.prototype.chat = mockChat as unknown as typeof originalCopilotChat;
-    AnthropicAdapter.prototype.chat = mockChat as unknown as typeof originalAnthropicChat;
-
-    // Use mock.module for MCPClient
-    const originalInitialize = MCPClient.prototype.initialize;
-    const originalListTools = MCPClient.prototype.listTools;
-    const originalStop = MCPClient.prototype.stop;
-
-    const mockInitialize = mock(async () => ({}) as MCPResponse);
-    const mockListTools = mock(async () => [
-      {
-        name: 'mcp-tool',
-        description: 'MCP tool',
-        inputSchema: { type: 'object', properties: {} },
-      },
-    ]);
-    const mockStop = mock(() => {});
-
-    MCPClient.prototype.initialize = mockInitialize;
-    MCPClient.prototype.listTools = mockListTools;
-    MCPClient.prototype.stop = mockStop;
+    const mockClient = createMockMcpClient({
+      tools: [
+        {
+          name: 'mcp-tool',
+          description: 'MCP tool',
+          inputSchema: { type: 'object', properties: {} },
+        },
+      ],
+    });
+    const mcpManager = createMockMcpManager({
+      clients: { 'test-mcp': mockClient },
+    });
 
     const step: LlmStep = {
       id: 'l1',
@@ -110,26 +121,21 @@ Test system prompt`;
     await executeLlmStep(
       step,
       context,
-      executeStepFn as unknown as (step: Step, context: ExpressionContext) => Promise<StepResult>
+      executeStepFn as unknown as (step: Step, context: ExpressionContext) => Promise<StepResult>,
+      undefined,
+      mcpManager as unknown as { getClient: () => Promise<unknown> },
+      undefined,
+      undefined,
+      getAdapter
     );
 
     const toolNames = capturedTools.map((t) => t.function.name);
     expect(toolNames).toContain('agent-tool');
     expect(toolNames).toContain('step-tool');
     expect(toolNames).toContain('mcp-tool');
-
-    OpenAIAdapter.prototype.chat = originalOpenAIChat;
-    CopilotAdapter.prototype.chat = originalCopilotChat;
-    AnthropicAdapter.prototype.chat = originalAnthropicChat;
-    MCPClient.prototype.initialize = originalInitialize;
-    MCPClient.prototype.listTools = originalListTools;
-    MCPClient.prototype.stop = originalStop;
   });
 
   it('should execute MCP tool when called', async () => {
-    const originalOpenAIChat = OpenAIAdapter.prototype.chat;
-    const originalCopilotChat = CopilotAdapter.prototype.chat;
-    const originalAnthropicChat = AnthropicAdapter.prototype.chat;
     let chatCount = 0;
 
     const mockChat = mock(async () => {
@@ -151,32 +157,23 @@ Test system prompt`;
       return {
         message: { role: 'assistant', content: 'Done' },
       };
-    });
+    }) as unknown as LLMAdapter['chat'];
+    const getAdapter = createMockGetAdapter(mockChat);
 
-    OpenAIAdapter.prototype.chat = mockChat as unknown as typeof originalOpenAIChat;
-    CopilotAdapter.prototype.chat = mockChat as unknown as typeof originalCopilotChat;
-    AnthropicAdapter.prototype.chat = mockChat as unknown as typeof originalAnthropicChat;
-
-    const originalInitialize = MCPClient.prototype.initialize;
-    const originalListTools = MCPClient.prototype.listTools;
-    const originalCallTool = MCPClient.prototype.callTool;
-    const originalStop = MCPClient.prototype.stop;
-
-    const mockInitialize = mock(async () => ({}) as MCPResponse);
-    const mockListTools = mock(async () => [
-      {
-        name: 'mcp-tool',
-        description: 'MCP tool',
-        inputSchema: { type: 'object', properties: {} },
-      },
-    ]);
     const mockCallTool = mock(async () => ({ result: 'mcp success' }));
-    const mockStop = mock(() => {});
-
-    MCPClient.prototype.initialize = mockInitialize;
-    MCPClient.prototype.listTools = mockListTools;
-    MCPClient.prototype.callTool = mockCallTool;
-    MCPClient.prototype.stop = mockStop;
+    const mockClient = createMockMcpClient({
+      tools: [
+        {
+          name: 'mcp-tool',
+          description: 'MCP tool',
+          inputSchema: { type: 'object', properties: {} },
+        },
+      ],
+      callTool: mockCallTool,
+    });
+    const mcpManager = createMockMcpManager({
+      clients: { 'test-mcp': mockClient },
+    });
 
     const step: LlmStep = {
       id: 'l1',
@@ -194,18 +191,15 @@ Test system prompt`;
     await executeLlmStep(
       step,
       context,
-      executeStepFn as unknown as (step: Step, context: ExpressionContext) => Promise<StepResult>
+      executeStepFn as unknown as (step: Step, context: ExpressionContext) => Promise<StepResult>,
+      undefined,
+      mcpManager as unknown as { getClient: () => Promise<unknown> },
+      undefined,
+      undefined,
+      getAdapter
     );
 
     expect(mockCallTool).toHaveBeenCalledWith('mcp-tool', {});
     expect(chatCount).toBe(2);
-
-    OpenAIAdapter.prototype.chat = originalOpenAIChat;
-    CopilotAdapter.prototype.chat = originalCopilotChat;
-    AnthropicAdapter.prototype.chat = originalAnthropicChat;
-    MCPClient.prototype.initialize = originalInitialize;
-    MCPClient.prototype.listTools = originalListTools;
-    MCPClient.prototype.callTool = originalCallTool;
-    MCPClient.prototype.stop = originalStop;
   });
 });