@flink-app/flink 0.14.3 → 2.0.0-alpha.48

package/spec/AgentRunner.spec.ts

@@ -0,0 +1,527 @@
+import { z } from "zod";
+import { AgentRunner } from "../src/ai/AgentRunner";
+import { FlinkAgentProps } from "../src/ai/FlinkAgent";
+import { ToolExecutor } from "../src/ai/ToolExecutor";
+import { FlinkContext } from "../src/FlinkContext";
+import { FlinkToolProps } from "../src/ai/FlinkTool";
+import { LLMAdapter } from "../src/ai/LLMAdapter";
+import { createStreamingMock } from "./testHelpers";
+
+describe("AgentRunner", () => {
+    let mockCtx: FlinkContext;
+    let mockLLMAdapter: LLMAdapter;
+
+    beforeEach(() => {
+        mockCtx = {
+            repos: {},
+            plugins: {},
+        };
+
+        // Mock LLM Adapter with default response
+        mockLLMAdapter = createStreamingMock([{
+            textContent: "Test response",
+            toolCalls: [],
+            usage: { inputTokens: 10, outputTokens: 20 },
+            stopReason: "end_turn" as const,
+        }]);
+    });
+
+    describe("Basic execution", () => {
+        it("should execute simple agent without tools", async () => {
+            const agentProps: FlinkAgentProps = {
+                id: "test_agent",
+                description: "Test agent",
+                instructions: "You are a helpful assistant",
+                tools: [],
+            };
+
+            const toolsMap = new Map();
+            const llmAdapters = new Map();
+            llmAdapters.set("default", mockLLMAdapter);
+            const runner = new AgentRunner(agentProps, toolsMap, llmAdapters);
+
+            const generator = runner.streamGenerator({ message: "Hello" });
+            const chunks: any[] = [];
+
+            for await (const chunk of generator) {
+                chunks.push(chunk);
+            }
+
+            // Should have complete event
+            expect(chunks.length).toBeGreaterThan(0);
+            const completeChunk = chunks.find((c) => c.type === "complete");
+            expect(completeChunk).toBeDefined();
+            expect(completeChunk.result.message).toBe("Test response");
+            expect(completeChunk.result.stepsUsed).toBeGreaterThan(0);
+            expect(completeChunk.result.toolCalls).toEqual([]);
+        });
+
+        it("should track token usage", async () => {
+            const agentProps: FlinkAgentProps = {
+                id: "test_agent",
+                description: "Test agent",
+                instructions: "You are a helpful assistant",
+                tools: [],
+            };
+
+            const toolsMap = new Map();
+            const llmAdapters = new Map();
+            llmAdapters.set("default", mockLLMAdapter);
+            const runner = new AgentRunner(agentProps, toolsMap, llmAdapters);
+
+            const generator = runner.streamGenerator({ message: "Hello" });
+
+            for await (const chunk of generator) {
+                if (chunk.type === "complete") {
+                    expect(chunk.result.usage).toBeDefined();
+                    expect(chunk.result.usage?.inputTokens).toBeGreaterThan(0);
+                    expect(chunk.result.usage?.outputTokens).toBeGreaterThan(0);
+                }
+            }
+        });
+    });
+
+    describe("Tool calling", () => {
+        it("should execute tool calls", async () => {
+            // Create mock tool
+            const toolProps: FlinkToolProps = {
+                id: "get_weather",
+                description: "Get weather",
+                inputSchema: z.object({ city: z.string() }),
+            };
+
+            const toolFn = jasmine.createSpy("toolFn").and.returnValue(
+                Promise.resolve({
+                    success: true,
+                    data: { temperature: 22, conditions: "sunny" },
+                })
+            );
+
+            const toolExecutor = new ToolExecutor(toolProps, toolFn as any, mockCtx);
+            const toolsMap = new Map([["get_weather", toolExecutor]]);
+
+            // Mock LLM adapter response with tool call
+            const weatherMockAdapter = createStreamingMock([
+                // First call: agent requests tool
+                {
+                    textContent: "Let me check the weather",
+                    toolCalls: [
+                        {
+                            id: "tool_1",
+                            name: "get_weather",
+                            input: { city: "Stockholm" },
+                        },
+                    ],
+                    usage: { inputTokens: 10, outputTokens: 20 },
+                    stopReason: "tool_use" as const,
+                },
+                // Second call: agent responds with result
+                {
+                    textContent: "It's sunny and 22°C",
+                    toolCalls: [],
+                    usage: { inputTokens: 15, outputTokens: 10 },
+                    stopReason: "end_turn" as const,
+                }
+            ]);
+
+            const agentProps: FlinkAgentProps = {
+                id: "weather_agent",
+                description: "Weather assistant",
+                instructions: "You help with weather",
+                tools: ["get_weather"],
+            };
+
+            const llmAdapters = new Map();
+            llmAdapters.set("default", weatherMockAdapter);
+            const runner = new AgentRunner(agentProps, toolsMap, llmAdapters);
+
+            const generator = runner.streamGenerator({ message: "What's the weather in Stockholm?" });
+
+            for await (const chunk of generator) {
+                if (chunk.type === "complete") {
+                    expect(chunk.result.message).toBe("It's sunny and 22°C");
+                    expect(chunk.result.toolCalls.length).toBe(1);
+                    expect(chunk.result.toolCalls[0].name).toBe("get_weather");
+                    expect(chunk.result.toolCalls[0].input).toEqual({ city: "Stockholm" });
+                    expect(chunk.result.toolCalls[0].output).toEqual({
+                        temperature: 22,
+                        conditions: "sunny",
+                    });
+                    expect(chunk.result.stepsUsed).toBe(2);
+                }
+            }
+
+            // Verify tool was called
+            expect(toolFn).toHaveBeenCalledWith({
+                input: { city: "Stockholm" },
+                ctx: mockCtx,
+                user: undefined,
+            });
+        });
+
+        it("should handle tool errors gracefully", async () => {
+            // Create mock tool that returns error
+            const toolProps: FlinkToolProps = {
+                id: "get_weather",
+                description: "Get weather",
+                inputSchema: z.object({ city: z.string() }),
+            };
+
+            const toolFn = jasmine.createSpy("toolFn").and.returnValue(
+                Promise.resolve({
+                    success: false,
+                    error: "API unavailable",
+                    code: "SERVICE_ERROR",
+                })
+            );
+
+            const toolExecutor = new ToolExecutor(toolProps, toolFn as any, mockCtx);
+            const toolsMap = new Map([["get_weather", toolExecutor]]);
+
+            // Mock LLM adapter response with tool call
+            const errorMockAdapter = createStreamingMock([
+                // First call: agent requests tool
+                {
+                    textContent: undefined,
+                    toolCalls: [
+                        {
+                            id: "tool_1",
+                            name: "get_weather",
+                            input: { city: "Stockholm" },
+                        },
+                    ],
+                    usage: { inputTokens: 10, outputTokens: 20 },
+                    stopReason: "tool_use" as const,
+                },
+                // Second call: agent handles error
+                {
+                    textContent: "Sorry, weather service is unavailable",
+                    toolCalls: [],
+                    usage: { inputTokens: 15, outputTokens: 10 },
+                    stopReason: "end_turn" as const,
+                }
+            ]);
+
+            const agentProps: FlinkAgentProps = {
+                id: "weather_agent",
+                description: "Weather assistant",
+                instructions: "You help with weather",
+                tools: ["get_weather"],
+            };
+
+            const llmAdapters = new Map();
+            llmAdapters.set("default", errorMockAdapter);
+            const runner = new AgentRunner(agentProps, toolsMap, llmAdapters);
+
+            const generator = runner.streamGenerator({ message: "What's the weather?" });
+
+            for await (const chunk of generator) {
+                if (chunk.type === "complete") {
+                    expect(chunk.result.toolCalls.length).toBe(1);
+                    expect(chunk.result.toolCalls[0].error).toBe("API unavailable");
+                    expect(chunk.result.toolCalls[0].output).toBeNull();
+                }
+            }
+        });
+
+        it("should handle missing tools", async () => {
+            const toolsMap = new Map(); // No tools registered
+
+            // Mock LLM adapter response requesting non-existent tool
+            const missingToolMockAdapter = createStreamingMock([
+                {
+                    textContent: undefined,
+                    toolCalls: [
+                        {
+                            id: "tool_1",
+                            name: "missing_tool",
+                            input: {},
+                        },
+                    ],
+                    usage: { inputTokens: 10, outputTokens: 20 },
+                    stopReason: "tool_use" as const,
+                },
+                {
+                    textContent: "Tool not available",
+                    toolCalls: [],
+                    usage: { inputTokens: 15, outputTokens: 10 },
+                    stopReason: "end_turn" as const,
+                }
+            ]);
+
+            const agentProps: FlinkAgentProps = {
+                id: "test_agent",
+                description: "Test agent",
+                instructions: "You are helpful",
+                tools: [],
+            };
+
+            const llmAdapters = new Map();
+            llmAdapters.set("default", missingToolMockAdapter);
+            const runner = new AgentRunner(agentProps, toolsMap, llmAdapters);
+
+            const generator = runner.streamGenerator({ message: "Test" });
+
+            for await (const chunk of generator) {
+                if (chunk.type === "complete") {
+                    expect(chunk.result.toolCalls.length).toBe(1);
+                    expect(chunk.result.toolCalls[0].error).toContain("not found");
+                }
+            }
+        });
+    });
+
+    describe("Step limits", () => {
+        it("should respect maxSteps limit", async () => {
+            // Mock infinite tool calling loop
+            const infiniteLoopMockAdapter = createStreamingMock([{
+                textContent: undefined,
+                toolCalls: [
+                    {
+                        id: "tool_1",
+                        name: "test_tool",
+                        input: {},
+                    },
+                ],
+                usage: { inputTokens: 10, outputTokens: 20 },
+                stopReason: "tool_use" as const,
+            }]);
+
+            const toolProps: FlinkToolProps = {
+                id: "test_tool",
+                description: "Test tool",
+                inputSchema: z.object({}),
+            };
+
+            const toolFn = async () => ({ success: true as const, data: {} });
+            const toolExecutor = new ToolExecutor(toolProps, toolFn, mockCtx);
+            const toolsMap = new Map([["test_tool", toolExecutor]]);
+
+            const agentProps: FlinkAgentProps = {
+                id: "test_agent",
+                description: "Test agent",
+                instructions: "You are helpful",
+                tools: ["test_tool"],
+                limits: { maxSteps: 3 },
+            };
+
+            const llmAdapters = new Map();
+            llmAdapters.set("default", infiniteLoopMockAdapter);
+            const runner = new AgentRunner(agentProps, toolsMap, llmAdapters);
+
+            const generator = runner.streamGenerator({ message: "Test" });
+
+            for await (const chunk of generator) {
+                if (chunk.type === "complete") {
+                    expect(chunk.result.stepsUsed).toBe(3);
+                    expect(chunk.result.stoppedEarly).toBe(true);
+                }
+            }
+        });
+
+        it("should allow runtime maxSteps override", async () => {
+            const runtimeOverrideMockAdapter = createStreamingMock([{
+                textContent: undefined,
+                toolCalls: [
+                    {
+                        id: "tool_1",
+                        name: "test_tool",
+                        input: {},
+                    },
+                ],
+                usage: { inputTokens: 10, outputTokens: 20 },
+                stopReason: "tool_use" as const,
+            }]);
+
+            const toolProps: FlinkToolProps = {
+                id: "test_tool",
+                description: "Test tool",
+                inputSchema: z.object({}),
+            };
+
+            const toolFn = async () => ({ success: true as const, data: {} });
+            const toolExecutor = new ToolExecutor(toolProps, toolFn, mockCtx);
+            const toolsMap = new Map([["test_tool", toolExecutor]]);
+
+            const agentProps: FlinkAgentProps = {
+                id: "test_agent",
+                description: "Test agent",
+                instructions: "You are helpful",
+                tools: ["test_tool"],
+                limits: { maxSteps: 10 },
+            };
+
+            const llmAdapters = new Map();
+            llmAdapters.set("default", runtimeOverrideMockAdapter);
+            const runner = new AgentRunner(agentProps, toolsMap, llmAdapters);
+
+            const generator = runner.streamGenerator({
+                message: "Test",
+                options: { maxSteps: 2 },
+            });
+
+            for await (const chunk of generator) {
+                if (chunk.type === "complete") {
+                    expect(chunk.result.stepsUsed).toBe(2);
+                    expect(chunk.result.stoppedEarly).toBe(true);
+                }
+            }
+        });
+    });
+
+    describe("Message format conversion", () => {
+        it("should convert string to message array", async () => {
+            const agentProps: FlinkAgentProps = {
+                id: "test_agent",
+                description: "Test agent",
+                instructions: "You are helpful",
+                tools: [],
+            };
+
+            const toolsMap = new Map();
+            const llmAdapters = new Map();
+            llmAdapters.set("default", mockLLMAdapter);
+            const runner = new AgentRunner(agentProps, toolsMap, llmAdapters);
+
+            const generator = runner.streamGenerator({ message: "Hello" });
+
+            for await (const chunk of generator) {
+                if (chunk.type === "complete") {
+                    expect(chunk.result).toBeDefined();
+                }
+            }
+
+            // Verify LLM adapter was called with user message
+            expect(mockLLMAdapter.stream).toHaveBeenCalled();
+            const callArgs = (mockLLMAdapter.stream as jasmine.Spy).calls.first().args[0];
+            expect(callArgs.messages.length).toBeGreaterThanOrEqual(1);
+            expect(callArgs.messages[0]).toEqual({ role: "user", content: "Hello" });
+        });
+
+        it("should convert Message[] to Anthropic format", async () => {
+            const agentProps: FlinkAgentProps = {
+                id: "test_agent",
+                description: "Test agent",
+                instructions: "You are helpful",
+                tools: [],
+            };
+
+            const toolsMap = new Map();
+            const llmAdapters = new Map();
+            llmAdapters.set("default", mockLLMAdapter);
+            const runner = new AgentRunner(agentProps, toolsMap, llmAdapters);
+
+            const generator = runner.streamGenerator({
+                message: [
+                    { role: "user", content: "Hello" },
+                    { role: "user", content: "How are you?" },
+                ],
+            });
+
+            for await (const chunk of generator) {
+                if (chunk.type === "complete") {
+                    expect(chunk.result).toBeDefined();
+                }
+            }
+
+            expect(mockLLMAdapter.stream).toHaveBeenCalled();
+        });
+    });
+
+    describe("Model configuration", () => {
+        it("should pass configuration to LLM adapter", async () => {
+            const agentProps: FlinkAgentProps = {
+                id: "test_agent",
+                description: "Test agent",
+                instructions: "You are helpful",
+                tools: [],
+            };
+
+            const toolsMap = new Map();
+            const llmAdapters = new Map();
+            llmAdapters.set("default", mockLLMAdapter);
+            const runner = new AgentRunner(agentProps, toolsMap, llmAdapters);
+
+            const generator = runner.streamGenerator({ message: "Hello" });
+
+            for await (const chunk of generator) {
+                if (chunk.type === "complete") {
+                    break;
+                }
+            }
+
+            // Verify LLM adapter was called via stream
+            expect(mockLLMAdapter.stream).toHaveBeenCalled();
+            const callArgs = (mockLLMAdapter.stream as jasmine.Spy).calls.mostRecent().args[0];
+            expect(callArgs.instructions).toBe("You are helpful");
+        });
+
+        it("should pass custom temperature and maxTokens", async () => {
+            const agentProps: FlinkAgentProps = {
+                id: "test_agent",
+                description: "Test agent",
+                instructions: "You are helpful",
+                tools: [],
+                model: {
+                    temperature: 0.3,
+                    maxTokens: 2000,
+                },
+            };
+
+            const toolsMap = new Map();
+            const llmAdapters = new Map();
+            llmAdapters.set("default", mockLLMAdapter);
+            const runner = new AgentRunner(agentProps, toolsMap, llmAdapters);
+
+            const generator = runner.streamGenerator({ message: "Hello" });
+
+            for await (const chunk of generator) {
+                if (chunk.type === "complete") {
+                    break;
+                }
+            }
+
+            const callArgs = (mockLLMAdapter.stream as jasmine.Spy).calls.mostRecent().args[0];
+            expect(callArgs.temperature).toBe(0.3);
+            expect(callArgs.maxTokens).toBe(2000);
+        });
+    });
+
+    describe("Error handling", () => {
+        it("should throw error when LLM adapter not configured", () => {
+            const agentProps: FlinkAgentProps = {
+                id: "test_agent",
+                description: "Test agent",
+                instructions: "You are helpful",
+                tools: [],
+            };
+
+            const toolsMap = new Map();
+            const llmAdapters = new Map(); // Empty map
+
+            expect(() => {
+                new AgentRunner(agentProps, toolsMap, llmAdapters);
+            }).toThrowError(/not configured/);
+        });
+
+        it("should throw error for unregistered adapter", () => {
+            const agentProps: FlinkAgentProps = {
+                id: "test_agent",
+                description: "Test agent",
+                instructions: "You are helpful",
+                tools: [],
+                model: {
+                    adapterId: "openai",
+                },
+            };
+
+            const toolsMap = new Map();
+            const llmAdapters = new Map();
+            llmAdapters.set("default", mockLLMAdapter);
+
+            expect(() => {
+                new AgentRunner(agentProps, toolsMap, llmAdapters);
+            }).toThrowError(/LLM adapter "openai" not configured/);
+        });
+    });
+});