@copilotkit/runtime 0.0.0-0.0.0-max-changeset-10101010101010-20260109191632

package/tests/service-adapters/anthropic/anthropic-adapter.test.ts

@@ -0,0 +1,389 @@
+/**
+ * @jest-environment node
+ */
+
+import { AnthropicAdapter } from "../../../src/service-adapters/anthropic/anthropic-adapter";
+
+// Mock only the Anthropic SDK, not our adapter
+jest.mock("@anthropic-ai/sdk", () => {
+  return {
+    default: jest.fn().mockImplementation(() => ({
+      messages: {
+        create: jest.fn(),
+      },
+    })),
+  };
+});
+
+// Mock the message classes
+jest.mock("../../../src/graphql/types/converted", () => {
+  class MockTextMessage {
+    content: string;
+    role: string;
+    id: string;
+
+    constructor(role: string, content: string) {
+      this.role = role;
+      this.content = content;
+      this.id = "mock-text-" + Math.random().toString(36).substring(7);
+    }
+
+    isTextMessage() {
+      return true;
+    }
+    isImageMessage() {
+      return false;
+    }
+    isActionExecutionMessage() {
+      return false;
+    }
+    isResultMessage() {
+      return false;
+    }
+  }
+
+  class MockActionExecutionMessage {
+    id: string;
+    name: string;
+    arguments: string;
+
+    constructor(params: { id: string; name: string; arguments: string }) {
+      this.id = params.id;
+      this.name = params.name;
+      this.arguments = params.arguments;
+    }
+
+    isTextMessage() {
+      return false;
+    }
+    isImageMessage() {
+      return false;
+    }
+    isActionExecutionMessage() {
+      return true;
+    }
+    isResultMessage() {
+      return false;
+    }
+  }
+
+  class MockResultMessage {
+    actionExecutionId: string;
+    result: string;
+    id: string;
+
+    constructor(params: { actionExecutionId: string; result: string }) {
+      this.actionExecutionId = params.actionExecutionId;
+      this.result = params.result;
+      this.id = "mock-result-" + Math.random().toString(36).substring(7);
+    }
+
+    isTextMessage() {
+      return false;
+    }
+    isImageMessage() {
+      return false;
+    }
+    isActionExecutionMessage() {
+      return false;
+    }
+    isResultMessage() {
+      return true;
+    }
+  }
+
+  return {
+    TextMessage: MockTextMessage,
+    ActionExecutionMessage: MockActionExecutionMessage,
+    ResultMessage: MockResultMessage,
+  };
+});
+
+describe("AnthropicAdapter", () => {
+  let adapter: AnthropicAdapter;
+  let mockEventSource: any;
+  let mockAnthropicCreate: jest.Mock;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+
+    // Create a mock Anthropic instance
+    const mockAnthropic = {
+      messages: {
+        create: jest.fn(),
+      },
+    };
+
+    // Create adapter with the mocked instance
+    adapter = new AnthropicAdapter({ anthropic: mockAnthropic as any });
+
+    // Mock the create method to capture what's being sent
+    mockAnthropicCreate = mockAnthropic.messages.create;
+
+    mockEventSource = {
+      stream: jest.fn((callback) => {
+        const mockStream = {
+          sendTextMessageStart: jest.fn(),
+          sendTextMessageContent: jest.fn(),
+          sendTextMessageEnd: jest.fn(),
+          sendActionExecutionStart: jest.fn(),
+          sendActionExecutionArgs: jest.fn(),
+          sendActionExecutionEnd: jest.fn(),
+          complete: jest.fn(),
+        };
+        callback(mockStream);
+        return Promise.resolve();
+      }),
+    };
+  });
+
+  describe("Deduplication Logic", () => {
+    it("should filter out duplicate result messages", async () => {
+      const {
+        TextMessage,
+        ActionExecutionMessage,
+        ResultMessage,
+      } = require("../../../src/graphql/types/converted");
+
+      const systemMessage = new TextMessage("system", "System message");
+      const userMessage = new TextMessage("user", "Set theme to orange");
+
+      // Tool execution
+      const toolExecution = new ActionExecutionMessage({
+        id: "tool-123",
+        name: "setThemeColor",
+        arguments: '{"themeColor": "orange"}',
+      });
+
+      // Multiple duplicate results (this was causing the infinite loop)
+      const result1 = new ResultMessage({
+        actionExecutionId: "tool-123",
+        result: "Theme color set to orange",
+      });
+      const result2 = new ResultMessage({
+        actionExecutionId: "tool-123",
+        result: "Theme color set to orange",
+      });
+      const result3 = new ResultMessage({
+        actionExecutionId: "tool-123",
+        result: "Theme color set to orange",
+      });
+
+      // Mock Anthropic to return empty stream (simulating the original problem)
+      mockAnthropicCreate.mockResolvedValue({
+        [Symbol.asyncIterator]: async function* () {
+          // Empty stream - no content from Anthropic
+        },
+      });
+
+      await adapter.process({
+        threadId: "test-thread",
+        model: "claude-3-5-sonnet-latest",
+        messages: [systemMessage, userMessage, toolExecution, result1, result2, result3],
+        actions: [],
+        eventSource: mockEventSource,
+        forwardedParameters: {},
+      });
+
+      // Check that only one result message was sent to Anthropic
+      expect(mockAnthropicCreate).toHaveBeenCalledWith(
+        expect.objectContaining({
+          messages: expect.arrayContaining([
+            expect.objectContaining({
+              role: "user",
+              content: [{ type: "text", text: "Set theme to orange" }],
+            }),
+            expect.objectContaining({
+              role: "assistant",
+              content: [
+                expect.objectContaining({
+                  type: "tool_use",
+                  id: "tool-123",
+                  name: "setThemeColor",
+                }),
+              ],
+            }),
+            expect.objectContaining({
+              role: "user",
+              content: [
+                expect.objectContaining({
+                  type: "tool_result",
+                  content: "Theme color set to orange",
+                  tool_use_id: "tool-123",
+                }),
+              ],
+            }),
+          ]),
+        }),
+      );
+
+      // Verify only 3 messages sent (user, assistant tool_use, user tool_result) - no duplicates
+      const sentMessages = mockAnthropicCreate.mock.calls[0][0].messages;
+      expect(sentMessages).toHaveLength(3);
+
+      // Count tool_result messages - should be only 1
+      const toolResults = sentMessages.filter(
+        (msg: any) => msg.role === "user" && msg.content.some((c: any) => c.type === "tool_result"),
+      );
+      expect(toolResults).toHaveLength(1);
+    });
+
+    it("should filter out invalid result messages without corresponding tool_use", async () => {
+      const {
+        TextMessage,
+        ActionExecutionMessage,
+        ResultMessage,
+      } = require("../../../src/graphql/types/converted");
+
+      const systemMessage = new TextMessage("system", "System message");
+
+      // Valid tool execution
+      const validTool = new ActionExecutionMessage({
+        id: "valid-tool",
+        name: "validAction",
+        arguments: "{}",
+      });
+      const validResult = new ResultMessage({
+        actionExecutionId: "valid-tool",
+        result: "Valid result",
+      });
+
+      // Invalid result with no corresponding tool_use
+      const invalidResult = new ResultMessage({
+        actionExecutionId: "nonexistent-tool",
+        result: "Invalid result",
+      });
+
+      mockAnthropicCreate.mockResolvedValue({
+        [Symbol.asyncIterator]: async function* () {},
+      });
+
+      await adapter.process({
+        threadId: "test-thread",
+        messages: [systemMessage, validTool, validResult, invalidResult],
+        actions: [],
+        eventSource: mockEventSource,
+        forwardedParameters: {},
+      });
+
+      const sentMessages = mockAnthropicCreate.mock.calls[0][0].messages;
+
+      // Should only include the valid tool result
+      const toolResults = sentMessages.filter(
+        (msg: any) => msg.role === "user" && msg.content.some((c: any) => c.type === "tool_result"),
+      );
+      expect(toolResults).toHaveLength(1);
+      expect(toolResults[0].content[0].tool_use_id).toBe("valid-tool");
+    });
+  });
+
+  describe("Fallback Response Logic", () => {
+    it("should generate contextual fallback when Anthropic returns no content", async () => {
+      const {
+        TextMessage,
+        ActionExecutionMessage,
+        ResultMessage,
+      } = require("../../../src/graphql/types/converted");
+
+      const systemMessage = new TextMessage("system", "System message");
+      const userMessage = new TextMessage("user", "Set theme to orange");
+
+      const toolExecution = new ActionExecutionMessage({
+        id: "tool-123",
+        name: "setThemeColor",
+        arguments: '{"themeColor": "orange"}',
+      });
+
+      const toolResult = new ResultMessage({
+        actionExecutionId: "tool-123",
+        result: "Theme color set to orange",
+      });
+
+      // Mock Anthropic to return empty stream
+      mockAnthropicCreate.mockResolvedValue({
+        [Symbol.asyncIterator]: async function* () {
+          // No content blocks - simulates Anthropic not responding
+        },
+      });
+
+      const mockStream = {
+        sendTextMessageStart: jest.fn(),
+        sendTextMessageContent: jest.fn(),
+        sendTextMessageEnd: jest.fn(),
+        complete: jest.fn(),
+      };
+
+      mockEventSource.stream.mockImplementation((callback) => {
+        callback(mockStream);
+        return Promise.resolve();
+      });
+
+      await adapter.process({
+        threadId: "test-thread",
+        messages: [systemMessage, userMessage, toolExecution, toolResult],
+        actions: [],
+        eventSource: mockEventSource,
+        forwardedParameters: {},
+      });
+
+      // Should generate fallback response with the tool result content
+      expect(mockStream.sendTextMessageStart).toHaveBeenCalled();
+      expect(mockStream.sendTextMessageContent).toHaveBeenCalledWith({
+        messageId: expect.any(String),
+        content: "Theme color set to orange", // Uses the actual result content
+      });
+      expect(mockStream.sendTextMessageEnd).toHaveBeenCalled();
+    });
+
+    it("should use generic fallback when no tool result content available", async () => {
+      const {
+        TextMessage,
+        ActionExecutionMessage,
+        ResultMessage,
+      } = require("../../../src/graphql/types/converted");
+
+      const systemMessage = new TextMessage("system", "System message");
+
+      const toolExecution = new ActionExecutionMessage({
+        id: "tool-123",
+        name: "someAction",
+        arguments: "{}",
+      });
+
+      const toolResult = new ResultMessage({
+        actionExecutionId: "tool-123",
+        result: "", // Empty result
+      });
+
+      mockAnthropicCreate.mockResolvedValue({
+        [Symbol.asyncIterator]: async function* () {},
+      });
+
+      const mockStream = {
+        sendTextMessageStart: jest.fn(),
+        sendTextMessageContent: jest.fn(),
+        sendTextMessageEnd: jest.fn(),
+        complete: jest.fn(),
+      };
+
+      mockEventSource.stream.mockImplementation((callback) => {
+        callback(mockStream);
+        return Promise.resolve();
+      });
+
+      await adapter.process({
+        threadId: "test-thread",
+        messages: [systemMessage, toolExecution, toolResult],
+        actions: [],
+        eventSource: mockEventSource,
+        forwardedParameters: {},
+      });
+
+      // Should use generic fallback
+      expect(mockStream.sendTextMessageContent).toHaveBeenCalledWith({
+        messageId: expect.any(String),
+        content: "Task completed successfully.",
+      });
+    });
+  });
+});

package/tests/service-adapters/openai/allowlist-approach.test.ts

@@ -0,0 +1,238 @@
+/**
+ * @jest-environment node
+ */
+
+import { describe, it, expect } from "@jest/globals";
+
+describe("OpenAI Adapter - Allowlist Approach", () => {
+  it("should filter out tool_result messages with no corresponding tool_call ID", () => {
+    // Setup test data
+    const validToolCallIds = new Set<string>(["valid-id-1", "valid-id-2"]);
+
+    // Messages to filter - valid and invalid ones
+    const messages = [
+      { type: "text", role: "user", content: "Hello" },
+      { type: "tool_result", actionExecutionId: "valid-id-1", result: "result1" },
+      { type: "tool_result", actionExecutionId: "invalid-id", result: "invalid" },
+      { type: "tool_result", actionExecutionId: "valid-id-2", result: "result2" },
+      { type: "tool_result", actionExecutionId: "valid-id-1", result: "duplicate" }, // Duplicate ID
+    ];
+
+    // Implement the filtering logic, similar to the adapter
+    const filteredMessages = messages.filter((message) => {
+      if (message.type === "tool_result") {
+        // Skip if there's no corresponding tool_call
+        if (!validToolCallIds.has(message.actionExecutionId)) {
+          return false;
+        }
+
+        // Remove this ID from valid IDs so we don't process duplicates
+        validToolCallIds.delete(message.actionExecutionId);
+      }
+
+      // Keep all non-tool-result messages
+      return true;
+    });
+
+    // Verify results
+    expect(filteredMessages.length).toBe(3); // text + 2 valid tool results (no duplicates or invalid)
+
+    // Valid results should be included
+    expect(
+      filteredMessages.some(
+        (m) => m.type === "tool_result" && m.actionExecutionId === "valid-id-1",
+      ),
+    ).toBe(true);
+
+    expect(
+      filteredMessages.some(
+        (m) => m.type === "tool_result" && m.actionExecutionId === "valid-id-2",
+      ),
+    ).toBe(true);
+
+    // Invalid result should be excluded
+    expect(
+      filteredMessages.some(
+        (m) => m.type === "tool_result" && m.actionExecutionId === "invalid-id",
+      ),
+    ).toBe(false);
+
+    // Duplicate should be excluded - we used a different approach than Anthropic
+    const validId1Count = filteredMessages.filter(
+      (m) => m.type === "tool_result" && m.actionExecutionId === "valid-id-1",
+    ).length;
+
+    expect(validId1Count).toBe(1);
+  });
+
+  it("should maintain correct order of messages when filtering", () => {
+    // Setup test data
+    const validToolCallIds = new Set<string>(["tool-1", "tool-2", "tool-3"]);
+
+    // Test with a complex conversation pattern
+    const messages = [
+      { type: "text", role: "user", content: "Initial message" },
+      { type: "text", role: "assistant", content: "I'll help with that" },
+      { type: "tool_call", id: "tool-1", name: "firstTool" },
+      { type: "tool_result", actionExecutionId: "tool-1", result: "result1" },
+      { type: "text", role: "assistant", content: "Got the first result" },
+      { type: "tool_call", id: "tool-2", name: "secondTool" },
+      { type: "tool_result", actionExecutionId: "tool-2", result: "result2" },
+      { type: "tool_result", actionExecutionId: "invalid-id", result: "invalid-result" },
+      { type: "tool_call", id: "tool-3", name: "thirdTool" },
+      { type: "tool_result", actionExecutionId: "tool-1", result: "duplicate-result" }, // Duplicate
+      { type: "tool_result", actionExecutionId: "tool-3", result: "result3" },
+      { type: "text", role: "user", content: "Final message" },
+    ];
+
+    // Apply OpenAI's filter approach (using filter instead of loop)
+    const filteredMessages = messages.filter((message) => {
+      if (message.type === "tool_result") {
+        // Skip if there's no corresponding tool_call
+        if (!validToolCallIds.has(message.actionExecutionId)) {
+          return false;
+        }
+
+        // Remove this ID from valid IDs so we don't process duplicates
+        validToolCallIds.delete(message.actionExecutionId);
+      }
+
+      // Keep all non-tool-result messages
+      return true;
+    });
+
+    // Verify results
+    expect(filteredMessages.length).toBe(10); // 12 original - 2 filtered out
+
+    // Check that the message order is preserved
+    expect(filteredMessages[0].type).toBe("text"); // Initial user message
+    expect(filteredMessages[0].content).toBe("Initial message");
+    expect(filteredMessages[1].type).toBe("text"); // Assistant response
+    expect(filteredMessages[2].type).toBe("tool_call"); // First tool
+    expect(filteredMessages[3].type).toBe("tool_result"); // First result
+    expect(filteredMessages[3].actionExecutionId).toBe("tool-1");
+    expect(filteredMessages[4].type).toBe("text"); // Assistant comment
+    expect(filteredMessages[5].type).toBe("tool_call"); // Second tool
+    expect(filteredMessages[6].type).toBe("tool_result"); // Second result
+    expect(filteredMessages[6].actionExecutionId).toBe("tool-2");
+    expect(filteredMessages[7].type).toBe("tool_call"); // Third tool
+    expect(filteredMessages[8].type).toBe("tool_result"); // Third result
+    expect(filteredMessages[8].actionExecutionId).toBe("tool-3");
+    expect(filteredMessages[9].type).toBe("text"); // Final user message
+
+    // Each valid tool result should appear exactly once
+    const toolResultCounts = new Map();
+    filteredMessages.forEach((message) => {
+      if (message.type === "tool_result") {
+        const id = message.actionExecutionId;
+        toolResultCounts.set(id, (toolResultCounts.get(id) || 0) + 1);
+      }
+    });
+
+    expect(toolResultCounts.size).toBe(3); // Should have 3 different tool results
+    expect(toolResultCounts.get("tool-1")).toBe(1);
+    expect(toolResultCounts.get("tool-2")).toBe(1);
+    expect(toolResultCounts.get("tool-3")).toBe(1);
+    expect(toolResultCounts.has("invalid-id")).toBe(false);
+  });
+
+  it("should handle empty message array", () => {
+    const validToolCallIds = new Set<string>(["valid-id-1", "valid-id-2"]);
+    const messages = [];
+
+    // Apply OpenAI's filter approach
+    const filteredMessages = messages.filter((message) => {
+      if (message.type === "tool_result") {
+        if (!validToolCallIds.has(message.actionExecutionId)) {
+          return false;
+        }
+        validToolCallIds.delete(message.actionExecutionId);
+      }
+      return true;
+    });
+
+    expect(filteredMessages.length).toBe(0);
+  });
+
+  it("should handle edge cases with mixed message types", () => {
+    // Setup test data with various message types
+    const validToolCallIds = new Set<string>(["valid-id-1"]);
+
+    const messages = [
+      { type: "text", role: "user", content: "Hello" },
+      { type: "image", url: "https://example.com/image.jpg" }, // Non-tool message type
+      { type: "tool_result", actionExecutionId: "valid-id-1", result: "result1" },
+      { type: "custom", data: { key: "value" } }, // Another custom type
+      { type: "tool_result", actionExecutionId: "valid-id-1", result: "duplicate" }, // Duplicate
+      { type: "null", value: null }, // Edge case
+      { type: "undefined" }, // Edge case
+    ];
+
+    // Apply OpenAI's filter approach
+    const filteredMessages = messages.filter((message) => {
+      if (message.type === "tool_result") {
+        if (!validToolCallIds.has(message.actionExecutionId)) {
+          return false;
+        }
+        validToolCallIds.delete(message.actionExecutionId);
+      }
+      return true;
+    });
+
+    // Should have all non-tool_result messages + 1 valid tool_result
+    expect(filteredMessages.length).toBe(6); // 7 original - 1 duplicate
+
+    // Valid tool_result should be included exactly once
+    const toolResults = filteredMessages.filter((m) => m.type === "tool_result");
+    expect(toolResults.length).toBe(1);
+    expect(toolResults[0].actionExecutionId).toBe("valid-id-1");
+
+    // All non-tool_result messages should be preserved
+    expect(filteredMessages.filter((m) => m.type === "text").length).toBe(1);
+    expect(filteredMessages.filter((m) => m.type === "image").length).toBe(1);
+    expect(filteredMessages.filter((m) => m.type === "custom").length).toBe(1);
+    expect(filteredMessages.filter((m) => m.type === "null").length).toBe(1);
+    expect(filteredMessages.filter((m) => m.type === "undefined").length).toBe(1);
+  });
+
+  it("should properly handle multiple duplicate tool results", () => {
+    // Setup test data with multiple duplicates
+    const validToolCallIds = new Set<string>(["tool-1", "tool-2"]);
+
+    const messages = [
+      { type: "text", role: "user", content: "Initial prompt" },
+      { type: "tool_call", id: "tool-1", name: "firstTool" },
+      { type: "tool_result", actionExecutionId: "tool-1", result: "first-result" },
+      { type: "tool_result", actionExecutionId: "tool-1", result: "duplicate-1" }, // Duplicate 1
+      { type: "tool_call", id: "tool-2", name: "secondTool" },
+      { type: "tool_result", actionExecutionId: "tool-1", result: "duplicate-2" }, // Duplicate 2
+      { type: "tool_result", actionExecutionId: "tool-2", result: "second-result" },
+      { type: "tool_result", actionExecutionId: "tool-2", result: "duplicate-3" }, // Duplicate 3
+      { type: "tool_result", actionExecutionId: "tool-1", result: "duplicate-4" }, // Duplicate 4
+    ];
+
+    // Apply OpenAI's filter approach
+    const filteredMessages = messages.filter((message) => {
+      if (message.type === "tool_result") {
+        if (!validToolCallIds.has(message.actionExecutionId)) {
+          return false;
+        }
+        validToolCallIds.delete(message.actionExecutionId);
+      }
+      return true;
+    });
+
+    // Should have text + tool calls + only the first occurrence of each tool result
+    expect(filteredMessages.length).toBe(5);
+
+    // Check that only the first occurrence of each tool result is kept
+    const toolResults = filteredMessages.filter((m) => m.type === "tool_result");
+    expect(toolResults.length).toBe(2);
+
+    expect(toolResults[0].actionExecutionId).toBe("tool-1");
+    expect(toolResults[0].result).toBe("first-result"); // First occurrence should be kept
+
+    expect(toolResults[1].actionExecutionId).toBe("tool-2");
+    expect(toolResults[1].result).toBe("second-result"); // First occurrence should be kept
+  });
+});