@copilotkit/runtime 0.0.0-feat-dynamic-copilotcloud-qa-20250117190454

package/src/service-adapters/anthropic/anthropic-adapter.ts

@@ -0,0 +1,205 @@
+/**
+ * Copilot Runtime adapter for Anthropic.
+ *
+ * ## Example
+ *
+ * ```ts
+ * import { CopilotRuntime, AnthropicAdapter } from "@copilotkit/runtime";
+ * import Anthropic from "@anthropic-ai/sdk";
+ *
+ * const copilotKit = new CopilotRuntime();
+ *
+ * const anthropic = new Anthropic({
+ *   apiKey: "<your-api-key>",
+ * });
+ *
+ * return new AnthropicAdapter({ anthropic });
+ * ```
+ */
+import Anthropic from "@anthropic-ai/sdk";
+import {
+  CopilotServiceAdapter,
+  CopilotRuntimeChatCompletionRequest,
+  CopilotRuntimeChatCompletionResponse,
+} from "../service-adapter";
+import {
+  convertActionInputToAnthropicTool,
+  convertMessageToAnthropicMessage,
+  groupAnthropicMessagesByRole,
+  limitMessagesToTokenCount,
+} from "./utils";
+
+import { randomId } from "@copilotkit/shared";
+import { TextMessage } from "../../graphql/types/converted";
+
+const DEFAULT_MODEL = "claude-3-sonnet-20240229";
+
+export interface AnthropicAdapterParams {
+  /**
+   * An optional Anthropic instance to use.  If not provided, a new instance will be
+   * created.
+   */
+  anthropic?: Anthropic;
+
+  /**
+   * The model to use.
+   */
+  model?: string;
+}
+
+export class AnthropicAdapter implements CopilotServiceAdapter {
+  private model: string = DEFAULT_MODEL;
+
+  private _anthropic: Anthropic;
+  public get anthropic(): Anthropic {
+    return this._anthropic;
+  }
+
+  constructor(params?: AnthropicAdapterParams) {
+    this._anthropic = params?.anthropic || new Anthropic({});
+    if (params?.model) {
+      this.model = params.model;
+    }
+  }
+
+  async process(
+    request: CopilotRuntimeChatCompletionRequest,
+  ): Promise<CopilotRuntimeChatCompletionResponse> {
+    const {
+      threadId,
+      model = this.model,
+      messages: rawMessages,
+      actions,
+      eventSource,
+      forwardedParameters,
+    } = request;
+    const tools = actions.map(convertActionInputToAnthropicTool);
+
+    const messages = [...rawMessages];
+
+    // get the instruction message
+    const instructionsMessage = messages.shift();
+    const instructions = instructionsMessage.isTextMessage() ? instructionsMessage.content : "";
+
+    let anthropicMessages = messages.map(convertMessageToAnthropicMessage);
+    anthropicMessages = limitMessagesToTokenCount(anthropicMessages, tools, model);
+    anthropicMessages = groupAnthropicMessagesByRole(anthropicMessages);
+
+    let toolChoice: any = forwardedParameters?.toolChoice;
+    if (forwardedParameters?.toolChoice === "function") {
+      toolChoice = {
+        type: "tool",
+        name: forwardedParameters.toolChoiceFunctionName,
+      };
+    }
+
+    const stream = this.anthropic.messages.create({
+      system: instructions,
+      model: this.model,
+      messages: anthropicMessages,
+      max_tokens: forwardedParameters?.maxTokens || 1024,
+      ...(forwardedParameters?.temperature ? { temperature: forwardedParameters.temperature } : {}),
+      ...(tools.length > 0 && { tools }),
+      ...(toolChoice && { tool_choice: toolChoice }),
+      stream: true,
+    });
+
+    eventSource.stream(async (eventStream$) => {
+      let mode: "function" | "message" | null = null;
+      let didOutputText = false;
+      let currentMessageId = randomId();
+      let currentToolCallId = randomId();
+      let filterThinkingTextBuffer = new FilterThinkingTextBuffer();
+
+      for await (const chunk of await stream) {
+        if (chunk.type === "message_start") {
+          currentMessageId = chunk.message.id;
+        } else if (chunk.type === "content_block_start") {
+          if (chunk.content_block.type === "text") {
+            didOutputText = false;
+            filterThinkingTextBuffer.reset();
+            mode = "message";
+          } else if (chunk.content_block.type === "tool_use") {
+            currentToolCallId = chunk.content_block.id;
+            eventStream$.sendActionExecutionStart({
+              actionExecutionId: currentToolCallId,
+              actionName: chunk.content_block.name,
+              parentMessageId: currentMessageId,
+            });
+            mode = "function";
+          }
+        } else if (chunk.type === "content_block_delta") {
+          if (chunk.delta.type === "text_delta") {
+            const text = filterThinkingTextBuffer.onTextChunk(chunk.delta.text);
+            if (text.length > 0) {
+              if (!didOutputText) {
+                eventStream$.sendTextMessageStart({ messageId: currentMessageId });
+                didOutputText = true;
+              }
+              eventStream$.sendTextMessageContent({
+                messageId: currentMessageId,
+                content: text,
+              });
+            }
+          } else if (chunk.delta.type === "input_json_delta") {
+            eventStream$.sendActionExecutionArgs({
+              actionExecutionId: currentToolCallId,
+              args: chunk.delta.partial_json,
+            });
+          }
+        } else if (chunk.type === "content_block_stop") {
+          if (mode === "message") {
+            if (didOutputText) {
+              eventStream$.sendTextMessageEnd({ messageId: currentMessageId });
+            }
+          } else if (mode === "function") {
+            eventStream$.sendActionExecutionEnd({ actionExecutionId: currentToolCallId });
+          }
+        }
+      }
+
+      eventStream$.complete();
+    });
+
+    return {
+      threadId: threadId || randomId(),
+    };
+  }
+}
+
+const THINKING_TAG = "<thinking>";
+const THINKING_TAG_END = "</thinking>";
+
+class FilterThinkingTextBuffer {
+  private buffer: string;
+  private didFilterThinkingTag: boolean = false;
+
+  constructor() {
+    this.buffer = "";
+  }
+
+  onTextChunk(text: string): string {
+    this.buffer += text;
+    if (this.didFilterThinkingTag) {
+      return text;
+    }
+    const potentialTag = this.buffer.slice(0, THINKING_TAG.length);
+    if (THINKING_TAG.startsWith(potentialTag)) {
+      if (this.buffer.includes(THINKING_TAG_END)) {
+        const end = this.buffer.indexOf(THINKING_TAG_END);
+        const filteredText = this.buffer.slice(end + THINKING_TAG_END.length);
+        this.buffer = filteredText;
+        this.didFilterThinkingTag = true;
+        return filteredText;
+      } else {
+        return "";
+      }
+    }
+    return text;
+  }
+
+  reset() {
+    this.buffer = "";
+    this.didFilterThinkingTag = false;
+  }
+}

package/src/service-adapters/anthropic/utils.ts

@@ -0,0 +1,144 @@
+import {
+  ActionExecutionMessage,
+  Message,
+  ResultMessage,
+  TextMessage,
+} from "../../graphql/types/converted";
+import { ActionInput } from "../../graphql/inputs/action.input";
+import { Anthropic } from "@anthropic-ai/sdk";
+
+export function limitMessagesToTokenCount(
+  messages: any[],
+  tools: any[],
+  model: string,
+  maxTokens?: number,
+): any[] {
+  maxTokens ||= MAX_TOKENS;
+
+  const result: any[] = [];
+  const toolsNumTokens = countToolsTokens(model, tools);
+  if (toolsNumTokens > maxTokens) {
+    throw new Error(`Too many tokens in function definitions: ${toolsNumTokens} > ${maxTokens}`);
+  }
+  maxTokens -= toolsNumTokens;
+
+  for (const message of messages) {
+    if (message.role === "system") {
+      const numTokens = countMessageTokens(model, message);
+      maxTokens -= numTokens;
+
+      if (maxTokens < 0) {
+        throw new Error("Not enough tokens for system message.");
+      }
+    }
+  }
+
+  let cutoff: boolean = false;
+
+  const reversedMessages = [...messages].reverse();
+  for (const message of reversedMessages) {
+    if (message.role === "system") {
+      result.unshift(message);
+      continue;
+    } else if (cutoff) {
+      continue;
+    }
+    let numTokens = countMessageTokens(model, message);
+    if (maxTokens < numTokens) {
+      cutoff = true;
+      continue;
+    }
+    result.unshift(message);
+    maxTokens -= numTokens;
+  }
+
+  return result;
+}
+
+const MAX_TOKENS = 128000;
+
+function countToolsTokens(model: string, tools: any[]): number {
+  if (tools.length === 0) {
+    return 0;
+  }
+  const json = JSON.stringify(tools);
+  return countTokens(model, json);
+}
+
+function countMessageTokens(model: string, message: any): number {
+  return countTokens(model, JSON.stringify(message.content) || "");
+}
+
+function countTokens(model: string, text: string): number {
+  return text.length / 3;
+}
+
+export function convertActionInputToAnthropicTool(action: ActionInput): Anthropic.Messages.Tool {
+  return {
+    name: action.name,
+    description: action.description,
+    input_schema: JSON.parse(action.jsonSchema),
+  };
+}
+
+export function convertMessageToAnthropicMessage(
+  message: Message,
+): Anthropic.Messages.MessageParam {
+  if (message.isTextMessage()) {
+    if (message.role === "system") {
+      return {
+        role: "assistant",
+        content: [
+          { type: "text", text: "THE FOLLOWING MESSAGE IS A SYSTEM MESSAGE: " + message.content },
+        ],
+      };
+    } else {
+      return {
+        role: message.role === "user" ? "user" : "assistant",
+        content: [{ type: "text", text: message.content }],
+      };
+    }
+  } else if (message.isActionExecutionMessage()) {
+    return {
+      role: "assistant",
+      content: [
+        {
+          id: message.id,
+          type: "tool_use",
+          input: message.arguments,
+          name: message.name,
+        },
+      ],
+    };
+  } else if (message.isResultMessage()) {
+    return {
+      role: "user",
+      content: [
+        {
+          type: "tool_result",
+          content: message.result,
+          tool_use_id: message.actionExecutionId,
+        },
+      ],
+    };
+  }
+}
+
+export function groupAnthropicMessagesByRole(
+  messageParams: Anthropic.Messages.MessageParam[],
+): Anthropic.Messages.MessageParam[] {
+  return messageParams.reduce((acc, message) => {
+    const lastGroup = acc[acc.length - 1];
+
+    if (lastGroup && lastGroup.role === message.role) {
+      lastGroup.content = lastGroup.content.concat(message.content as any);
+    } else {
+      acc.push({
+        role: message.role,
+        content: [...(message.content as any)],
+      });
+    }
+
+    return acc;
+  }, [] as Anthropic.Messages.MessageParam[]);
+}

package/src/service-adapters/conversion.ts

@@ -0,0 +1,64 @@
+import {
+  ActionExecutionMessage,
+  Message,
+  ResultMessage,
+  TextMessage,
+  AgentStateMessage,
+} from "../graphql/types/converted";
+import { MessageInput } from "../graphql/inputs/message.input";
+import { plainToInstance } from "class-transformer";
+
+export function convertGqlInputToMessages(inputMessages: MessageInput[]): Message[] {
+  const messages: Message[] = [];
+
+  for (const message of inputMessages) {
+    if (message.textMessage) {
+      messages.push(
+        plainToInstance(TextMessage, {
+          id: message.id,
+          createdAt: message.createdAt,
+          role: message.textMessage.role,
+          content: message.textMessage.content,
+          parentMessageId: message.textMessage.parentMessageId,
+        }),
+      );
+    } else if (message.actionExecutionMessage) {
+      messages.push(
+        plainToInstance(ActionExecutionMessage, {
+          id: message.id,
+          createdAt: message.createdAt,
+          name: message.actionExecutionMessage.name,
+          arguments: JSON.parse(message.actionExecutionMessage.arguments),
+          parentMessageId: message.actionExecutionMessage.parentMessageId,
+        }),
+      );
+    } else if (message.resultMessage) {
+      messages.push(
+        plainToInstance(ResultMessage, {
+          id: message.id,
+          createdAt: message.createdAt,
+          actionExecutionId: message.resultMessage.actionExecutionId,
+          actionName: message.resultMessage.actionName,
+          result: message.resultMessage.result,
+        }),
+      );
+    } else if (message.agentStateMessage) {
+      messages.push(
+        plainToInstance(AgentStateMessage, {
+          id: message.id,
+          threadId: message.agentStateMessage.threadId,
+          createdAt: message.createdAt,
+          agentName: message.agentStateMessage.agentName,
+          nodeName: message.agentStateMessage.nodeName,
+          runId: message.agentStateMessage.runId,
+          active: message.agentStateMessage.active,
+          role: message.agentStateMessage.role,
+          state: JSON.parse(message.agentStateMessage.state),
+          running: message.agentStateMessage.running,
+        }),
+      );
+    }
+  }
+
+  return messages;
+}