@copilotkit/runtime 0.0.0-0.0.0-max-changeset-10101010101010-20260109191632

package/src/service-adapters/langchain/utils.ts

@@ -0,0 +1,313 @@
+import {
+  ActionExecutionMessage,
+  Message,
+  ResultMessage,
+  TextMessage,
+} from "../../graphql/types/converted";
+import {
+  AIMessage,
+  AIMessageChunk,
+  BaseMessage,
+  BaseMessageChunk,
+  HumanMessage,
+  SystemMessage,
+  ToolMessage,
+} from "@langchain/core/messages";
+import { DynamicStructuredTool } from "@langchain/core/tools";
+import { z } from "zod";
+import { ActionInput } from "../../graphql/inputs/action.input";
+import { LangChainReturnType } from "./types";
+import { RuntimeEventSubject } from "../events";
+import { randomId, convertJsonSchemaToZodSchema } from "@copilotkit/shared";
+
+export function convertMessageToLangChainMessage(message: Message): BaseMessage {
+  if (message.isTextMessage()) {
+    if (message.role == "user") {
+      return new HumanMessage(message.content);
+    } else if (message.role == "assistant") {
+      return new AIMessage(message.content);
+    } else if (message.role === "system") {
+      return new SystemMessage(message.content);
+    }
+  } else if (message.isActionExecutionMessage()) {
+    return new AIMessage({
+      content: "",
+      tool_calls: [
+        {
+          id: message.id,
+          args: message.arguments,
+          name: message.name,
+        },
+      ],
+    });
+  } else if (message.isResultMessage()) {
+    return new ToolMessage({
+      content: message.result,
+      tool_call_id: message.actionExecutionId,
+    });
+  }
+}
+
+export function convertActionInputToLangChainTool(actionInput: ActionInput): any {
+  return new DynamicStructuredTool({
+    ...actionInput,
+    name: actionInput.name,
+    description: actionInput.description,
+    schema: convertJsonSchemaToZodSchema(
+      JSON.parse(actionInput.jsonSchema),
+      true,
+    ) as z.ZodObject<any>,
+    func: async () => {
+      return "";
+    },
+  });
+}
+
+interface StreamLangChainResponseParams {
+  result: LangChainReturnType;
+  eventStream$: RuntimeEventSubject;
+  actionExecution?: {
+    id: string;
+    name: string;
+    returnDirect?: boolean;
+  };
+}
+
+function getConstructorName(object: any): string {
+  if (object && typeof object === "object" && object.constructor && object.constructor.name) {
+    return object.constructor.name;
+  }
+  return "";
+}
+
+function isAIMessage(message: any): message is AIMessage {
+  return Object.prototype.toString.call(message) === "[object AIMessage]";
+}
+
+function isAIMessageChunk(message: any): message is AIMessageChunk {
+  return Object.prototype.toString.call(message) === "[object AIMessageChunk]";
+}
+
+function isBaseMessageChunk(message: any): message is BaseMessageChunk {
+  return Object.prototype.toString.call(message) === "[object BaseMessageChunk]";
+}
+
+function maybeSendActionExecutionResultIsMessage(
+  eventStream$: RuntimeEventSubject,
+  actionExecution?: { id: string; name: string },
+) {
+  // language models need a result after the function call
+  // we simply let them know that we are sending a message
+  if (actionExecution) {
+    eventStream$.sendActionExecutionResult({
+      actionExecutionId: actionExecution.id,
+      actionName: actionExecution.name,
+      result: "Sending a message",
+    });
+  }
+}
+
+export async function streamLangChainResponse({
+  result,
+  eventStream$,
+  actionExecution,
+}: StreamLangChainResponseParams) {
+  // We support several types of return values from LangChain functions:
+
+  // 1. string
+
+  if (typeof result === "string") {
+    if (!actionExecution || actionExecution?.returnDirect) {
+      // Just send one chunk with the string as the content.
+      eventStream$.sendActionExecutionResult({
+        actionExecutionId: actionExecution.id,
+        actionName: actionExecution.name,
+        result: result,
+      });
+      eventStream$.sendTextMessage(randomId(), result);
+    } else {
+      // Send as a result
+      eventStream$.sendActionExecutionResult({
+        actionExecutionId: actionExecution.id,
+        actionName: actionExecution.name,
+        result: result,
+      });
+    }
+  }
+
+  // 2. AIMessage
+  // Send the content and function call of the AIMessage as the content of the chunk.
+  else if (isAIMessage(result)) {
+    maybeSendActionExecutionResultIsMessage(eventStream$, actionExecution);
+
+    if (result.content) {
+      eventStream$.sendTextMessage(randomId(), result.content as string);
+    }
+    for (const toolCall of result.tool_calls) {
+      eventStream$.sendActionExecution({
+        actionExecutionId: toolCall.id || randomId(),
+        actionName: toolCall.name,
+        args: JSON.stringify(toolCall.args),
+      });
+    }
+  }
+
+  // 3. BaseMessageChunk
+  // Send the content and function call of the AIMessage as the content of the chunk.
+  else if (isBaseMessageChunk(result)) {
+    maybeSendActionExecutionResultIsMessage(eventStream$, actionExecution);
+
+    if (result.lc_kwargs?.content) {
+      eventStream$.sendTextMessage(randomId(), result.content as string);
+    }
+    if (result.lc_kwargs?.tool_calls) {
+      for (const toolCall of result.lc_kwargs?.tool_calls) {
+        eventStream$.sendActionExecution({
+          actionExecutionId: toolCall.id || randomId(),
+          actionName: toolCall.name,
+          args: JSON.stringify(toolCall.args),
+        });
+      }
+    }
+  }
+
+  // 4. IterableReadableStream
+  // Stream the result of the LangChain function.
+  else if (result && "getReader" in result) {
+    maybeSendActionExecutionResultIsMessage(eventStream$, actionExecution);
+
+    let reader = result.getReader();
+
+    let mode: "function" | "message" | null = null;
+    let currentMessageId: string;
+
+    const toolCallDetails = {
+      name: null,
+      id: null,
+      index: null,
+      prevIndex: null,
+    };
+
+    while (true) {
+      try {
+        const { done, value } = await reader.read();
+
+        let toolCallName: string | undefined = undefined;
+        let toolCallId: string | undefined = undefined;
+        let toolCallArgs: string | undefined = undefined;
+        let hasToolCall: boolean = false;
+        let content = "";
+        if (value && value.content) {
+          content = Array.isArray(value.content)
+            ? (((value.content[0] as any)?.text ?? "") as string)
+            : value.content;
+        }
+
+        if (isAIMessageChunk(value)) {
+          let chunk = value.tool_call_chunks?.[0];
+          toolCallArgs = chunk?.args;
+          hasToolCall = chunk != undefined;
+          if (chunk?.name) toolCallDetails.name = chunk.name;
+          // track different index on the same tool cool
+          if (chunk?.index != null) {
+            toolCallDetails.index = chunk.index; // 1
+            if (toolCallDetails.prevIndex == null) toolCallDetails.prevIndex = chunk.index;
+          }
+          // Differentiate when calling the same tool but with different index
+          if (chunk?.id)
+            toolCallDetails.id = chunk.index != null ? `${chunk.id}-idx-${chunk.index}` : chunk.id;
+
+          // Assign to internal variables that the entire script here knows how to work with
+          toolCallName = toolCallDetails.name;
+          toolCallId = toolCallDetails.id;
+        } else if (isBaseMessageChunk(value)) {
+          let chunk = value.additional_kwargs?.tool_calls?.[0];
+          toolCallName = chunk?.function?.name;
+          toolCallId = chunk?.id;
+          toolCallArgs = chunk?.function?.arguments;
+          hasToolCall = chunk?.function != undefined;
+        }
+
+        // When switching from message to function or vice versa,
+        // send the respective end event.
+        // If toolCallName is defined, it means a new tool call starts.
+        if (mode === "message" && (toolCallId || done)) {
+          mode = null;
+          eventStream$.sendTextMessageEnd({ messageId: currentMessageId });
+        } else if (mode === "function" && (!hasToolCall || done)) {
+          mode = null;
+          eventStream$.sendActionExecutionEnd({ actionExecutionId: toolCallId });
+        }
+
+        if (done) {
+          break;
+        }
+
+        // If we send a new message type, send the appropriate start event.
+        if (mode === null) {
+          if (hasToolCall && toolCallId && toolCallName) {
+            mode = "function";
+            eventStream$.sendActionExecutionStart({
+              actionExecutionId: toolCallId,
+              actionName: toolCallName,
+              parentMessageId: value.lc_kwargs?.id,
+            });
+          } else if (content) {
+            mode = "message";
+            currentMessageId = value.lc_kwargs?.id || randomId();
+            eventStream$.sendTextMessageStart({ messageId: currentMessageId });
+          }
+        }
+
+        // send the content events
+        if (mode === "message" && content) {
+          eventStream$.sendTextMessageContent({
+            messageId: currentMessageId,
+            content,
+          });
+        } else if (mode === "function" && toolCallArgs) {
+          // For calls of the same tool with different index, we seal last tool call and register a new one
+          if (toolCallDetails.index !== toolCallDetails.prevIndex) {
+            eventStream$.sendActionExecutionEnd({ actionExecutionId: toolCallId });
+            eventStream$.sendActionExecutionStart({
+              actionExecutionId: toolCallId,
+              actionName: toolCallName,
+              parentMessageId: value.lc_kwargs?.id,
+            });
+            toolCallDetails.prevIndex = toolCallDetails.index;
+          }
+          eventStream$.sendActionExecutionArgs({
+            actionExecutionId: toolCallId,
+            args: toolCallArgs,
+          });
+        }
+      } catch (error) {
+        console.error("Error reading from stream", error);
+        break;
+      }
+    }
+  } else if (actionExecution) {
+    eventStream$.sendActionExecutionResult({
+      actionExecutionId: actionExecution.id,
+      actionName: actionExecution.name,
+      result: encodeResult(result),
+    });
+  }
+
+  // unsupported type
+  else {
+    throw new Error("Invalid return type from LangChain function.");
+  }
+
+  eventStream$.complete();
+}
+
+function encodeResult(result: any): string {
+  if (result === undefined) {
+    return "";
+  } else if (typeof result === "string") {
+    return result;
+  } else {
+    return JSON.stringify(result);
+  }
+}

package/src/service-adapters/openai/openai-adapter.ts

@@ -0,0 +1,283 @@
+/**
+ * Copilot Runtime adapter for OpenAI.
+ *
+ * ## Example
+ *
+ * ```ts
+ * import { CopilotRuntime, OpenAIAdapter } from "@copilotkit/runtime";
+ * import OpenAI from "openai";
+ *
+ * const copilotKit = new CopilotRuntime();
+ *
+ * const openai = new OpenAI({
+ *   organization: "<your-organization-id>", // optional
+ *   apiKey: "<your-api-key>",
+ * });
+ *
+ * return new OpenAIAdapter({ openai });
+ * ```
+ *
+ * ## Example with Azure OpenAI
+ *
+ * ```ts
+ * import { CopilotRuntime, OpenAIAdapter } from "@copilotkit/runtime";
+ * import OpenAI from "openai";
+ *
+ * // The name of your Azure OpenAI Instance.
+ * // https://learn.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource
+ * const instance = "<your instance name>";
+ *
+ * // Corresponds to your Model deployment within your OpenAI resource, e.g. my-gpt35-16k-deployment
+ * // Navigate to the Azure OpenAI Studio to deploy a model.
+ * const model = "<your model>";
+ *
+ * const apiKey = process.env["AZURE_OPENAI_API_KEY"];
+ * if (!apiKey) {
+ *   throw new Error("The AZURE_OPENAI_API_KEY environment variable is missing or empty.");
+ * }
+ *
+ * const copilotKit = new CopilotRuntime();
+ *
+ * const openai = new OpenAI({
+ *   apiKey,
+ *   baseURL: `https://${instance}.openai.azure.com/openai/deployments/${model}`,
+ *   defaultQuery: { "api-version": "2024-04-01-preview" },
+ *   defaultHeaders: { "api-key": apiKey },
+ * });
+ *
+ * return new OpenAIAdapter({ openai });
+ * ```
+ */
+import type OpenAI from "openai";
+import {
+  CopilotServiceAdapter,
+  CopilotRuntimeChatCompletionRequest,
+  CopilotRuntimeChatCompletionResponse,
+} from "../service-adapter";
+import {
+  convertActionInputToOpenAITool,
+  convertMessageToOpenAIMessage,
+  limitMessagesToTokenCount,
+} from "./utils";
+import { randomUUID } from "@copilotkit/shared";
+import { convertServiceAdapterError } from "../shared";
+
+const DEFAULT_MODEL = "gpt-4o";
+
+export interface OpenAIAdapterParams {
+  /**
+   * An optional OpenAI instance to use.  If not provided, a new instance will be
+   * created.
+   */
+  openai?: OpenAI;
+
+  /**
+   * The model to use.
+   */
+  model?: string;
+
+  /**
+   * Whether to disable parallel tool calls.
+   * You can disable parallel tool calls to force the model to execute tool calls sequentially.
+   * This is useful if you want to execute tool calls in a specific order so that the state changes
+   * introduced by one tool call are visible to the next tool call. (i.e. new actions or readables)
+   *
+   * @default false
+   */
+  disableParallelToolCalls?: boolean;
+
+  /**
+   * Whether to keep the role in system messages as "System".
+   * By default, it is converted to "developer", which is used by newer OpenAI models
+   *
+   * @default false
+   */
+  keepSystemRole?: boolean;
+}
+
+export class OpenAIAdapter implements CopilotServiceAdapter {
+  public model: string = DEFAULT_MODEL;
+  public provider = "openai";
+
+  private disableParallelToolCalls: boolean = false;
+  private _openai: OpenAI;
+  private keepSystemRole: boolean = false;
+
+  public get openai(): OpenAI {
+    return this._openai;
+  }
+  public get name() {
+    return "OpenAIAdapter";
+  }
+
+  constructor(params?: OpenAIAdapterParams) {
+    if (params?.openai) {
+      this._openai = params.openai;
+    }
+    // If no instance provided, we'll lazy-load in ensureOpenAI()
+
+    if (params?.model) {
+      this.model = params.model;
+    }
+    this.disableParallelToolCalls = params?.disableParallelToolCalls || false;
+    this.keepSystemRole = params?.keepSystemRole ?? false;
+  }
+
+  private ensureOpenAI(): OpenAI {
+    if (!this._openai) {
+      // eslint-disable-next-line @typescript-eslint/no-var-requires
+      const OpenAI = require("openai").default;
+      this._openai = new OpenAI();
+    }
+    return this._openai;
+  }
+
+  async process(
+    request: CopilotRuntimeChatCompletionRequest,
+  ): Promise<CopilotRuntimeChatCompletionResponse> {
+    const {
+      threadId: threadIdFromRequest,
+      model = this.model,
+      messages,
+      actions,
+      eventSource,
+      forwardedParameters,
+    } = request;
+    const tools = actions.map(convertActionInputToOpenAITool);
+    const threadId = threadIdFromRequest ?? randomUUID();
+
+    // ALLOWLIST APPROACH: Only include tool_result messages that correspond to valid tool_calls
+    // Step 1: Extract valid tool_call IDs
+    const validToolUseIds = new Set<string>();
+
+    for (const message of messages) {
+      if (message.isActionExecutionMessage()) {
+        validToolUseIds.add(message.id);
+      }
+    }
+
+    // Step 2: Filter messages, keeping only those with valid tool_call IDs
+    const filteredMessages = messages.filter((message) => {
+      if (message.isResultMessage()) {
+        // Skip if there's no corresponding tool_call
+        if (!validToolUseIds.has(message.actionExecutionId)) {
+          return false;
+        }
+
+        // Remove this ID from valid IDs so we don't process duplicates
+        validToolUseIds.delete(message.actionExecutionId);
+        return true;
+      }
+
+      // Keep all non-tool-result messages
+      return true;
+    });
+
+    let openaiMessages = filteredMessages.map((m) =>
+      convertMessageToOpenAIMessage(m, { keepSystemRole: this.keepSystemRole }),
+    );
+    openaiMessages = limitMessagesToTokenCount(openaiMessages, tools, model);
+
+    let toolChoice: any = forwardedParameters?.toolChoice;
+    if (forwardedParameters?.toolChoice === "function") {
+      toolChoice = {
+        type: "function",
+        function: { name: forwardedParameters.toolChoiceFunctionName },
+      };
+    }
+
+    try {
+      const openai = this.ensureOpenAI();
+      const stream = openai.beta.chat.completions.stream({
+        model: model,
+        stream: true,
+        messages: openaiMessages,
+        ...(tools.length > 0 && { tools }),
+        ...(forwardedParameters?.maxTokens && {
+          max_completion_tokens: forwardedParameters.maxTokens,
+        }),
+        ...(forwardedParameters?.stop && { stop: forwardedParameters.stop }),
+        ...(toolChoice && { tool_choice: toolChoice }),
+        ...(this.disableParallelToolCalls && { parallel_tool_calls: false }),
+        ...(forwardedParameters?.temperature && { temperature: forwardedParameters.temperature }),
+      });
+
+      eventSource.stream(async (eventStream$) => {
+        let mode: "function" | "message" | null = null;
+        let currentMessageId: string;
+        let currentToolCallId: string;
+
+        try {
+          for await (const chunk of stream) {
+            if (chunk.choices.length === 0) {
+              continue;
+            }
+
+            const toolCall = chunk.choices[0].delta.tool_calls?.[0];
+            const content = chunk.choices[0].delta.content;
+
+            // When switching from message to function or vice versa,
+            // send the respective end event.
+            // If toolCall?.id is defined, it means a new tool call starts.
+            if (mode === "message" && toolCall?.id) {
+              mode = null;
+              eventStream$.sendTextMessageEnd({ messageId: currentMessageId });
+            } else if (mode === "function" && (toolCall === undefined || toolCall?.id)) {
+              mode = null;
+              eventStream$.sendActionExecutionEnd({ actionExecutionId: currentToolCallId });
+            }
+
+            // If we send a new message type, send the appropriate start event.
+            if (mode === null) {
+              if (toolCall?.id) {
+                mode = "function";
+                currentToolCallId = toolCall!.id;
+                eventStream$.sendActionExecutionStart({
+                  actionExecutionId: currentToolCallId,
+                  parentMessageId: chunk.id,
+                  actionName: toolCall!.function!.name,
+                });
+              } else if (content) {
+                mode = "message";
+                currentMessageId = chunk.id;
+                eventStream$.sendTextMessageStart({ messageId: currentMessageId });
+              }
+            }
+
+            // send the content events
+            if (mode === "message" && content) {
+              eventStream$.sendTextMessageContent({
+                messageId: currentMessageId,
+                content: content,
+              });
+            } else if (mode === "function" && toolCall?.function?.arguments) {
+              eventStream$.sendActionExecutionArgs({
+                actionExecutionId: currentToolCallId,
+                args: toolCall.function.arguments,
+              });
+            }
+          }
+
+          // send the end events
+          if (mode === "message") {
+            eventStream$.sendTextMessageEnd({ messageId: currentMessageId });
+          } else if (mode === "function") {
+            eventStream$.sendActionExecutionEnd({ actionExecutionId: currentToolCallId });
+          }
+        } catch (error) {
+          console.error("[OpenAI] Error during API call:", error);
+          throw convertServiceAdapterError(error, "OpenAI");
+        }
+
+        eventStream$.complete();
+      });
+    } catch (error) {
+      console.error("[OpenAI] Error during API call:", error);
+      throw convertServiceAdapterError(error, "OpenAI");
+    }
+
+    return {
+      threadId,
+    };
+  }
+}