@copilotkit/runtime 0.0.0-feat-dynamic-copilotcloud-qa-20250117190454

package/src/service-adapters/openai/openai-adapter.ts

@@ -0,0 +1,210 @@
+/**
+ * Copilot Runtime adapter for OpenAI.
+ *
+ * ## Example
+ *
+ * ```ts
+ * import { CopilotRuntime, OpenAIAdapter } from "@copilotkit/runtime";
+ * import OpenAI from "openai";
+ *
+ * const copilotKit = new CopilotRuntime();
+ *
+ * const openai = new OpenAI({
+ *   organization: "<your-organization-id>", // optional
+ *   apiKey: "<your-api-key>",
+ * });
+ *
+ * return new OpenAIAdapter({ openai });
+ * ```
+ *
+ * ## Example with Azure OpenAI
+ *
+ * ```ts
+ * import { CopilotRuntime, OpenAIAdapter } from "@copilotkit/runtime";
+ * import OpenAI from "openai";
+ *
+ * // The name of your Azure OpenAI Instance.
+ * // https://learn.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource
+ * const instance = "<your instance name>";
+ *
+ * // Corresponds to your Model deployment within your OpenAI resource, e.g. my-gpt35-16k-deployment
+ * // Navigate to the Azure OpenAI Studio to deploy a model.
+ * const model = "<your model>";
+ *
+ * const apiKey = process.env["AZURE_OPENAI_API_KEY"];
+ * if (!apiKey) {
+ *   throw new Error("The AZURE_OPENAI_API_KEY environment variable is missing or empty.");
+ * }
+ *
+ * const copilotKit = new CopilotRuntime();
+ *
+ * const openai = new OpenAI({
+ *   apiKey,
+ *   baseURL: `https://${instance}.openai.azure.com/openai/deployments/${model}`,
+ *   defaultQuery: { "api-version": "2024-04-01-preview" },
+ *   defaultHeaders: { "api-key": apiKey },
+ * });
+ *
+ * return new OpenAIAdapter({ openai });
+ * ```
+ */
+import OpenAI from "openai";
+import {
+  CopilotServiceAdapter,
+  CopilotRuntimeChatCompletionRequest,
+  CopilotRuntimeChatCompletionResponse,
+} from "../service-adapter";
+import {
+  convertActionInputToOpenAITool,
+  convertMessageToOpenAIMessage,
+  limitMessagesToTokenCount,
+} from "./utils";
+import { randomId } from "@copilotkit/shared";
+
+const DEFAULT_MODEL = "gpt-4o";
+
+export interface OpenAIAdapterParams {
+  /**
+   * An optional OpenAI instance to use.  If not provided, a new instance will be
+   * created.
+   */
+  openai?: OpenAI;
+
+  /**
+   * The model to use.
+   */
+  model?: string;
+
+  /**
+   * Whether to disable parallel tool calls.
+   * You can disable parallel tool calls to force the model to execute tool calls sequentially.
+   * This is useful if you want to execute tool calls in a specific order so that the state changes
+   * introduced by one tool call are visible to the next tool call. (i.e. new actions or readables)
+   *
+   * @default false
+   */
+  disableParallelToolCalls?: boolean;
+}
+
+export class OpenAIAdapter implements CopilotServiceAdapter {
+  private model: string = DEFAULT_MODEL;
+
+  private disableParallelToolCalls: boolean = false;
+  private _openai: OpenAI;
+  public get openai(): OpenAI {
+    return this._openai;
+  }
+
+  constructor(params?: OpenAIAdapterParams) {
+    this._openai = params?.openai || new OpenAI({});
+    if (params?.model) {
+      this.model = params.model;
+    }
+    this.disableParallelToolCalls = params?.disableParallelToolCalls || false;
+  }
+
+  async process(
+    request: CopilotRuntimeChatCompletionRequest,
+  ): Promise<CopilotRuntimeChatCompletionResponse> {
+    const {
+      threadId,
+      model = this.model,
+      messages,
+      actions,
+      eventSource,
+      forwardedParameters,
+    } = request;
+    const tools = actions.map(convertActionInputToOpenAITool);
+
+    let openaiMessages = messages.map(convertMessageToOpenAIMessage);
+    openaiMessages = limitMessagesToTokenCount(openaiMessages, tools, model);
+
+    let toolChoice: any = forwardedParameters?.toolChoice;
+    if (forwardedParameters?.toolChoice === "function") {
+      toolChoice = {
+        type: "function",
+        function: { name: forwardedParameters.toolChoiceFunctionName },
+      };
+    }
+
+    const stream = this.openai.beta.chat.completions.stream({
+      model: model,
+      stream: true,
+      messages: openaiMessages,
+      ...(tools.length > 0 && { tools }),
+      ...(forwardedParameters?.maxTokens && { max_tokens: forwardedParameters.maxTokens }),
+      ...(forwardedParameters?.stop && { stop: forwardedParameters.stop }),
+      ...(toolChoice && { tool_choice: toolChoice }),
+      ...(this.disableParallelToolCalls && { parallel_tool_calls: false }),
+      ...(forwardedParameters?.temperature && { temperature: forwardedParameters.temperature }),
+    });
+
+    eventSource.stream(async (eventStream$) => {
+      let mode: "function" | "message" | null = null;
+      let currentMessageId: string;
+      let currentToolCallId: string;
+      for await (const chunk of stream) {
+        if (chunk.choices.length === 0) {
+          continue;
+        }
+
+        const toolCall = chunk.choices[0].delta.tool_calls?.[0];
+        const content = chunk.choices[0].delta.content;
+
+        // When switching from message to function or vice versa,
+        // send the respective end event.
+        // If toolCall?.id is defined, it means a new tool call starts.
+        if (mode === "message" && toolCall?.id) {
+          mode = null;
+          eventStream$.sendTextMessageEnd({ messageId: currentMessageId });
+        } else if (mode === "function" && (toolCall === undefined || toolCall?.id)) {
+          mode = null;
+          eventStream$.sendActionExecutionEnd({ actionExecutionId: currentToolCallId });
+        }
+
+        // If we send a new message type, send the appropriate start event.
+        if (mode === null) {
+          if (toolCall?.id) {
+            mode = "function";
+            currentToolCallId = toolCall!.id;
+            eventStream$.sendActionExecutionStart({
+              actionExecutionId: currentToolCallId,
+              parentMessageId: chunk.id,
+              actionName: toolCall!.function!.name,
+            });
+          } else if (content) {
+            mode = "message";
+            currentMessageId = chunk.id;
+            eventStream$.sendTextMessageStart({ messageId: currentMessageId });
+          }
+        }
+
+        // send the content events
+        if (mode === "message" && content) {
+          eventStream$.sendTextMessageContent({
+            messageId: currentMessageId,
+            content: content,
+          });
+        } else if (mode === "function" && toolCall?.function?.arguments) {
+          eventStream$.sendActionExecutionArgs({
+            actionExecutionId: currentToolCallId,
+            args: toolCall.function.arguments,
+          });
+        }
+      }
+
+      // send the end events
+      if (mode === "message") {
+        eventStream$.sendTextMessageEnd({ messageId: currentMessageId });
+      } else if (mode === "function") {
+        eventStream$.sendActionExecutionEnd({ actionExecutionId: currentToolCallId });
+      }
+
+      eventStream$.complete();
+    });
+
+    return {
+      threadId: threadId || randomId(),
+    };
+  }
+}

package/src/service-adapters/openai/openai-assistant-adapter.ts

@@ -0,0 +1,304 @@
+/**
+ * Copilot Runtime adapter for the OpenAI Assistant API.
+ *
+ * ## Example
+ *
+ * ```ts
+ * import { CopilotRuntime, OpenAIAssistantAdapter } from "@copilotkit/runtime";
+ * import OpenAI from "openai";
+ *
+ * const copilotKit = new CopilotRuntime();
+ *
+ * const openai = new OpenAI({
+ *   organization: "<your-organization-id>",
+ *   apiKey: "<your-api-key>",
+ * });
+ *
+ * return new OpenAIAssistantAdapter({
+ *   openai,
+ *   assistantId: "<your-assistant-id>",
+ *   codeInterpreterEnabled: true,
+ *   fileSearchEnabled: true,
+ * });
+ * ```
+ */
+import OpenAI from "openai";
+import {
+  CopilotServiceAdapter,
+  CopilotRuntimeChatCompletionRequest,
+  CopilotRuntimeChatCompletionResponse,
+} from "../service-adapter";
+import { Message, ResultMessage, TextMessage } from "../../graphql/types/converted";
+import {
+  convertActionInputToOpenAITool,
+  convertMessageToOpenAIMessage,
+  convertSystemMessageToAssistantAPI,
+} from "./utils";
+import { RunSubmitToolOutputsStreamParams } from "openai/resources/beta/threads/runs/runs";
+import { AssistantStream } from "openai/lib/AssistantStream";
+import { RuntimeEventSource } from "../events";
+import { ActionInput } from "../../graphql/inputs/action.input";
+import { AssistantStreamEvent, AssistantTool } from "openai/resources/beta/assistants";
+import { ForwardedParametersInput } from "../../graphql/inputs/forwarded-parameters.input";
+
+export interface OpenAIAssistantAdapterParams {
+  /**
+   * The ID of the assistant to use.
+   */
+  assistantId: string;
+
+  /**
+   * An optional OpenAI instance to use. If not provided, a new instance will be created.
+   */
+  openai?: OpenAI;
+
+  /**
+   * Whether to enable code interpretation.
+   * @default true
+   */
+  codeInterpreterEnabled?: boolean;
+
+  /**
+   * Whether to enable file search.
+   * @default true
+   */
+  fileSearchEnabled?: boolean;
+
+  /**
+   * Whether to disable parallel tool calls.
+   * You can disable parallel tool calls to force the model to execute tool calls sequentially.
+   * This is useful if you want to execute tool calls in a specific order so that the state changes
+   * introduced by one tool call are visible to the next tool call. (i.e. new actions or readables)
+   *
+   * @default false
+   */
+  disableParallelToolCalls?: boolean;
+}
+
+export class OpenAIAssistantAdapter implements CopilotServiceAdapter {
+  private openai: OpenAI;
+  private codeInterpreterEnabled: boolean;
+  private assistantId: string;
+  private fileSearchEnabled: boolean;
+  private disableParallelToolCalls: boolean;
+
+  constructor(params: OpenAIAssistantAdapterParams) {
+    this.openai = params.openai || new OpenAI({});
+    this.codeInterpreterEnabled = params.codeInterpreterEnabled === false || true;
+    this.fileSearchEnabled = params.fileSearchEnabled === false || true;
+    this.assistantId = params.assistantId;
+    this.disableParallelToolCalls = params?.disableParallelToolCalls || false;
+  }
+
+  async process(
+    request: CopilotRuntimeChatCompletionRequest,
+  ): Promise<CopilotRuntimeChatCompletionResponse> {
+    const { messages, actions, eventSource, runId, forwardedParameters } = request;
+    // if we don't have a threadId, create a new thread
+    let threadId = request.threadId || (await this.openai.beta.threads.create()).id;
+
+    const lastMessage = messages.at(-1);
+
+    let nextRunId: string | undefined = undefined;
+
+    // submit function outputs
+    if (lastMessage.isResultMessage() && runId) {
+      nextRunId = await this.submitToolOutputs(threadId, runId, messages, eventSource);
+    }
+    // submit user message
+    else if (lastMessage.isTextMessage()) {
+      nextRunId = await this.submitUserMessage(
+        threadId,
+        messages,
+        actions,
+        eventSource,
+        forwardedParameters,
+      );
+    }
+    // unsupported message
+    else {
+      throw new Error("No actionable message found in the messages");
+    }
+
+    return {
+      threadId,
+      runId: nextRunId,
+    };
+  }
+
+  private async submitToolOutputs(
+    threadId: string,
+    runId: string,
+    messages: Message[],
+    eventSource: RuntimeEventSource,
+  ) {
+    let run = await this.openai.beta.threads.runs.retrieve(threadId, runId);
+    if (!run.required_action) {
+      throw new Error("No tool outputs required");
+    }
+
+    // get the required tool call ids
+    const toolCallsIds = run.required_action.submit_tool_outputs.tool_calls.map(
+      (toolCall) => toolCall.id,
+    );
+
+    // search for these tool calls
+    const resultMessages = messages.filter(
+      (message) => message.isResultMessage() && toolCallsIds.includes(message.actionExecutionId),
+    ) as ResultMessage[];
+
+    if (toolCallsIds.length != resultMessages.length) {
+      throw new Error("Number of function results does not match the number of tool calls");
+    }
+
+    // submit the tool outputs
+    const toolOutputs: RunSubmitToolOutputsStreamParams.ToolOutput[] = resultMessages.map(
+      (message) => {
+        return {
+          tool_call_id: message.actionExecutionId,
+          output: message.result,
+        };
+      },
+    );
+
+    const stream = this.openai.beta.threads.runs.submitToolOutputsStream(threadId, runId, {
+      tool_outputs: toolOutputs,
+      ...(this.disableParallelToolCalls && { parallel_tool_calls: false }),
+    });
+
+    await this.streamResponse(stream, eventSource);
+    return runId;
+  }
+
+  private async submitUserMessage(
+    threadId: string,
+    messages: Message[],
+    actions: ActionInput[],
+    eventSource: RuntimeEventSource,
+    forwardedParameters: ForwardedParametersInput,
+  ) {
+    messages = [...messages];
+
+    // get the instruction message
+    const instructionsMessage = messages.shift();
+    const instructions = instructionsMessage.isTextMessage() ? instructionsMessage.content : "";
+
+    // get the latest user message
+    const userMessage = messages
+      .map(convertMessageToOpenAIMessage)
+      .map(convertSystemMessageToAssistantAPI)
+      .at(-1);
+
+    if (userMessage.role !== "user") {
+      throw new Error("No user message found");
+    }
+
+    // create a new message on the thread
+    await this.openai.beta.threads.messages.create(threadId, {
+      role: "user",
+      content: userMessage.content,
+    });
+
+    const openaiTools = actions.map(convertActionInputToOpenAITool);
+
+    const tools = [
+      ...openaiTools,
+      ...(this.codeInterpreterEnabled ? [{ type: "code_interpreter" } as AssistantTool] : []),
+      ...(this.fileSearchEnabled ? [{ type: "file_search" } as AssistantTool] : []),
+    ];
+
+    // run the thread
+    let stream = this.openai.beta.threads.runs.stream(threadId, {
+      assistant_id: this.assistantId,
+      instructions,
+      tools: tools,
+      ...(forwardedParameters?.maxTokens && {
+        max_completion_tokens: forwardedParameters.maxTokens,
+      }),
+      ...(this.disableParallelToolCalls && { parallel_tool_calls: false }),
+    });
+
+    await this.streamResponse(stream, eventSource);
+
+    return getRunIdFromStream(stream);
+  }
+
+  private async streamResponse(stream: AssistantStream, eventSource: RuntimeEventSource) {
+    eventSource.stream(async (eventStream$) => {
+      let inFunctionCall = false;
+      let currentMessageId: string;
+      let currentToolCallId: string;
+
+      for await (const chunk of stream) {
+        switch (chunk.event) {
+          case "thread.message.created":
+            if (inFunctionCall) {
+              eventStream$.sendActionExecutionEnd({ actionExecutionId: currentToolCallId });
+            }
+            currentMessageId = chunk.data.id;
+            eventStream$.sendTextMessageStart({ messageId: currentMessageId });
+            break;
+          case "thread.message.delta":
+            if (chunk.data.delta.content?.[0].type === "text") {
+              eventStream$.sendTextMessageContent({
+                messageId: currentMessageId,
+                content: chunk.data.delta.content?.[0].text.value,
+              });
+            }
+            break;
+          case "thread.message.completed":
+            eventStream$.sendTextMessageEnd({ messageId: currentMessageId });
+            break;
+          case "thread.run.step.delta":
+            let toolCallId: string | undefined;
+            let toolCallName: string | undefined;
+            let toolCallArgs: string | undefined;
+            if (
+              chunk.data.delta.step_details.type === "tool_calls" &&
+              chunk.data.delta.step_details.tool_calls?.[0].type === "function"
+            ) {
+              toolCallId = chunk.data.delta.step_details.tool_calls?.[0].id;
+              toolCallName = chunk.data.delta.step_details.tool_calls?.[0].function.name;
+              toolCallArgs = chunk.data.delta.step_details.tool_calls?.[0].function.arguments;
+            }
+
+            if (toolCallName && toolCallId) {
+              if (inFunctionCall) {
+                eventStream$.sendActionExecutionEnd({ actionExecutionId: currentToolCallId });
+              }
+              inFunctionCall = true;
+              currentToolCallId = toolCallId;
+              eventStream$.sendActionExecutionStart({
+                actionExecutionId: currentToolCallId,
+                parentMessageId: chunk.data.id,
+                actionName: toolCallName,
+              });
+            } else if (toolCallArgs) {
+              eventStream$.sendActionExecutionArgs({
+                actionExecutionId: currentToolCallId,
+                args: toolCallArgs,
+              });
+            }
+            break;
+        }
+      }
+      if (inFunctionCall) {
+        eventStream$.sendActionExecutionEnd({ actionExecutionId: currentToolCallId });
+      }
+      eventStream$.complete();
+    });
+  }
+}
+
+function getRunIdFromStream(stream: AssistantStream): Promise<string> {
+  return new Promise<string>((resolve, reject) => {
+    let runIdGetter = (event: AssistantStreamEvent) => {
+      if (event.event === "thread.run.created") {
+        const runId = event.data.id;
+        stream.off("event", runIdGetter);
+        resolve(runId);
+      }
+    };
+    stream.on("event", runIdGetter);
+  });
+}

package/src/service-adapters/openai/utils.ts

@@ -0,0 +1,161 @@
+import { Message } from "../../graphql/types/converted";
+import { ActionInput } from "../../graphql/inputs/action.input";
+import {
+  ChatCompletionMessageParam,
+  ChatCompletionTool,
+  ChatCompletionUserMessageParam,
+  ChatCompletionAssistantMessageParam,
+  ChatCompletionSystemMessageParam,
+} from "openai/resources";
+
+export function limitMessagesToTokenCount(
+  messages: any[],
+  tools: any[],
+  model: string,
+  maxTokens?: number,
+): any[] {
+  maxTokens ||= maxTokensForOpenAIModel(model);
+
+  const result: any[] = [];
+  const toolsNumTokens = countToolsTokens(model, tools);
+  if (toolsNumTokens > maxTokens) {
+    throw new Error(`Too many tokens in function definitions: ${toolsNumTokens} > ${maxTokens}`);
+  }
+  maxTokens -= toolsNumTokens;
+
+  for (const message of messages) {
+    if (message.role === "system") {
+      const numTokens = countMessageTokens(model, message);
+      maxTokens -= numTokens;
+
+      if (maxTokens < 0) {
+        throw new Error("Not enough tokens for system message.");
+      }
+    }
+  }
+
+  let cutoff: boolean = false;
+
+  const reversedMessages = [...messages].reverse();
+  for (const message of reversedMessages) {
+    if (message.role === "system") {
+      result.unshift(message);
+      continue;
+    } else if (cutoff) {
+      continue;
+    }
+    let numTokens = countMessageTokens(model, message);
+    if (maxTokens < numTokens) {
+      cutoff = true;
+      continue;
+    }
+    result.unshift(message);
+    maxTokens -= numTokens;
+  }
+
+  return result;
+}
+
+export function maxTokensForOpenAIModel(model: string): number {
+  return maxTokensByModel[model] || DEFAULT_MAX_TOKENS;
+}
+
+const DEFAULT_MAX_TOKENS = 128000;
+
+const maxTokensByModel: { [key: string]: number } = {
+  // GPT-4
+  "gpt-4o": 128000,
+  "gpt-4o-2024-05-13": 128000,
+  "gpt-4-turbo": 128000,
+  "gpt-4-turbo-2024-04-09": 128000,
+  "gpt-4-0125-preview": 128000,
+  "gpt-4-turbo-preview": 128000,
+  "gpt-4-1106-preview": 128000,
+  "gpt-4-vision-preview": 128000,
+  "gpt-4-1106-vision-preview": 128000,
+  "gpt-4-32k": 32768,
+  "gpt-4-32k-0613": 32768,
+  "gpt-4-32k-0314": 32768,
+  "gpt-4": 8192,
+  "gpt-4-0613": 8192,
+  "gpt-4-0314": 8192,
+
+  // GPT-3.5
+  "gpt-3.5-turbo-0125": 16385,
+  "gpt-3.5-turbo": 16385,
+  "gpt-3.5-turbo-1106": 16385,
+  "gpt-3.5-turbo-instruct": 4096,
+  "gpt-3.5-turbo-16k": 16385,
+  "gpt-3.5-turbo-0613": 4096,
+  "gpt-3.5-turbo-16k-0613": 16385,
+  "gpt-3.5-turbo-0301": 4097,
+};
+
+function countToolsTokens(model: string, tools: any[]): number {
+  if (tools.length === 0) {
+    return 0;
+  }
+  const json = JSON.stringify(tools);
+  return countTokens(model, json);
+}
+
+function countMessageTokens(model: string, message: any): number {
+  return countTokens(model, message.content || "");
+}
+
+function countTokens(model: string, text: string): number {
+  return text.length / 3;
+}
+
+export function convertActionInputToOpenAITool(action: ActionInput): ChatCompletionTool {
+  return {
+    type: "function",
+    function: {
+      name: action.name,
+      description: action.description,
+      parameters: JSON.parse(action.jsonSchema),
+    },
+  };
+}
+
+export function convertMessageToOpenAIMessage(message: Message): ChatCompletionMessageParam {
+  if (message.isTextMessage()) {
+    return {
+      role: message.role as ChatCompletionUserMessageParam["role"],
+      content: message.content,
+    } satisfies
+      | ChatCompletionUserMessageParam
+      | ChatCompletionAssistantMessageParam
+      | ChatCompletionSystemMessageParam;
+  } else if (message.isActionExecutionMessage()) {
+    return {
+      role: "assistant",
+      tool_calls: [
+        {
+          id: message.id,
+          type: "function",
+          function: {
+            name: message.name,
+            arguments: JSON.stringify(message.arguments),
+          },
+        },
+      ],
+    };
+  } else if (message.isResultMessage()) {
+    return {
+      role: "tool",
+      content: message.result,
+      tool_call_id: message.actionExecutionId,
+    };
+  }
+}
+
+export function convertSystemMessageToAssistantAPI(message: ChatCompletionMessageParam) {
+  return {
+    ...message,
+    ...(message.role === "system" && {
+      role: "assistant",
+      content: "THE FOLLOWING MESSAGE IS A SYSTEM MESSAGE: " + message.content,
+    }),
+  };
+}

package/src/service-adapters/service-adapter.ts

@@ -0,0 +1,30 @@
+import { Message } from "../graphql/types/converted";
+import { RuntimeEventSource } from "./events";
+import { ActionInput } from "../graphql/inputs/action.input";
+import { ForwardedParametersInput } from "../graphql/inputs/forwarded-parameters.input";
+
+export interface CopilotKitResponse {
+  stream: ReadableStream;
+  headers?: Record<string, string>;
+}
+
+export interface CopilotRuntimeChatCompletionRequest {
+  eventSource: RuntimeEventSource;
+  messages: Message[];
+  actions: ActionInput[];
+  model?: string;
+  threadId?: string;
+  runId?: string;
+  forwardedParameters?: ForwardedParametersInput;
+}
+
+export interface CopilotRuntimeChatCompletionResponse {
+  threadId: string;
+  runId?: string;
+}
+
+export interface CopilotServiceAdapter {
+  process(
+    request: CopilotRuntimeChatCompletionRequest,
+  ): Promise<CopilotRuntimeChatCompletionResponse>;
+}