@copilotkit/runtime 0.0.0-0.0.0-max-changeset-10101010101010-20260109191632

package/src/service-adapters/groq/groq-adapter.ts

@@ -0,0 +1,203 @@
+/**
+ * Copilot Runtime adapter for Groq.
+ *
+ * ## Example
+ *
+ * ```ts
+ * import { CopilotRuntime, GroqAdapter } from "@copilotkit/runtime";
+ * import { Groq } from "groq-sdk";
+ *
+ * const groq = new Groq({ apiKey: process.env["GROQ_API_KEY"] });
+ *
+ * const copilotKit = new CopilotRuntime();
+ *
+ * return new GroqAdapter({ groq, model: "<model-name>" });
+ * ```
+ */
+import type { Groq } from "groq-sdk";
+import type { ChatCompletionMessageParam } from "groq-sdk/resources/chat";
+import {
+  CopilotServiceAdapter,
+  CopilotRuntimeChatCompletionRequest,
+  CopilotRuntimeChatCompletionResponse,
+} from "../service-adapter";
+import {
+  convertActionInputToOpenAITool,
+  convertMessageToOpenAIMessage,
+  limitMessagesToTokenCount,
+} from "../openai/utils";
+import { randomUUID } from "@copilotkit/shared";
+import { convertServiceAdapterError } from "../shared";
+
+const DEFAULT_MODEL = "llama-3.3-70b-versatile";
+
+export interface GroqAdapterParams {
+  /**
+   * An optional Groq instance to use.
+   */
+  groq?: Groq;
+
+  /**
+   * The model to use.
+   */
+  model?: string;
+
+  /**
+   * Whether to disable parallel tool calls.
+   * You can disable parallel tool calls to force the model to execute tool calls sequentially.
+   * This is useful if you want to execute tool calls in a specific order so that the state changes
+   * introduced by one tool call are visible to the next tool call. (i.e. new actions or readables)
+   *
+   * @default false
+   */
+  disableParallelToolCalls?: boolean;
+}
+
+export class GroqAdapter implements CopilotServiceAdapter {
+  public model: string = DEFAULT_MODEL;
+  public provider = "groq";
+
+  private disableParallelToolCalls: boolean = false;
+  private _groq: Groq;
+  public get groq(): Groq {
+    return this._groq;
+  }
+  public get name() {
+    return "GroqAdapter";
+  }
+
+  constructor(params?: GroqAdapterParams) {
+    if (params?.groq) {
+      this._groq = params.groq;
+    }
+    // If no instance provided, we'll lazy-load in ensureGroq()
+    if (params?.model) {
+      this.model = params.model;
+    }
+    this.disableParallelToolCalls = params?.disableParallelToolCalls || false;
+  }
+
+  private ensureGroq(): Groq {
+    if (!this._groq) {
+      // eslint-disable-next-line @typescript-eslint/no-var-requires
+      const { Groq } = require("groq-sdk");
+      this._groq = new Groq({});
+    }
+    return this._groq;
+  }
+
+  async process(
+    request: CopilotRuntimeChatCompletionRequest,
+  ): Promise<CopilotRuntimeChatCompletionResponse> {
+    const {
+      threadId,
+      model = this.model,
+      messages,
+      actions,
+      eventSource,
+      forwardedParameters,
+    } = request;
+    const tools = actions.map(convertActionInputToOpenAITool);
+
+    let openaiMessages = messages.map((m) =>
+      convertMessageToOpenAIMessage(m, { keepSystemRole: true }),
+    );
+    openaiMessages = limitMessagesToTokenCount(openaiMessages, tools, model);
+
+    let toolChoice: any = forwardedParameters?.toolChoice;
+    if (forwardedParameters?.toolChoice === "function") {
+      toolChoice = {
+        type: "function",
+        function: { name: forwardedParameters.toolChoiceFunctionName },
+      };
+    }
+    let stream;
+    try {
+      const groq = this.ensureGroq();
+      stream = await groq.chat.completions.create({
+        model: model,
+        stream: true,
+        messages: openaiMessages as unknown as ChatCompletionMessageParam[],
+        ...(tools.length > 0 && { tools }),
+        ...(forwardedParameters?.maxTokens && {
+          max_tokens: forwardedParameters.maxTokens,
+        }),
+        ...(forwardedParameters?.stop && { stop: forwardedParameters.stop }),
+        ...(toolChoice && { tool_choice: toolChoice }),
+        ...(this.disableParallelToolCalls && { parallel_tool_calls: false }),
+        ...(forwardedParameters?.temperature && { temperature: forwardedParameters.temperature }),
+      });
+    } catch (error) {
+      throw convertServiceAdapterError(error, "Groq");
+    }
+
+    eventSource.stream(async (eventStream$) => {
+      let mode: "function" | "message" | null = null;
+      let currentMessageId: string;
+      let currentToolCallId: string;
+
+      try {
+        for await (const chunk of stream) {
+          const toolCall = chunk.choices[0].delta.tool_calls?.[0];
+          const content = chunk.choices[0].delta.content;
+
+          // When switching from message to function or vice versa,
+          // send the respective end event.
+          // If toolCall?.id is defined, it means a new tool call starts.
+          if (mode === "message" && toolCall?.id) {
+            mode = null;
+            eventStream$.sendTextMessageEnd({ messageId: currentMessageId });
+          } else if (mode === "function" && (toolCall === undefined || toolCall?.id)) {
+            mode = null;
+            eventStream$.sendActionExecutionEnd({ actionExecutionId: currentToolCallId });
+          }
+
+          // If we send a new message type, send the appropriate start event.
+          if (mode === null) {
+            if (toolCall?.id) {
+              mode = "function";
+              currentToolCallId = toolCall!.id;
+              eventStream$.sendActionExecutionStart({
+                actionExecutionId: currentToolCallId,
+                actionName: toolCall!.function!.name,
+                parentMessageId: chunk.id,
+              });
+            } else if (content) {
+              mode = "message";
+              currentMessageId = chunk.id;
+              eventStream$.sendTextMessageStart({ messageId: currentMessageId });
+            }
+          }
+
+          // send the content events
+          if (mode === "message" && content) {
+            eventStream$.sendTextMessageContent({
+              messageId: currentMessageId,
+              content,
+            });
+          } else if (mode === "function" && toolCall?.function?.arguments) {
+            eventStream$.sendActionExecutionArgs({
+              actionExecutionId: currentToolCallId,
+              args: toolCall.function.arguments,
+            });
+          }
+        }
+
+        // send the end events
+        if (mode === "message") {
+          eventStream$.sendTextMessageEnd({ messageId: currentMessageId });
+        } else if (mode === "function") {
+          eventStream$.sendActionExecutionEnd({ actionExecutionId: currentToolCallId });
+        }
+      } catch (error) {
+        throw convertServiceAdapterError(error, "Groq");
+      }
+
+      eventStream$.complete();
+    });
+
+    return {
+      threadId: request.threadId || randomUUID(),
+    };
+  }
+}

package/src/service-adapters/index.ts

@@ -0,0 +1,18 @@
+export type {
+  CopilotRuntimeChatCompletionRequest,
+  CopilotRuntimeChatCompletionResponse,
+  CopilotServiceAdapter,
+} from "./service-adapter";
+export type { RemoteChainParameters } from "./langchain/langserve";
+export { RemoteChain } from "./langchain/langserve";
+export * from "./shared";
+export * from "./openai/openai-adapter";
+export * from "./langchain/langchain-adapter";
+export * from "./google/google-genai-adapter";
+export * from "./openai/openai-assistant-adapter";
+export * from "./unify/unify-adapter";
+export * from "./groq/groq-adapter";
+export * from "./anthropic/anthropic-adapter";
+export * from "./experimental/ollama/ollama-adapter";
+export * from "./bedrock/bedrock-adapter";
+export * from "./empty/empty-adapter";

package/src/service-adapters/langchain/langchain-adapter.ts

@@ -0,0 +1,111 @@
+/**
+ * Copilot Runtime adapter for LangChain.
+ *
+ * ## Example
+ *
+ * ```ts
+ * import { CopilotRuntime, LangChainAdapter } from "@copilotkit/runtime";
+ * import { ChatOpenAI } from "@langchain/openai";
+ *
+ * const copilotKit = new CopilotRuntime();
+ *
+ * const model = new ChatOpenAI({
+ *   model: "gpt-4o",
+ *   apiKey: "<your-api-key>",
+ * });
+ *
+ * return new LangChainAdapter({
+ *   chainFn: async ({ messages, tools }) => {
+ *     return model.bindTools(tools).stream(messages);
+ *     // or optionally enable strict mode
+ *     // return model.bindTools(tools, { strict: true }).stream(messages);
+ *   }
+ * });
+ * ```
+ *
+ * The asynchronous handler function (`chainFn`) can return any of the following:
+ *
+ * - A simple `string` response
+ * - A LangChain stream (`IterableReadableStream`)
+ * - A LangChain `BaseMessageChunk` object
+ * - A LangChain `AIMessage` object
+ */
+
+import type { BaseMessage } from "@langchain/core/messages";
+import { CopilotServiceAdapter } from "../service-adapter";
+import {
+  CopilotRuntimeChatCompletionRequest,
+  CopilotRuntimeChatCompletionResponse,
+} from "../service-adapter";
+import {
+  convertActionInputToLangChainTool,
+  convertMessageToLangChainMessage,
+  streamLangChainResponse,
+} from "./utils";
+import type { DynamicStructuredTool } from "@langchain/core/tools";
+import { LangChainReturnType } from "./types";
+import { randomUUID } from "@copilotkit/shared";
+
+interface ChainFnParameters {
+  model: string;
+  messages: BaseMessage[];
+  tools: DynamicStructuredTool[];
+  threadId?: string;
+  runId?: string;
+}
+
+interface LangChainAdapterOptions {
+  /**
+   * A function that uses the LangChain API to generate a response.
+   */
+  chainFn: (parameters: ChainFnParameters) => Promise<LangChainReturnType>;
+}
+
+export class LangChainAdapter implements CopilotServiceAdapter {
+  /**
+   * To use LangChain as a backend, provide a handler function to the adapter with your custom LangChain logic.
+   */
+  public get name() {
+    return "LangChainAdapter";
+  }
+  constructor(private options: LangChainAdapterOptions) {}
+
+  async process(
+    request: CopilotRuntimeChatCompletionRequest,
+  ): Promise<CopilotRuntimeChatCompletionResponse> {
+    try {
+      const {
+        eventSource,
+        model,
+        actions,
+        messages,
+        runId,
+        threadId: threadIdFromRequest,
+      } = request;
+      const threadId = threadIdFromRequest ?? randomUUID();
+      const result = await this.options.chainFn({
+        messages: messages.map(convertMessageToLangChainMessage),
+        tools: actions.map(convertActionInputToLangChainTool),
+        model,
+        threadId,
+        runId,
+      });
+
+      eventSource.stream(async (eventStream$) => {
+        await streamLangChainResponse({
+          result,
+          eventStream$,
+        });
+      });
+
+      return {
+        threadId,
+      };
+    } finally {
+      // Lazy require for optional peer dependency
+      // eslint-disable-next-line @typescript-eslint/no-var-requires
+      const { awaitAllCallbacks } = require("@langchain/core/callbacks/promises");
+      await awaitAllCallbacks();
+    }
+  }
+}

package/src/service-adapters/langchain/langserve.ts

@@ -0,0 +1,88 @@
+import { Parameter, Action } from "@copilotkit/shared";
+
+export interface RemoteChainParameters {
+  name: string;
+  description: string;
+  chainUrl: string;
+  parameters?: Parameter[];
+  parameterType?: "single" | "multi";
+}
+
+export class RemoteChain {
+  name: string;
+  description: string;
+  chainUrl: string;
+  parameters?: Parameter[];
+  parameterType: "single" | "multi";
+
+  constructor(options: RemoteChainParameters) {
+    this.name = options.name;
+    this.description = options.description;
+    this.chainUrl = options.chainUrl;
+    this.parameters = options.parameters;
+    this.parameterType = options.parameterType || "multi";
+  }
+
+  async toAction(): Promise<Action<any>> {
+    if (!this.parameters) {
+      await this.inferLangServeParameters();
+    }
+
+    return {
+      name: this.name,
+      description: this.description,
+      parameters: this.parameters!,
+      handler: async (args: any) => {
+        // eslint-disable-next-line @typescript-eslint/no-var-requires
+        const { RemoteRunnable } = require("langchain/runnables/remote");
+        const runnable = new RemoteRunnable({ url: this.chainUrl });
+        let input: any;
+        if (this.parameterType === "single") {
+          input = args[Object.keys(args)[0]];
+        } else {
+          input = args;
+        }
+        return await runnable.invoke(input);
+      },
+    };
+  }
+
+  async inferLangServeParameters() {
+    const supportedTypes = ["string", "number", "boolean"];
+
+    let schemaUrl = this.chainUrl.replace(/\/+$/, "") + "/input_schema";
+    let schema = await fetch(schemaUrl)
+      .then((res) => res.json())
+      .catch(() => {
+        throw new Error("Failed to fetch langserve schema at " + schemaUrl);
+      });
+    // for now, don't use json schema, just do a simple conversion
+
+    if (supportedTypes.includes(schema.type)) {
+      this.parameterType = "single";
+      this.parameters = [
+        {
+          name: "input",
+          type: schema.type,
+          description: "The input to the chain",
+        },
+      ];
+    } else if (schema.type === "object") {
+      this.parameterType = "multi";
+      this.parameters = Object.keys(schema.properties).map((key) => {
+        let property = schema.properties[key];
+        if (!supportedTypes.includes(property.type)) {
+          throw new Error("Unsupported schema type");
+        }
+        return {
+          name: key,
+          type: property.type,
+          description: property.description || "",
+          required: schema.required?.includes(key) || false,
+        };
+      });
+    } else {
+      throw new Error("Unsupported schema type");
+    }
+  }
+}

package/src/service-adapters/langchain/types.ts

@@ -0,0 +1,14 @@
+import { AIMessage, AIMessageChunk, BaseMessageChunk } from "@langchain/core/messages";
+import {
+  IterableReadableStream,
+  IterableReadableStreamInterface,
+} from "@langchain/core/utils/stream";
+
+export type LangChainBaseMessageChunkStream = IterableReadableStream<BaseMessageChunk>;
+export type LangChainAIMessageChunkStream = IterableReadableStreamInterface<AIMessageChunk>;
+export type LangChainReturnType =
+  | LangChainBaseMessageChunkStream
+  | LangChainAIMessageChunkStream
+  | BaseMessageChunk
+  | string
+  | AIMessage;