@copilotkit/runtime 0.0.0-test-custom-tag-prerelease-1-20250108200215

package/src/service-adapters/events.ts

@@ -0,0 +1,377 @@
+import { Action, randomId } from "@copilotkit/shared";
+import {
+  of,
+  concat,
+  map,
+  scan,
+  concatMap,
+  ReplaySubject,
+  Subject,
+  firstValueFrom,
+  from,
+} from "rxjs";
+import { streamLangChainResponse } from "./langchain/utils";
+import { GuardrailsResult } from "../graphql/types/guardrails-result.type";
+import telemetry from "../lib/telemetry-client";
+import { isLangGraphAgentAction } from "../lib/runtime/remote-actions";
+import { ActionInput } from "../graphql/inputs/action.input";
+import { ActionExecutionMessage, ResultMessage } from "../graphql/types/converted";
+import { plainToInstance } from "class-transformer";
+
+export enum RuntimeEventTypes {
+  TextMessageStart = "TextMessageStart",
+  TextMessageContent = "TextMessageContent",
+  TextMessageEnd = "TextMessageEnd",
+  ActionExecutionStart = "ActionExecutionStart",
+  ActionExecutionArgs = "ActionExecutionArgs",
+  ActionExecutionEnd = "ActionExecutionEnd",
+  ActionExecutionResult = "ActionExecutionResult",
+  AgentStateMessage = "AgentStateMessage",
+}
+
+export type RuntimeEvent =
+  | { type: RuntimeEventTypes.TextMessageStart; messageId: string; parentMessageId?: string }
+  | {
+      type: RuntimeEventTypes.TextMessageContent;
+      messageId: string;
+      content: string;
+    }
+  | { type: RuntimeEventTypes.TextMessageEnd; messageId: string }
+  | {
+      type: RuntimeEventTypes.ActionExecutionStart;
+      actionExecutionId: string;
+      actionName: string;
+      parentMessageId?: string;
+    }
+  | { type: RuntimeEventTypes.ActionExecutionArgs; actionExecutionId: string; args: string }
+  | { type: RuntimeEventTypes.ActionExecutionEnd; actionExecutionId: string }
+  | {
+      type: RuntimeEventTypes.ActionExecutionResult;
+      actionName: string;
+      actionExecutionId: string;
+      result: string;
+    }
+  | {
+      type: RuntimeEventTypes.AgentStateMessage;
+      threadId: string;
+      agentName: string;
+      nodeName: string;
+      runId: string;
+      active: boolean;
+      role: string;
+      state: string;
+      running: boolean;
+    };
+
+interface RuntimeEventWithState {
+  event: RuntimeEvent | null;
+  callActionServerSide: boolean;
+  action: Action<any> | null;
+  actionExecutionId: string | null;
+  args: string;
+  actionExecutionParentMessageId: string | null;
+}
+
+type EventSourceCallback = (eventStream$: RuntimeEventSubject) => Promise<void>;
+
+export class RuntimeEventSubject extends ReplaySubject<RuntimeEvent> {
+  constructor() {
+    super();
+  }
+
+  sendTextMessageStart({
+    messageId,
+    parentMessageId,
+  }: {
+    messageId: string;
+    parentMessageId?: string;
+  }) {
+    this.next({ type: RuntimeEventTypes.TextMessageStart, messageId, parentMessageId });
+  }
+
+  sendTextMessageContent({ messageId, content }: { messageId: string; content: string }) {
+    this.next({ type: RuntimeEventTypes.TextMessageContent, content, messageId });
+  }
+
+  sendTextMessageEnd({ messageId }: { messageId: string }) {
+    this.next({ type: RuntimeEventTypes.TextMessageEnd, messageId });
+  }
+
+  sendTextMessage(messageId: string, content: string) {
+    this.sendTextMessageStart({ messageId });
+    this.sendTextMessageContent({ messageId, content });
+    this.sendTextMessageEnd({ messageId });
+  }
+
+  sendActionExecutionStart({
+    actionExecutionId,
+    actionName,
+    parentMessageId,
+  }: {
+    actionExecutionId: string;
+    actionName: string;
+    parentMessageId?: string;
+  }) {
+    this.next({
+      type: RuntimeEventTypes.ActionExecutionStart,
+      actionExecutionId,
+      actionName,
+      parentMessageId,
+    });
+  }
+
+  sendActionExecutionArgs({
+    actionExecutionId,
+    args,
+  }: {
+    actionExecutionId: string;
+    args: string;
+  }) {
+    this.next({ type: RuntimeEventTypes.ActionExecutionArgs, args, actionExecutionId });
+  }
+
+  sendActionExecutionEnd({ actionExecutionId }: { actionExecutionId: string }) {
+    this.next({ type: RuntimeEventTypes.ActionExecutionEnd, actionExecutionId });
+  }
+
+  sendActionExecution({
+    actionExecutionId,
+    actionName,
+    args,
+    parentMessageId,
+  }: {
+    actionExecutionId: string;
+    actionName: string;
+    args: string;
+    parentMessageId?: string;
+  }) {
+    this.sendActionExecutionStart({ actionExecutionId, actionName, parentMessageId });
+    this.sendActionExecutionArgs({ actionExecutionId, args });
+    this.sendActionExecutionEnd({ actionExecutionId });
+  }
+
+  sendActionExecutionResult({
+    actionExecutionId,
+    actionName,
+    result,
+  }: {
+    actionExecutionId: string;
+    actionName: string;
+    result: string;
+  }) {
+    this.next({
+      type: RuntimeEventTypes.ActionExecutionResult,
+      actionName,
+      actionExecutionId,
+      result,
+    });
+  }
+
+  sendAgentStateMessage({
+    threadId,
+    agentName,
+    nodeName,
+    runId,
+    active,
+    role,
+    state,
+    running,
+  }: {
+    threadId: string;
+    agentName: string;
+    nodeName: string;
+    runId: string;
+    active: boolean;
+    role: string;
+    state: string;
+    running: boolean;
+  }) {
+    this.next({
+      type: RuntimeEventTypes.AgentStateMessage,
+      threadId,
+      agentName,
+      nodeName,
+      runId,
+      active,
+      role,
+      state,
+      running,
+    });
+  }
+}
+
+export class RuntimeEventSource {
+  private eventStream$ = new RuntimeEventSubject();
+  private callback!: EventSourceCallback;
+
+  async stream(callback: EventSourceCallback): Promise<void> {
+    this.callback = callback;
+  }
+
+  sendErrorMessageToChat(message = "An error occurred. Please try again.") {
+    const errorMessage = `❌ ${message}`;
+    if (!this.callback) {
+      this.stream(async (eventStream$) => {
+        eventStream$.sendTextMessage(randomId(), errorMessage);
+      });
+    } else {
+      this.eventStream$.sendTextMessage(randomId(), errorMessage);
+    }
+  }
+
+  processRuntimeEvents({
+    serverSideActions,
+    guardrailsResult$,
+    actionInputsWithoutAgents,
+  }: {
+    serverSideActions: Action<any>[];
+    guardrailsResult$?: Subject<GuardrailsResult>;
+    actionInputsWithoutAgents: ActionInput[];
+  }) {
+    this.callback(this.eventStream$).catch((error) => {
+      console.error("Error in event source callback", error);
+      this.sendErrorMessageToChat();
+    });
+    return this.eventStream$.pipe(
+      // track state
+      scan(
+        (acc, event) => {
+          // It seems like this is needed so that rxjs recognizes the object has changed
+          // This fixes an issue where action were executed multiple times
+          // Not investigating further for now (Markus)
+          acc = { ...acc };
+
+          if (event.type === RuntimeEventTypes.ActionExecutionStart) {
+            acc.callActionServerSide =
+              serverSideActions.find((action) => action.name === event.actionName) !== undefined;
+            acc.args = "";
+            acc.actionExecutionId = event.actionExecutionId;
+            if (acc.callActionServerSide) {
+              acc.action = serverSideActions.find((action) => action.name === event.actionName);
+            }
+            acc.actionExecutionParentMessageId = event.parentMessageId;
+          } else if (event.type === RuntimeEventTypes.ActionExecutionArgs) {
+            acc.args += event.args;
+          }
+
+          acc.event = event;
+
+          return acc;
+        },
+        {
+          event: null,
+          callActionServerSide: false,
+          args: "",
+          actionExecutionId: null,
+          action: null,
+          actionExecutionParentMessageId: null,
+        } as RuntimeEventWithState,
+      ),
+      concatMap((eventWithState) => {
+        if (
+          eventWithState.event!.type === RuntimeEventTypes.ActionExecutionEnd &&
+          eventWithState.callActionServerSide
+        ) {
+          const toolCallEventStream$ = new RuntimeEventSubject();
+          executeAction(
+            toolCallEventStream$,
+            guardrailsResult$ ? guardrailsResult$ : null,
+            eventWithState.action!,
+            eventWithState.args,
+            eventWithState.actionExecutionParentMessageId,
+            eventWithState.actionExecutionId,
+            actionInputsWithoutAgents,
+          ).catch((error) => {
+            console.error(error);
+          });
+
+          telemetry.capture("oss.runtime.server_action_executed", {});
+          return concat(of(eventWithState.event!), toolCallEventStream$);
+        } else {
+          return of(eventWithState.event!);
+        }
+      }),
+    );
+  }
+}
+
+async function executeAction(
+  eventStream$: RuntimeEventSubject,
+  guardrailsResult$: Subject<GuardrailsResult> | null,
+  action: Action<any>,
+  actionArguments: string,
+  actionExecutionParentMessageId: string | null,
+  actionExecutionId: string,
+  actionInputsWithoutAgents: ActionInput[],
+) {
+  if (guardrailsResult$) {
+    const { status } = await firstValueFrom(guardrailsResult$);
+
+    if (status === "denied") {
+      eventStream$.complete();
+      return;
+    }
+  }
+
+  // Prepare arguments for function calling
+  let args: Record<string, any>[] = [];
+  if (actionArguments) {
+    try {
+      args = JSON.parse(actionArguments);
+    } catch (e) {
+      console.warn("Action argument unparsable", { actionArguments });
+    }
+  }
+
+  // handle LangGraph agents
+  if (isLangGraphAgentAction(action)) {
+    const result = `${action.name} agent started`;
+
+    const agentExecution = plainToInstance(ActionExecutionMessage, {
+      id: actionExecutionId,
+      createdAt: new Date(),
+      name: action.name,
+      arguments: JSON.parse(actionArguments),
+      parentMessageId: actionExecutionParentMessageId ?? actionExecutionId,
+    });
+
+    const agentExecutionResult = plainToInstance(ResultMessage, {
+      id: "result-" + actionExecutionId,
+      createdAt: new Date(),
+      actionExecutionId,
+      actionName: action.name,
+      result,
+    });
+
+    eventStream$.sendActionExecutionResult({
+      actionExecutionId,
+      actionName: action.name,
+      result,
+    });
+
+    const stream = await action.langGraphAgentHandler({
+      name: action.name,
+      actionInputsWithoutAgents,
+      additionalMessages: [agentExecution, agentExecutionResult],
+    });
+
+    // forward to eventStream$
+    from(stream).subscribe({
+      next: (event) => eventStream$.next(event),
+      error: (err) => console.error("Error in stream", err),
+      complete: () => eventStream$.complete(),
+    });
+  } else {
+    // call the function
+    const result = await action.handler?.(args);
+
+    await streamLangChainResponse({
+      result,
+      eventStream$,
+      actionExecution: {
+        name: action.name,
+        id: actionExecutionId,
+      },
+    });
+  }
+}

package/src/service-adapters/experimental/empty/empty-adapter.ts

@@ -0,0 +1,33 @@
+/**
+ * CopilotKit Empty Adapter
+ *
+ * This adapter is meant to preserve adherence to runtime requirements, while doing nothing
+ * Ideal if you don't want to connect an LLM the to the runtime, and only use your LangGraph agent.
+ * Be aware that Copilot Suggestions will not work if you use this adapter
+ *
+ * ## Example
+ *
+ * ```ts
+ * import { CopilotRuntime, ExperimentalEmptyAdapter } from "@copilotkit/runtime";
+ *
+ * const copilotKit = new CopilotRuntime();
+ *
+ * return new ExperimentalEmptyAdapter();
+ * ```
+ */
+import {
+  CopilotServiceAdapter,
+  CopilotRuntimeChatCompletionRequest,
+  CopilotRuntimeChatCompletionResponse,
+} from "../../service-adapter";
+import { randomId } from "@copilotkit/shared";
+
+export class ExperimentalEmptyAdapter implements CopilotServiceAdapter {
+  async process(
+    request: CopilotRuntimeChatCompletionRequest,
+  ): Promise<CopilotRuntimeChatCompletionResponse> {
+    return {
+      threadId: request.threadId || randomId(),
+    };
+  }
+}

package/src/service-adapters/experimental/ollama/ollama-adapter.ts

@@ -0,0 +1,79 @@
+/**
+ * CopilotKit Adapter for Ollama
+ *
+ * <RequestExample>
+ * ```jsx CopilotRuntime Example
+ * const copilotKit = new CopilotRuntime();
+ * return copilotKit.response(req, new OllamaAdapter());
+ * ```
+ * </RequestExample>
+ *
+ * You can easily set the model to use by passing it to the constructor.
+ * ```jsx
+ * const copilotKit = new CopilotRuntime();
+ * return copilotKit.response(
+ *   req,
+ *   new OllamaAdapter({ model: "llama3-70b-8192" }),
+ * );
+ * ```
+ */
+import { TextMessage } from "../../../graphql/types/converted";
+import {
+  CopilotServiceAdapter,
+  CopilotRuntimeChatCompletionRequest,
+  CopilotRuntimeChatCompletionResponse,
+} from "../../service-adapter";
+import { Ollama } from "@langchain/community/llms/ollama";
+import { randomId } from "@copilotkit/shared";
+
+const DEFAULT_MODEL = "llama3:latest";
+
+interface OllamaAdapterOptions {
+  model?: string;
+}
+
+export class ExperimentalOllamaAdapter implements CopilotServiceAdapter {
+  private model: string;
+
+  constructor(options?: OllamaAdapterOptions) {
+    if (options?.model) {
+      this.model = options.model;
+    } else {
+      this.model = DEFAULT_MODEL;
+    }
+  }
+
+  async process(
+    request: CopilotRuntimeChatCompletionRequest,
+  ): Promise<CopilotRuntimeChatCompletionResponse> {
+    const { messages, actions, eventSource } = request;
+    // const messages = this.transformMessages(forwardedProps.messages);
+
+    const ollama = new Ollama({
+      model: this.model,
+    });
+    const contents = (messages.filter((m) => m.isTextMessage()) as TextMessage[]).map(
+      (m) => m.content,
+    );
+    const _stream = await ollama.stream(contents); // [TODO] role info is dropped...
+
+    eventSource.stream(async (eventStream$) => {
+      const currentMessageId = randomId();
+      eventStream$.sendTextMessageStart({ messageId: currentMessageId });
+      for await (const chunkText of _stream) {
+        eventStream$.sendTextMessageContent({
+          messageId: currentMessageId,
+          content: chunkText,
+        });
+      }
+      eventStream$.sendTextMessageEnd({ messageId: currentMessageId });
+      // we may need to add this later.. [nc]
+      // let calls = (await result.response).functionCalls();
+
+      eventStream$.complete();
+    });
+    return {
+      threadId: request.threadId || randomId(),
+    };
+  }
+}

package/src/service-adapters/google/google-genai-adapter.ts

@@ -0,0 +1,39 @@
+/**
+ * Copilot Runtime adapter for Google Generative AI (e.g. Gemini).
+ *
+ * ## Example
+ *
+ * ```ts
+ * import { CopilotRuntime, GoogleGenerativeAIAdapter } from "@copilotkit/runtime";
+ * const { GoogleGenerativeAI } = require("@google/generative-ai");
+ *
+ * const genAI = new GoogleGenerativeAI(process.env["GOOGLE_API_KEY"]);
+ *
+ * const copilotKit = new CopilotRuntime();
+ *
+ * return new GoogleGenerativeAIAdapter({ model: "gemini-1.5-pro" });
+ * ```
+ */
+import { ChatGoogle } from "@langchain/google-gauth";
+import { LangChainAdapter } from "../langchain/langchain-adapter";
+
+interface GoogleGenerativeAIAdapterOptions {
+  /**
+   * A custom Google Generative AI model to use.
+   */
+  model?: string;
+}
+
+export class GoogleGenerativeAIAdapter extends LangChainAdapter {
+  constructor(options?: GoogleGenerativeAIAdapterOptions) {
+    super({
+      chainFn: async ({ messages, tools, threadId }) => {
+        const model = new ChatGoogle({
+          modelName: options?.model ?? "gemini-1.5-pro",
+          apiVersion: "v1beta",
+        }).bindTools(tools);
+        return model.stream(messages, { metadata: { conversation_id: threadId } });
+      },
+    });
+  }
+}

package/src/service-adapters/groq/groq-adapter.ts

@@ -0,0 +1,173 @@
+/**
+ * Copilot Runtime adapter for Groq.
+ *
+ * ## Example
+ *
+ * ```ts
+ * import { CopilotRuntime, GroqAdapter } from "@copilotkit/runtime";
+ * import { Groq } from "groq-sdk";
+ *
+ * const groq = new Groq({ apiKey: process.env["GROQ_API_KEY"] });
+ *
+ * const copilotKit = new CopilotRuntime();
+ *
+ * return new GroqAdapter({ groq, model: "<model-name>" });
+ * ```
+ */
+import { Groq } from "groq-sdk";
+import {
+  CopilotServiceAdapter,
+  CopilotRuntimeChatCompletionRequest,
+  CopilotRuntimeChatCompletionResponse,
+} from "../service-adapter";
+import {
+  convertActionInputToOpenAITool,
+  convertMessageToOpenAIMessage,
+  limitMessagesToTokenCount,
+} from "../openai/utils";
+import { randomId } from "@copilotkit/shared";
+
+const DEFAULT_MODEL = "llama3-groq-70b-8192-tool-use-preview";
+
+export interface GroqAdapterParams {
+  /**
+   * An optional Groq instance to use.
+   */
+  groq?: Groq;
+
+  /**
+   * The model to use.
+   */
+  model?: string;
+
+  /**
+   * Whether to disable parallel tool calls.
+   * You can disable parallel tool calls to force the model to execute tool calls sequentially.
+   * This is useful if you want to execute tool calls in a specific order so that the state changes
+   * introduced by one tool call are visible to the next tool call. (i.e. new actions or readables)
+   *
+   * @default false
+   */
+  disableParallelToolCalls?: boolean;
+}
+
+export class GroqAdapter implements CopilotServiceAdapter {
+  private model: string = DEFAULT_MODEL;
+
+  private disableParallelToolCalls: boolean = false;
+  private _groq: Groq;
+  public get groq(): Groq {
+    return this._groq;
+  }
+
+  constructor(params?: GroqAdapterParams) {
+    this._groq = params?.groq || new Groq({});
+    if (params?.model) {
+      this.model = params.model;
+    }
+    this.disableParallelToolCalls = params?.disableParallelToolCalls || false;
+  }
+
+  async process(
+    request: CopilotRuntimeChatCompletionRequest,
+  ): Promise<CopilotRuntimeChatCompletionResponse> {
+    const {
+      threadId,
+      model = this.model,
+      messages,
+      actions,
+      eventSource,
+      forwardedParameters,
+    } = request;
+    const tools = actions.map(convertActionInputToOpenAITool);
+
+    let openaiMessages = messages.map(convertMessageToOpenAIMessage);
+    openaiMessages = limitMessagesToTokenCount(openaiMessages, tools, model);
+
+    let toolChoice: any = forwardedParameters?.toolChoice;
+    if (forwardedParameters?.toolChoice === "function") {
+      toolChoice = {
+        type: "function",
+        function: { name: forwardedParameters.toolChoiceFunctionName },
+      };
+    }
+    const stream = await this.groq.chat.completions.create({
+      model: model,
+      stream: true,
+      messages: openaiMessages,
+      ...(tools.length > 0 && { tools }),
+      ...(forwardedParameters?.maxTokens && {
+        max_tokens: forwardedParameters.maxTokens,
+      }),
+      ...(forwardedParameters?.stop && { stop: forwardedParameters.stop }),
+      ...(toolChoice && { tool_choice: toolChoice }),
+      ...(this.disableParallelToolCalls && { parallel_tool_calls: false }),
+      ...(forwardedParameters?.temperature && { temperature: forwardedParameters.temperature }),
+    });
+
+    eventSource.stream(async (eventStream$) => {
+      let mode: "function" | "message" | null = null;
+      let currentMessageId: string;
+      let currentToolCallId: string;
+
+      for await (const chunk of stream) {
+        const toolCall = chunk.choices[0].delta.tool_calls?.[0];
+        const content = chunk.choices[0].delta.content;
+
+        // When switching from message to function or vice versa,
+        // send the respective end event.
+        // If toolCall?.id is defined, it means a new tool call starts.
+        if (mode === "message" && toolCall?.id) {
+          mode = null;
+          eventStream$.sendTextMessageEnd({ messageId: currentMessageId });
+        } else if (mode === "function" && (toolCall === undefined || toolCall?.id)) {
+          mode = null;
+          eventStream$.sendActionExecutionEnd({ actionExecutionId: currentToolCallId });
+        }
+
+        // If we send a new message type, send the appropriate start event.
+        if (mode === null) {
+          if (toolCall?.id) {
+            mode = "function";
+            currentToolCallId = toolCall!.id;
+            eventStream$.sendActionExecutionStart({
+              actionExecutionId: currentToolCallId,
+              actionName: toolCall!.function!.name,
+              parentMessageId: chunk.id,
+            });
+          } else if (content) {
+            mode = "message";
+            currentMessageId = chunk.id;
+            eventStream$.sendTextMessageStart({ messageId: currentMessageId });
+          }
+        }
+
+        // send the content events
+        if (mode === "message" && content) {
+          eventStream$.sendTextMessageContent({
+            messageId: currentMessageId,
+            content,
+          });
+        } else if (mode === "function" && toolCall?.function?.arguments) {
+          eventStream$.sendActionExecutionArgs({
+            actionExecutionId: currentToolCallId,
+            args: toolCall.function.arguments,
+          });
+        }
+      }
+
+      // send the end events
+      if (mode === "message") {
+        eventStream$.sendTextMessageEnd({ messageId: currentMessageId });
+      } else if (mode === "function") {
+        eventStream$.sendActionExecutionEnd({ actionExecutionId: currentToolCallId });
+      }
+
+      eventStream$.complete();
+    });
+
+    return {
+      threadId: threadId || randomId(),
+    };
+  }
+}