@copilotkit/runtime 0.37.0

package/src/lib/integrations/shared.ts

@@ -0,0 +1,68 @@
+import { YogaInitialContext } from "graphql-yoga";
+import { GuardrailsOptions } from "../guardrails";
+import { buildSchemaSync } from "type-graphql";
+import { CopilotResolver } from "../../graphql/resolvers/copilot.resolver";
+import { useDeferStream } from "@graphql-yoga/plugin-defer-stream";
+import { CopilotRuntime } from "../copilot-runtime";
+import { CopilotServiceAdapter } from "../../service-adapters";
+
+type AnyPrimitive = string | boolean | number | null;
+export type CopilotRequestContextProperties = Record<
+  string,
+  AnyPrimitive | Record<string, AnyPrimitive>
+>;
+
+type CopilotKitContext = {
+  runtime: CopilotRuntime;
+  serviceAdapter: CopilotServiceAdapter;
+  properties: CopilotRequestContextProperties;
+};
+
+export type GraphQLContext = YogaInitialContext & {
+  _copilotkit: CopilotKitContext;
+};
+
+export interface CreateCopilotRuntimeServerOptions {
+  runtime: CopilotRuntime;
+  serviceAdapter: CopilotServiceAdapter;
+  guardrails?: GuardrailsOptions;
+}
+
+export async function createContext(
+  initialContext: YogaInitialContext,
+  copilotKitContext: CopilotKitContext,
+): Promise<Partial<GraphQLContext>> {
+  const ctx: GraphQLContext = {
+    ...initialContext,
+    _copilotkit: {
+      ...copilotKitContext,
+    },
+  };
+
+  return ctx;
+}
+
+export function buildSchema(
+  options: {
+    emitSchemaFile?: string;
+  } = {},
+) {
+  const schema = buildSchemaSync({
+    resolvers: [CopilotResolver],
+    emitSchemaFile: options.emitSchemaFile,
+  });
+  return schema;
+}
+
+export function getCommonConfig(options?: CreateCopilotRuntimeServerOptions) {
+  return {
+    schema: buildSchema(),
+    plugins: [useDeferStream()],
+    context: (ctx: YogaInitialContext): Promise<Partial<GraphQLContext>> =>
+      createContext(ctx, {
+        runtime: options.runtime,
+        serviceAdapter: options.serviceAdapter,
+        properties: {},
+      }),
+  };
+}

package/src/service-adapters/conversion.ts

@@ -0,0 +1,47 @@
+import {
+  ActionExecutionMessage,
+  Message,
+  ResultMessage,
+  TextMessage,
+} from "../graphql/types/converted";
+import { MessageInput } from "../graphql/inputs/message.input";
+import { plainToInstance } from "class-transformer";
+
+export function convertGqlInputToMessages(inputMessages: MessageInput[]): Message[] {
+  const messages: Message[] = [];
+
+  for (const message of inputMessages) {
+    if (message.textMessage) {
+      messages.push(
+        plainToInstance(TextMessage, {
+          id: message.id,
+          createdAt: message.createdAt,
+          role: message.textMessage.role,
+          content: message.textMessage.content,
+        }),
+      );
+    } else if (message.actionExecutionMessage) {
+      messages.push(
+        plainToInstance(ActionExecutionMessage, {
+          id: message.id,
+          createdAt: message.createdAt,
+          name: message.actionExecutionMessage.name,
+          arguments: JSON.parse(message.actionExecutionMessage.arguments),
+          scope: message.actionExecutionMessage.scope,
+        }),
+      );
+    } else if (message.resultMessage) {
+      messages.push(
+        plainToInstance(ResultMessage, {
+          id: message.id,
+          createdAt: message.createdAt,
+          actionExecutionId: message.resultMessage.actionExecutionId,
+          actionName: message.resultMessage.actionName,
+          result: message.resultMessage.result,
+        }),
+      );
+    }
+  }
+
+  return messages;
+}

package/src/service-adapters/events.ts

@@ -0,0 +1,197 @@
+import { Action } from "@copilotkit/shared";
+import { of, concat, map, scan, concatMap, ReplaySubject } from "rxjs";
+import { streamLangChainResponse } from "./langchain/utils";
+
+export enum RuntimeEventTypes {
+  TextMessageStart = "TextMessageStart",
+  TextMessageContent = "TextMessageContent",
+  TextMessageEnd = "TextMessageEnd",
+  ActionExecutionStart = "ActionExecutionStart",
+  ActionExecutionArgs = "ActionExecutionArgs",
+  ActionExecutionEnd = "ActionExecutionEnd",
+  ActionExecutionResult = "ActionExecutionResult",
+}
+
+type FunctionCallScope = "client" | "server";
+
+export type RuntimeEvent =
+  | { type: RuntimeEventTypes.TextMessageStart; messageId: string }
+  | {
+      type: RuntimeEventTypes.TextMessageContent;
+      content: string;
+    }
+  | { type: RuntimeEventTypes.TextMessageEnd }
+  | {
+      type: RuntimeEventTypes.ActionExecutionStart;
+      actionExecutionId: string;
+      actionName: string;
+      scope?: FunctionCallScope;
+    }
+  | { type: RuntimeEventTypes.ActionExecutionArgs; args: string }
+  | { type: RuntimeEventTypes.ActionExecutionEnd }
+  | {
+      type: RuntimeEventTypes.ActionExecutionResult;
+      actionName: string;
+      actionExecutionId: string;
+      result: string;
+    };
+
+interface RuntimeEventWithState {
+  event: RuntimeEvent | null;
+  callActionServerSide: boolean;
+  action: Action<any> | null;
+  actionExecutionId: string | null;
+  args: string;
+}
+
+type EventSourceCallback = (eventStream$: RuntimeEventSubject) => Promise<void>;
+
+export class RuntimeEventSubject extends ReplaySubject<RuntimeEvent> {
+  constructor() {
+    super();
+  }
+
+  sendTextMessageStart(messageId: string) {
+    this.next({ type: RuntimeEventTypes.TextMessageStart, messageId });
+  }
+
+  sendTextMessageContent(content: string) {
+    this.next({ type: RuntimeEventTypes.TextMessageContent, content });
+  }
+
+  sendTextMessageEnd() {
+    this.next({ type: RuntimeEventTypes.TextMessageEnd });
+  }
+
+  sendTextMessage(messageId: string, content: string) {
+    this.sendTextMessageStart(messageId);
+    this.sendTextMessageContent(content);
+    this.sendTextMessageEnd();
+  }
+
+  sendActionExecutionStart(actionExecutionId: string, actionName: string) {
+    this.next({
+      type: RuntimeEventTypes.ActionExecutionStart,
+      actionExecutionId,
+      actionName,
+    });
+  }
+
+  sendActionExecutionArgs(args: string) {
+    this.next({ type: RuntimeEventTypes.ActionExecutionArgs, args });
+  }
+
+  sendActionExecutionEnd() {
+    this.next({ type: RuntimeEventTypes.ActionExecutionEnd });
+  }
+
+  sendActionExecution(actionExecutionId: string, toolName: string, args: string) {
+    this.sendActionExecutionStart(actionExecutionId, toolName);
+    this.sendActionExecutionArgs(args);
+    this.sendActionExecutionEnd();
+  }
+
+  sendActionExecutionResult(actionExecutionId: string, actionName: string, result: string) {
+    this.next({
+      type: RuntimeEventTypes.ActionExecutionResult,
+      actionName,
+      actionExecutionId,
+      result,
+    });
+  }
+}
+
+export class RuntimeEventSource {
+  private eventStream$ = new RuntimeEventSubject();
+  private callback!: EventSourceCallback;
+
+  async stream(callback: EventSourceCallback): Promise<void> {
+    this.callback = callback;
+  }
+
+  process(serversideActions: Action<any>[]) {
+    this.callback(this.eventStream$).catch((error) => {
+      console.error("Error in event source callback", error);
+    });
+    return this.eventStream$.pipe(
+      // mark tools for server side execution
+      map((event) => {
+        if (event.type === RuntimeEventTypes.ActionExecutionStart) {
+          event.scope = serversideActions.find((action) => action.name === event.actionName)
+            ? "server"
+            : "client";
+        }
+        return event;
+      }),
+      // track state
+      scan(
+        (acc, event) => {
+          if (event.type === RuntimeEventTypes.ActionExecutionStart) {
+            acc.callActionServerSide = event.scope === "server";
+            acc.args = "";
+            acc.actionExecutionId = event.actionExecutionId;
+            if (acc.callActionServerSide) {
+              acc.action = serversideActions.find((action) => action.name === event.actionName);
+            }
+          } else if (event.type === RuntimeEventTypes.ActionExecutionArgs) {
+            acc.args += event.args;
+          }
+
+          acc.event = event;
+          return acc;
+        },
+        {
+          event: null,
+          callActionServerSide: false,
+          args: "",
+          actionExecutionId: null,
+          action: null,
+        } as RuntimeEventWithState,
+      ),
+      concatMap((eventWithState) => {
+        if (
+          eventWithState.event!.type === RuntimeEventTypes.ActionExecutionEnd &&
+          eventWithState.callActionServerSide
+        ) {
+          const toolCallEventStream$ = new RuntimeEventSubject();
+          executeAction(
+            toolCallEventStream$,
+            eventWithState.action!,
+            eventWithState.args,
+            eventWithState.actionExecutionId,
+          ).catch((error) => {
+            console.error(error);
+          });
+          return concat(of(eventWithState.event!), toolCallEventStream$);
+        } else {
+          return of(eventWithState.event!);
+        }
+      }),
+    );
+  }
+}
+
+async function executeAction(
+  eventStream$: RuntimeEventSubject,
+  action: Action<any>,
+  actionArguments: string,
+  actionExecutionId: string,
+) {
+  // Prepare arguments for function calling
+  let args: Record<string, any>[] = [];
+  if (actionArguments) {
+    args = JSON.parse(actionArguments);
+  }
+
+  // call the function
+  const result = await action.handler(args);
+
+  await streamLangChainResponse({
+    result,
+    eventStream$,
+    actionExecution: {
+      name: action.name,
+      id: actionExecutionId,
+    },
+  });
+}

package/src/service-adapters/experimental/groq/groq-adapter.ts

@@ -0,0 +1,124 @@
+/**
+ * CopilotKit Adapter for Groq
+ *
+ * <RequestExample>
+ * ```jsx CopilotRuntime Example
+ * const copilotKit = new CopilotRuntime();
+ * return copilotKit.response(req, new GroqAdapter());
+ * ```
+ * </RequestExample>
+ *
+ * You can easily set the model to use by passing it to the constructor.
+ * ```jsx
+ * const copilotKit = new CopilotRuntime();
+ * return copilotKit.response(
+ *   req,
+ *   new GroqAdapter({ model: "llama3-70b-8192" }),
+ * );
+ * ```
+ */
+import { nanoid } from "nanoid";
+import { TextMessage } from "../../../graphql/types/converted";
+import {
+  CopilotServiceAdapter,
+  CopilotRuntimeChatCompletionRequest,
+  CopilotRuntimeChatCompletionResponse,
+} from "../../service-adapter";
+
+import Groq from "groq-sdk";
+
+const DEFAULT_MODEL = "llama3-70b-8192";
+
+interface GroqAdapterOptions {
+  model?: string;
+}
+
+export class ExperimentalGroqAdapter implements CopilotServiceAdapter {
+  private model: string;
+
+  constructor(options?: GroqAdapterOptions) {
+    if (options?.model) {
+      this.model = options.model;
+    } else {
+      this.model = DEFAULT_MODEL;
+    }
+  }
+
+  async process(
+    request: CopilotRuntimeChatCompletionRequest,
+  ): Promise<CopilotRuntimeChatCompletionResponse> {
+    const groq = new Groq();
+
+    const messages = (
+      request.messages.filter((m) => m instanceof TextMessage) as TextMessage[]
+    ).map((m) => ({
+      role: m.role,
+      content: m.content,
+    }));
+
+    const max_tokens = maxTokensForGroqModel(this.model);
+
+    const _stream = await groq.chat.completions.create({
+      //
+      // Required parameters
+      //
+      messages: messages,
+
+      // The language model which will generate the completion.
+      model: this.model,
+
+      //
+      // Optional parameters
+      //
+
+      // Controls randomness: lowering results in less random completions.
+      // As the temperature approaches zero, the model will become deterministic
+      // and repetitive.
+      temperature: 0.5, // [TODO]
+
+      // The maximum number of tokens to generate. Requests can use up to
+      // 2048 tokens shared between prompt and completion.
+      max_tokens: max_tokens,
+
+      // Controls diversity via nucleus sampling: 0.5 means half of all
+      // likelihood-weighted options are considered.
+      top_p: 1, // [TODO]
+
+      // A stop sequence is a predefined or user-specified text string that
+      // signals an AI to stop generating content, ensuring its responses
+      // remain focused and concise. Examples include punctuation marks and
+      // markers like "[end]".
+      stop: null,
+
+      // If set, partial message deltas will be sent.
+      stream: true,
+    });
+
+    request.eventSource.stream(async (eventStream$) => {
+      eventStream$.sendTextMessageStart(nanoid());
+      for await (const chunk of _stream) {
+        if (chunk.choices[0]?.delta?.content) {
+          eventStream$.sendTextMessageContent(chunk.choices[0]?.delta?.content);
+        }
+      }
+      eventStream$.sendTextMessageEnd();
+      // we may need to add this later.. [nc]
+      // let calls = (await result.response).functionCalls();
+
+      eventStream$.complete();
+    });
+    return {};
+  }
+}
+
+export function maxTokensForGroqModel(model: string): number {
+  return maxTokensByModel[model] || DEFAULT_MAX_TOKENS;
+}
+
+const DEFAULT_MAX_TOKENS = 8192;
+
+const maxTokensByModel: { [key: string]: number } = {
+  // llama3
+  "llama3-8b-8192": DEFAULT_MAX_TOKENS,
+  "llama3-70b-8192": DEFAULT_MAX_TOKENS,
+};

package/src/service-adapters/experimental/ollama/ollama-adapter.ts

@@ -0,0 +1,75 @@
+/**
+ * CopilotKit Adapter for Ollama
+ *
+ * <RequestExample>
+ * ```jsx CopilotRuntime Example
+ * const copilotKit = new CopilotRuntime();
+ * return copilotKit.response(req, new OllamaAdapter());
+ * ```
+ * </RequestExample>
+ *
+ * You can easily set the model to use by passing it to the constructor.
+ * ```jsx
+ * const copilotKit = new CopilotRuntime();
+ * return copilotKit.response(
+ *   req,
+ *   new OllamaAdapter({ model: "llama3-70b-8192" }),
+ * );
+ * ```
+ */
+import { nanoid } from "nanoid";
+import { TextMessage } from "../../../graphql/types/converted";
+import {
+  CopilotServiceAdapter,
+  CopilotRuntimeChatCompletionRequest,
+  CopilotRuntimeChatCompletionResponse,
+} from "../../service-adapter";
+// import { writeChatCompletionChunk, writeChatCompletionEnd } from "../utils";
+// import { ChatCompletionChunk, Message } from "@copilotkit/shared";
+import { Ollama } from "@langchain/community/llms/ollama";
+
+const DEFAULT_MODEL = "llama3:latest";
+
+interface OllamaAdapterOptions {
+  model?: string;
+}
+
+export class ExperimentalOllamaAdapter implements CopilotServiceAdapter {
+  private model: string;
+
+  constructor(options?: OllamaAdapterOptions) {
+    if (options?.model) {
+      this.model = options.model;
+    } else {
+      this.model = DEFAULT_MODEL;
+    }
+  }
+
+  async process(
+    request: CopilotRuntimeChatCompletionRequest,
+  ): Promise<CopilotRuntimeChatCompletionResponse> {
+    const { messages, actions, eventSource } = request;
+    // const messages = this.transformMessages(forwardedProps.messages);
+
+    const ollama = new Ollama({
+      model: this.model,
+    });
+    const contents = (messages.filter((m) => m instanceof TextMessage) as TextMessage[]).map(
+      (m) => m.content,
+    );
+    const _stream = await ollama.stream(contents); // [TODO] role info is dropped...
+
+    eventSource.stream(async (eventStream$) => {
+      eventStream$.sendTextMessageStart(nanoid());
+      for await (const chunkText of _stream) {
+        eventStream$.sendTextMessageContent(chunkText);
+      }
+      eventStream$.sendTextMessageEnd();
+      // we may need to add this later.. [nc]
+      // let calls = (await result.response).functionCalls();
+
+      eventStream$.complete();
+    });
+    return {};
+  }
+}

package/src/service-adapters/google/google-genai-adapter.ts

@@ -0,0 +1,149 @@
+/**
+ * CopilotKit Adapter for Google Gemini
+ *
+ * Use this adapter for a Google Gemini backend.
+ *
+ * <RequestExample>
+ * ```typescript
+ * const copilotKit = new CopilotRuntime();
+ * return copilotKit.response(
+ *   req,
+ *   new GoogleGenerativeAIAdapter()
+ * );
+ * ```
+ * </RequestExample>
+ *
+ * To set up a different model, pass the model prop:
+ *
+ * ```typescript
+ * const copilotKit = new CopilotRuntime();
+ * const genAI = new GoogleGenerativeAI(
+ *  process.env["GOOGLE_API_KEY"]!
+ * );
+ * const model = genAI.getGenerativeModel(
+ *  { model: "gemini-pro" }
+ * );
+ * return copilotKit.response(
+ *   req,
+ *   new GoogleGenerativeAIAdapter()
+ * );
+ * ```
+ */
+import { CopilotServiceAdapter } from "../service-adapter";
+import {
+  CopilotRuntimeChatCompletionRequest,
+  CopilotRuntimeChatCompletionResponse,
+} from "../service-adapter";
+import { GenerativeModel, GoogleGenerativeAI } from "@google/generative-ai";
+import { TextMessage } from "../../graphql/types/converted";
+import { convertMessageToGoogleGenAIMessage, transformActionToGoogleGenAITool } from "./utils";
+import { nanoid } from "nanoid";
+
+interface GoogleGenerativeAIAdapterOptions {
+  /**
+   * A custom `GenerativeModel` to use for the request.
+   */
+  model?: GenerativeModel;
+}
+
+export class GoogleGenerativeAIAdapter implements CopilotServiceAdapter {
+  private model: GenerativeModel;
+
+  constructor(options?: GoogleGenerativeAIAdapterOptions) {
+    if (options?.model) {
+      this.model = options.model;
+    } else {
+      const genAI = new GoogleGenerativeAI(process.env["GOOGLE_API_KEY"]!);
+      this.model = genAI.getGenerativeModel({ model: "gemini-pro" });
+    }
+  }
+
+  async process(
+    request: CopilotRuntimeChatCompletionRequest,
+  ): Promise<CopilotRuntimeChatCompletionResponse> {
+    const { messages, actions, eventSource } = request;
+
+    // get the history (everything except the first and last message)
+    const history = messages.slice(1, -1).map(convertMessageToGoogleGenAIMessage);
+
+    // get the current message (the last message)
+    const currentMessage = convertMessageToGoogleGenAIMessage(messages.at(-1));
+    if (!currentMessage) {
+      throw new Error("No current message");
+    }
+
+    let systemMessage: string;
+    const firstMessage = messages.at(0);
+    if (firstMessage instanceof TextMessage && firstMessage.role === "system") {
+      systemMessage = firstMessage.content.trim();
+    } else {
+      throw new Error("First message is not a system message");
+    }
+
+    const tools = actions.map(transformActionToGoogleGenAITool);
+
+    const isFirstGenGeminiPro =
+      this.model.model === "gemini-pro" || this.model.model === "models/gemini-pro";
+
+    const chat = this.model.startChat({
+      history: [
+        ...history,
+        // gemini-pro does not support system instructions, so we need to add them to the history
+        ...(isFirstGenGeminiPro ? [{ role: "user", parts: [{ text: systemMessage }] }] : []),
+      ],
+      // only gemini-1.5-pro-latest and later supports setting system instructions
+      ...(isFirstGenGeminiPro
+        ? {}
+        : { systemInstruction: { role: "user", parts: [{ text: systemMessage }] } }),
+      tools,
+    });
+
+    const result = await chat.sendMessageStream(currentMessage.parts);
+
+    eventSource.stream(async (eventStream$) => {
+      let isTextMessage = false;
+      for await (const chunk of result.stream) {
+        const chunkText = chunk.text();
+        if (!isTextMessage) {
+          isTextMessage = true;
+          eventStream$.sendTextMessageStart(nanoid());
+        }
+        eventStream$.sendTextMessageContent(chunkText);
+      }
+      if (isTextMessage) {
+        eventStream$.sendTextMessageEnd();
+      }
+
+      let calls = (await result.response).functionCalls();
+      if (calls) {
+        for (let call of calls) {
+          eventStream$.sendActionExecution(
+            nanoid(),
+            call.name,
+            JSON.stringify(replaceNewlinesInObject(call.args)),
+          );
+        }
+      }
+      eventStream$.complete();
+    });
+
+    return {};
+  }
+}
+
+function replaceNewlinesInObject(obj: any): any {
+  if (typeof obj === "string") {
+    return obj.replace(/\\\\n/g, "\n");
+  } else if (Array.isArray(obj)) {
+    return obj.map(replaceNewlinesInObject);
+  } else if (typeof obj === "object" && obj !== null) {
+    const newObj: any = {};
+    for (const key in obj) {
+      if (obj.hasOwnProperty(key)) {
+        newObj[key] = replaceNewlinesInObject(obj[key]);
+      }
+    }
+    return newObj;
+  }
+  return obj;
+}