@copilotkit/runtime 0.37.0

package/src/graphql/types/copilot-response.type.ts

@@ -0,0 +1,75 @@
+import { Field, InterfaceType, ObjectType } from "type-graphql";
+import { MessageRole, ActionExecutionScope } from "./enums";
+import { MessageStatusUnion } from "./message-status.type";
+import { ResponseStatusUnion } from "./response-status.type";
+
+@InterfaceType({
+  resolveType(value) {
+    if (value.hasOwnProperty("content")) {
+      return TextMessageOutput;
+    } else if (value.hasOwnProperty("name")) {
+      return ActionExecutionMessageOutput;
+    } else if (value.hasOwnProperty("result")) {
+      return ResultMessageOutput;
+    }
+    return undefined;
+  },
+})
+abstract class BaseMessageOutput {
+  @Field(() => String)
+  id: string;
+
+  @Field(() => Date)
+  createdAt: Date;
+
+  @Field(() => MessageStatusUnion)
+  status: typeof MessageStatusUnion;
+}
+
+@ObjectType({ implements: BaseMessageOutput })
+export class TextMessageOutput {
+  @Field(() => MessageRole)
+  role: MessageRole;
+
+  @Field(() => [String])
+  content: string[];
+}
+
+@ObjectType({ implements: BaseMessageOutput })
+export class ActionExecutionMessageOutput {
+  @Field(() => String)
+  name: string;
+
+  @Field(() => ActionExecutionScope)
+  scope: ActionExecutionScope;
+
+  @Field(() => [String])
+  arguments: string[];
+}
+
+@ObjectType({ implements: BaseMessageOutput })
+export class ResultMessageOutput {
+  @Field(() => String)
+  actionExecutionId: string;
+
+  @Field(() => String)
+  actionName: string;
+
+  @Field(() => String)
+  result: string;
+}
+
+@ObjectType()
+export class CopilotResponse {
+  @Field(() => String)
+  threadId!: string;
+
+  @Field(() => ResponseStatusUnion)
+  status: typeof ResponseStatusUnion;
+
+  @Field({ nullable: true })
+  runId?: string;
+
+  @Field(() => [BaseMessageOutput])
+  messages: (typeof BaseMessageOutput)[];
+}

package/src/graphql/types/enums.ts

@@ -0,0 +1,22 @@
+import { registerEnumType } from "type-graphql";
+
+export enum MessageRole {
+  user = "user",
+  assistant = "assistant",
+  system = "system",
+}
+
+export enum ActionExecutionScope {
+  server = "server",
+  client = "client",
+}
+
+registerEnumType(MessageRole, {
+  name: "MessageRole",
+  description: "The role of the message",
+});
+
+registerEnumType(ActionExecutionScope, {
+  name: "ActionExecutionScope",
+  description: "The scope of the action",
+});

package/src/graphql/types/guardrails-result.type.ts

@@ -0,0 +1,20 @@
+import { Field, ObjectType, registerEnumType } from "type-graphql";
+
+export enum GuardrailsResultStatus {
+  ALLOWED = "allowed",
+  DENIED = "denied",
+}
+
+registerEnumType(GuardrailsResultStatus, {
+  name: "GuardrailsResultStatus",
+  description: "The status of the guardrails check",
+});
+
+@ObjectType()
+export class GuardrailsResult {
+  @Field(() => GuardrailsResultStatus)
+  status: GuardrailsResultStatus;
+
+  @Field(() => String, { nullable: true })
+  reason?: string;
+}

package/src/graphql/types/message-status.type.ts

@@ -0,0 +1,40 @@
+import { Field, ObjectType, createUnionType, registerEnumType } from "type-graphql";
+
+export enum MessageStatusCode {
+  Pending = "pending",
+  Success = "success",
+  Failed = "failed",
+}
+
+registerEnumType(MessageStatusCode, {
+  name: "MessageStatusCode",
+});
+
+@ObjectType()
+class BaseMessageStatus {
+  @Field(() => MessageStatusCode)
+  code: MessageStatusCode;
+}
+
+@ObjectType()
+export class PendingMessageStatus extends BaseMessageStatus {
+  code: MessageStatusCode = MessageStatusCode.Pending;
+}
+
+@ObjectType()
+export class SuccessMessageStatus extends BaseMessageStatus {
+  code: MessageStatusCode = MessageStatusCode.Success;
+}
+
+@ObjectType()
+export class FailedMessageStatus extends BaseMessageStatus {
+  code: MessageStatusCode = MessageStatusCode.Failed;
+
+  @Field(() => String)
+  reason: string;
+}
+
+export const MessageStatusUnion = createUnionType({
+  name: "MessageStatus",
+  types: () => [PendingMessageStatus, SuccessMessageStatus, FailedMessageStatus] as const,
+});

package/src/graphql/types/response-status.type.ts

@@ -0,0 +1,52 @@
+import { Field, InterfaceType, ObjectType, createUnionType, registerEnumType } from "type-graphql";
+
+export enum ResponseStatusCode {
+  Pending = "pending",
+  Success = "success",
+  Failed = "failed",
+}
+
+registerEnumType(ResponseStatusCode, {
+  name: "ResponseStatusCode",
+});
+
+@InterfaceType({
+  resolveType(value) {
+    if (value.code === ResponseStatusCode.Success) {
+      return SuccessResponseStatus;
+    } else if (value.code === ResponseStatusCode.Failed) {
+      return FailedResponseStatus;
+    } else if (value.code === ResponseStatusCode.Pending) {
+      return PendingResponseStatus;
+    }
+    return undefined;
+  },
+})
+@ObjectType()
+abstract class BaseResponseStatus {
+  @Field(() => ResponseStatusCode)
+  code: ResponseStatusCode;
+}
+
+@ObjectType({ implements: BaseResponseStatus })
+export class PendingResponseStatus extends BaseResponseStatus {
+  code: ResponseStatusCode = ResponseStatusCode.Pending;
+}
+
+@ObjectType({ implements: BaseResponseStatus })
+export class SuccessResponseStatus extends BaseResponseStatus {
+  code: ResponseStatusCode = ResponseStatusCode.Success;
+}
+
+@ObjectType({ implements: BaseResponseStatus })
+export class FailedResponseStatus extends BaseResponseStatus {
+  code: ResponseStatusCode = ResponseStatusCode.Failed;
+
+  @Field(() => String)
+  reason: string;
+}
+
+export const ResponseStatusUnion = createUnionType({
+  name: "ResponseStatus",
+  types: () => [PendingResponseStatus, SuccessResponseStatus, FailedResponseStatus] as const,
+});

package/src/index.ts

@@ -0,0 +1,2 @@
+import "reflect-metadata";
+export * from "./lib";

package/src/lib/copilot-cloud.ts

@@ -0,0 +1,63 @@
+import {
+  COPILOT_CLOUD_API_URL,
+  COPILOT_CLOUD_VERSION,
+  COPILOT_CLOUD_PUBLIC_API_KEY_HEADER,
+  CopilotCloudConfig,
+} from "@copilotkit/shared";
+
+export interface CloudCheckGuardrailsInputParams {
+  cloud: CopilotCloudConfig;
+  publicApiKey: string;
+  // TODO-PROTOCOL: switch to graphql
+  messages: any[];
+}
+
+export interface CloudCheckGuardrailsInputResponse {
+  status: "allowed" | "denied";
+  reason: string;
+}
+
+export abstract class CopilotCloud {
+  abstract checkGuardrailsInput(
+    params: CloudCheckGuardrailsInputParams,
+  ): Promise<CloudCheckGuardrailsInputResponse>;
+}
+
+export class RemoteCopilotCloud extends CopilotCloud {
+  constructor(private readonly cloudApiUrl: string = COPILOT_CLOUD_API_URL) {
+    super();
+  }
+
+  async checkGuardrailsInput({
+    cloud,
+    publicApiKey,
+    messages,
+  }: CloudCheckGuardrailsInputParams): Promise<CloudCheckGuardrailsInputResponse> {
+    const url = `${this.cloudApiUrl}/${COPILOT_CLOUD_VERSION}/copilotkit/guardrails/input`;
+
+    const response = await fetch(url, {
+      method: "PUT",
+      headers: {
+        "Content-Type": "application/json",
+        [COPILOT_CLOUD_PUBLIC_API_KEY_HEADER]: publicApiKey,
+      },
+      body: JSON.stringify({ messages, cloud }),
+    });
+
+    if (!response.ok) {
+      let message = response.statusText;
+
+      // try to parse the response body for a more detailed error message
+      try {
+        const json = await response.json();
+        if (json.message) {
+          message = json.message;
+        }
+      } catch (error) {}
+
+      throw new Error("Failed to check input guardrails: " + message);
+    }
+    const json = await response.json();
+    return { status: json.status, reason: json.reason };
+  }
+}

package/src/lib/copilot-runtime.ts

@@ -0,0 +1,261 @@
+/**
+ * Handles requests from frontend, provides function calling and various LLM backends.
+ *
+ * <img
+ *   referrerPolicy="no-referrer-when-downgrade"
+ *   src="https://static.scarf.sh/a.png?x-pxid=a9b290bb-38f9-4518-ac3b-8f54fdbf43be"
+ * />
+ *
+ * <RequestExample>
+ *   ```jsx CopilotRuntime Example
+ *   import {
+ *     CopilotRuntime,
+ *     OpenAIAdapter
+ *   } from "@copilotkit/runtime";
+ *
+ *   export async function POST(req: Request) {
+ *     const copilotKit = new CopilotRuntime();
+ *     return copilotKit.response(req, new OpenAIAdapter());
+ *   }
+ *
+ * ```
+ * </RequestExample>
+ *
+ * This class is the main entry point for the runtime. It handles requests from the frontend, provides function calling and various LLM backends.
+ *
+ * For example, to use OpenAI as a backend (check the [OpenAI Adapter](./OpenAIAdapter) docs for more info):
+ * ```typescript
+ * const copilotKit = new CopilotRuntime();
+ * return copilotKit.response(req, new OpenAIAdapter());
+ * ```
+ *
+ * Currently we support:
+ *
+ * - [OpenAI](./OpenAIAdapter)
+ * - [LangChain](./LangChainAdapter)
+ * - [OpenAI Assistant API](./OpenAIAssistantAdapter)
+ * - [Google Gemini](./GoogleGenerativeAIAdapter)
+ *
+ * ## Server Side Actions
+ *
+ * CopilotKit supports actions that can be executed on the server side. You can define server side actions by passing the `actions` parameter:
+ *
+ * ```typescript
+ * const copilotKit = new CopilotRuntime({
+ *   actions: [
+ *     {
+ *       name: "sayHello",
+ *       description: "Says hello to someone.",
+ *       argumentAnnotations: [
+ *         {
+ *           name: "arg",
+ *           type: "string",
+ *           description: "The name of the person to say hello to.",
+ *           required: true,
+ *         },
+ *       ],
+ *       implementation: async (arg) => {
+ *         console.log("Hello from the server", arg, "!");
+ *       },
+ *     },
+ *   ],
+ * });
+ * ```
+ *
+ * Server side actions can also return a result which becomes part of the message history.
+ *
+ * This is useful because it gives the LLM context about what happened on the server side. In addition,
+ * it can be used to look up information from a vector or relational database and other sources.
+ *
+ * In addition to that, server side actions can also come from LangChain, including support for streaming responses.
+ *
+ * Returned results can be of the following type:
+ *
+ * - anything serializable to JSON
+ * - `string`
+ * - LangChain types:
+ *   - `IterableReadableStream`
+ *   - `BaseMessageChunk`
+ *   - `AIMessage`
+ *
+ * ## LangServe
+ *
+ * The backend also supports LangServe, enabling you to connect to existing chains, for example python based chains.
+ * Use the `langserve` parameter to specify URLs for LangServe.
+ *
+ * ```typescript
+ * const copilotKit = new CopilotRuntime({
+ *   langserve: [
+ *     {
+ *       chainUrl: "http://my-langserve.chain",
+ *       name: "performResearch",
+ *       description: "Performs research on a given topic.",
+ *     },
+ *   ],
+ * });
+ * ```
+ *
+ * When left out, arguments are automatically inferred from the schema provided by LangServe.
+ */
+
+import { Action, actionParametersToJsonSchema, Parameter } from "@copilotkit/shared";
+import { RemoteChain, CopilotServiceAdapter } from "../service-adapters";
+import { CopilotCloud, RemoteCopilotCloud } from "./copilot-cloud";
+import { MessageInput } from "../graphql/inputs/message.input";
+import { ActionInput } from "../graphql/inputs/action.input";
+import { RuntimeEventSource } from "../service-adapters/events";
+import { convertGqlInputToMessages } from "../service-adapters/conversion";
+
+interface CopilotRuntimeRequest {
+  serviceAdapter: CopilotServiceAdapter;
+  messages: MessageInput[];
+  actions: ActionInput[];
+  threadId?: string;
+  runId?: string;
+  publicApiKey?: string;
+}
+
+interface CopilotRuntimeResponse {
+  threadId?: string;
+  runId?: string;
+  eventSource: RuntimeEventSource;
+}
+
+export interface CopilotRuntimeConstructorParams<T extends Parameter[] | [] = []> {
+  /*
+   * A list of server side actions that can be executed.
+   */
+  actions?: Action<T>[];
+
+  /*
+   * An array of LangServer URLs.
+   */
+  langserve?: RemoteChain[];
+
+  debug?: boolean;
+  copilotCloud?: CopilotCloud;
+}
+
+export class CopilotRuntime<const T extends Parameter[] | [] = []> {
+  public actions: Action<any>[] = [];
+  private langserve: Promise<Action<any>>[] = [];
+  private debug: boolean = false;
+  private copilotCloud: CopilotCloud;
+
+  constructor(params?: CopilotRuntimeConstructorParams<T>) {
+    this.actions = params?.actions || [];
+
+    for (const chain of params?.langserve || []) {
+      const remoteChain = new RemoteChain(chain);
+      this.langserve.push(remoteChain.toAction());
+    }
+    this.debug = params?.debug || false;
+    this.copilotCloud = params?.copilotCloud || new RemoteCopilotCloud();
+  }
+
+  addAction<const T extends Parameter[] | [] = []>(action: Action<T>): void {
+    this.removeAction(action.name);
+    this.actions.push(action);
+  }
+
+  removeAction(actionName: string): void {
+    this.actions = this.actions.filter((f) => f.name !== actionName);
+  }
+
+  async process({
+    serviceAdapter,
+    messages,
+    actions: clientSideActionsInput,
+    threadId,
+    runId,
+    publicApiKey,
+  }: CopilotRuntimeRequest): Promise<CopilotRuntimeResponse> {
+    // TODO-PROTOCOL: cloud configuration
+    // const cloud: CopilotCloudConfig = forwardedProps.cloud;
+
+    const langserveFunctions: Action<any>[] = [];
+
+    for (const chainPromise of this.langserve) {
+      try {
+        const chain = await chainPromise;
+        langserveFunctions.push(chain);
+      } catch (error) {
+        console.error("Error loading langserve chain:", error);
+      }
+    }
+
+    const serverSideActionsInput: ActionInput[] = [...this.actions, ...langserveFunctions].map(
+      (action) => ({
+        name: action.name,
+        description: action.description,
+        jsonSchema: JSON.stringify(actionParametersToJsonSchema(action.parameters)),
+      }),
+    );
+
+    const actions = flattenToolCallsNoDuplicates([
+      ...serverSideActionsInput,
+      ...clientSideActionsInput,
+    ]);
+
+    try {
+      const eventSource = new RuntimeEventSource();
+      // TODO-PROTOCOL: type this and support function calls
+      const result = await serviceAdapter.process({
+        messages: convertGqlInputToMessages(messages),
+        actions,
+        threadId,
+        runId,
+        eventSource,
+      });
+
+      // TODO-PROTOCOL add guardrails
+      //
+      // if (publicApiKey !== undefined) {
+      //   // wait for the cloud log chat to finish before streaming back the response
+      //   try {
+      //     const checkGuardrailsInputResult = await this.copilotCloud.checkGuardrailsInput({
+      //       cloud,
+      //       publicApiKey,
+      //       messages: forwardedProps.messages || [],
+      //     });
+
+      //     if (checkGuardrailsInputResult.status === "denied") {
+      //       // the chat was denied. instead of streaming back the response,
+      //       // we let the client know...
+      //       return {
+      //         stream: new SingleChunkReadableStream(checkGuardrailsInputResult.reason),
+      //         // headers: result.headers,
+      //       };
+      //     }
+      //   } catch (error) {
+      //     console.error("Error checking guardrails:", error);
+      //   }
+      // }
+      return {
+        threadId: result.threadId,
+        runId: result.runId,
+        eventSource,
+      };
+    } catch (error) {
+      console.error("Error getting response:", error);
+      throw error;
+    }
+  }
+}
+
+export function flattenToolCallsNoDuplicates(toolsByPriority: ActionInput[]): ActionInput[] {
+  let allTools: ActionInput[] = [];
+  const allToolNames: string[] = [];
+  for (const tool of toolsByPriority) {
+    if (!allToolNames.includes(tool.name)) {
+      allTools.push(tool);
+      allToolNames.push(tool.name);
+    }
+  }
+  return allTools;
+}
+
+/**
+ * @deprecated use CopilotRuntime instead
+ */
+export class CopilotBackend extends CopilotRuntime {}

package/src/lib/guardrails.ts

@@ -0,0 +1,3 @@
+export interface GuardrailsOptions {
+  baseUrl: string;
+}

package/src/lib/index.ts

@@ -0,0 +1,7 @@
+export * from "./copilot-runtime";
+export * from "../service-adapters/openai/openai-adapter";
+export * from "../service-adapters/langchain/langchain-adapter";
+export * from "../service-adapters/google/google-genai-adapter";
+export * from "../service-adapters/openai/openai-assistant-adapter";
+export * from "./copilot-cloud";
+export * from "./integrations";

package/src/lib/integrations/index.ts

@@ -0,0 +1,4 @@
+export * from "./shared";
+export * from "./nextjs/app-router";
+export * from "./nextjs/pages-router";
+export * from "./node-http";

package/src/lib/integrations/nextjs/app-router.ts

@@ -0,0 +1,29 @@
+import { createYoga } from "graphql-yoga";
+import { getCommonConfig } from "../shared";
+import { CopilotRuntime } from "../../copilot-runtime";
+import { CopilotServiceAdapter } from "../../../service-adapters";
+
+export function copilotRuntimeNextJSAppRouterEndpoint({
+  runtime,
+  endpoint,
+  serviceAdapter,
+}: {
+  runtime: CopilotRuntime;
+  serviceAdapter: CopilotServiceAdapter;
+  endpoint: string;
+}) {
+  const commonConfig = getCommonConfig({ runtime, serviceAdapter });
+
+  const yoga = createYoga({
+    ...commonConfig,
+    graphqlEndpoint: endpoint,
+    fetchAPI: { Response: globalThis.Response },
+  });
+
+  return {
+    handleRequest: yoga,
+    GET: yoga,
+    POST: yoga,
+    OPTIONS: yoga,
+  };
+}

package/src/lib/integrations/nextjs/pages-router.ts

@@ -0,0 +1,36 @@
+import { YogaServerInstance, createYoga } from "graphql-yoga";
+import { GraphQLContext, getCommonConfig } from "../shared";
+import { CopilotRuntime } from "../../copilot-runtime";
+import { CopilotServiceAdapter } from "../../../service-adapters";
+
+export const config = {
+  api: {
+    bodyParser: false,
+  },
+};
+
+export type CopilotRuntimeServerInstance<T> = YogaServerInstance<T, Partial<GraphQLContext>>;
+
+// Theis import is needed to fix the type error
+// Fix is currently in TypeScript 5.5 beta, waiting for stable version
+// https://github.com/microsoft/TypeScript/issues/42873#issuecomment-2066874644
+export type {} from "@whatwg-node/server";
+
+export function copilotRuntimeNextJSPagesRouterEndpoint({
+  runtime,
+  endpoint,
+  serviceAdapter,
+}: {
+  runtime: CopilotRuntime;
+  serviceAdapter: CopilotServiceAdapter;
+  endpoint: string;
+}): CopilotRuntimeServerInstance<GraphQLContext> {
+  const commonConfig = getCommonConfig({ runtime, serviceAdapter });
+
+  const yoga = createYoga({
+    ...commonConfig,
+    graphqlEndpoint: endpoint,
+  });
+
+  return yoga;
+}

package/src/lib/integrations/node-http/index.ts

@@ -0,0 +1,23 @@
+import { createYoga } from "graphql-yoga";
+import { getCommonConfig } from "../shared";
+import { CopilotRuntime } from "../../copilot-runtime";
+import { CopilotServiceAdapter } from "../../../service-adapters";
+
+export function copilotRuntimeNodeHttpEndpoint({
+  runtime,
+  endpoint,
+  serviceAdapter,
+}: {
+  runtime: CopilotRuntime;
+  serviceAdapter: CopilotServiceAdapter;
+  endpoint: string;
+}) {
+  const commonConfig = getCommonConfig({ runtime, serviceAdapter });
+
+  const yoga = createYoga({
+    ...commonConfig,
+    graphqlEndpoint: endpoint,
+  });
+
+  return yoga;
+}