@copilotkit/runtime 0.38.0-mme-alpha.1 → 1.0.0-beta.1

package/src/graphql/resolvers/copilot.resolver.ts

@@ -26,13 +26,15 @@ import {
 import { ResponseStatusUnion, SuccessResponseStatus } from "../types/response-status.type";
 import { GraphQLJSONObject } from "graphql-scalars";
 import { plainToInstance } from "class-transformer";
-import { GuardrailsResult, GuardrailsResultStatus } from "../types/guardrails-result.type";
+import { GuardrailsResult } from "../types/guardrails-result.type";
 import { GraphQLError } from "graphql";
 import {
   GuardrailsValidationFailureResponse,
   MessageStreamInterruptedResponse,
   UnknownErrorResponse,
 } from "../../utils";
+import { ActionExecutionMessage, Message, ResultMessage, TextMessage } from "../types/converted";
+import telemetry from "../../lib/telemetry-client";
 
 const invokeGuardrails = async ({
   baseUrl,
@@ -47,6 +49,8 @@ const invokeGuardrails = async ({
   onResult: (result: GuardrailsResult) => void;
   onError: (err: Error) => void;
 }) => {
+  console.log("invokeGuardrails.baseUrl", baseUrl);
+
   if (
     data.messages.length &&
     data.messages[data.messages.length - 1].textMessage?.role === MessageRole.user
@@ -104,6 +108,11 @@ export class CopilotResolver {
     @Arg("properties", () => GraphQLJSONObject, { nullable: true })
     properties?: CopilotRequestContextProperties,
   ) {
+    telemetry.capture("oss.runtime.copilot_request_created", {
+      "cloud.guardrails.enabled": data.cloud?.guardrails !== undefined,
+      requestType: data.metadata.requestType,
+    });
+
     let logger = ctx.logger.child({ component: "CopilotResolver.generateCopilotResponse" });
     logger.debug({ data }, "Generating Copilot response");
 
@@ -132,19 +141,29 @@ export class CopilotResolver {
 
       if (process.env.COPILOT_CLOUD_BASE_URL) {
         copilotCloudBaseUrl = process.env.COPILOT_CLOUD_BASE_URL;
-      } else if (ctx._copilotkit.baseUrl) {
-        copilotCloudBaseUrl = ctx._copilotkit.baseUrl;
+      } else if (ctx._copilotkit.cloud?.baseUrl) {
+        copilotCloudBaseUrl = ctx._copilotkit.cloud?.baseUrl;
       } else {
         copilotCloudBaseUrl = "https://api.cloud.copilotkit.ai";
       }
 
       logger = logger.child({ copilotCloudBaseUrl });
     }
+
     logger.debug("Setting up subjects");
     const responseStatus$ = new ReplaySubject<typeof ResponseStatusUnion>();
     const interruptStreaming$ = new ReplaySubject<{ reason: string; messageId?: string }>();
     const guardrailsResult$ = new ReplaySubject<GuardrailsResult>();
 
+    let outputMessages: Message[] = [];
+    let resolveOutputMessagesPromise: (messages: Message[]) => void;
+    let rejectOutputMessagesPromise: (err: Error) => void;
+
+    const outputMessagesPromise = new Promise<Message[]>((resolve, reject) => {
+      resolveOutputMessagesPromise = resolve;
+      rejectOutputMessagesPromise = reject;
+    });
+
     logger.debug("Processing");
     const {
       eventSource,
@@ -158,6 +177,8 @@ export class CopilotResolver {
       threadId: data.threadId,
       runId: data.runId,
       publicApiKey: undefined,
+      properties: ctx.properties || {},
+      outputMessagesPromise,
     });
 
     logger.debug("Event source created, creating response");
@@ -180,13 +201,27 @@ export class CopilotResolver {
             onResult: (result) => {
               logger.debug({ status: result.status }, "Guardrails validation done");
               guardrailsResult$.next(result);
+
+              // Guardrails validation failed
               if (result.status === "denied") {
+                // send the reason to the client and interrupt streaming
                 responseStatus$.next(
                   new GuardrailsValidationFailureResponse({ guardrailsReason: result.reason }),
                 );
                 interruptStreaming$.next({
                   reason: `Interrupted due to Guardrails validation failure. Reason: ${result.reason}`,
                 });
+
+                // resolve messages promise to the middleware
+                outputMessages = [
+                  plainToInstance(TextMessage, {
+                    id: nanoid(),
+                    createdAt: new Date(),
+                    content: result.reason,
+                    role: MessageRole.assistant,
+                  }),
+                ];
+                resolveOutputMessagesPromise(outputMessages);
               }
             },
             onError: (err) => {
@@ -199,6 +234,9 @@ export class CopilotResolver {
               interruptStreaming$.next({
                 reason: `Interrupted due to unknown error in guardrails validation`,
               });
+
+              // reject the middleware promise
+              rejectOutputMessagesPromise(err);
             },
           });
         }
@@ -251,6 +289,7 @@ export class CopilotResolver {
                   content: new Repeater(async (pushTextChunk, stopStreamingText) => {
                     logger.debug("Text message content repeater created");
 
+                    const textChunks: string[] = [];
                     let textSubscription: Subscription;
 
                     interruptStreaming$
@@ -277,6 +316,7 @@ export class CopilotResolver {
                       next: async (e: RuntimeEvent) => {
                         if (e.type == RuntimeEventTypes.TextMessageContent) {
                           await pushTextChunk(e.content);
+                          textChunks.push(e.content);
                         }
                       },
                       error: (err) => {
@@ -293,6 +333,15 @@ export class CopilotResolver {
                         streamingTextStatus.next(new SuccessMessageStatus());
                         stopStreamingText();
                         textSubscription?.unsubscribe();
+
+                        outputMessages.push(
+                          plainToInstance(TextMessage, {
+                            id: messageId,
+                            createdAt: new Date(),
+                            content: textChunks.join(""),
+                            role: MessageRole.assistant,
+                          }),
+                        );
                       },
                     });
                   }),
@@ -317,11 +366,14 @@ export class CopilotResolver {
                   arguments: new Repeater(async (pushArgumentsChunk, stopStreamingArguments) => {
                     logger.debug("Action execution argument stream created");
 
+                    const argumentChunks: string[] = [];
                     let actionExecutionArgumentSubscription: Subscription;
+
                     actionExecutionArgumentSubscription = actionExecutionArgumentStream.subscribe({
                       next: async (e: RuntimeEvent) => {
                         if (e.type == RuntimeEventTypes.ActionExecutionArgs) {
                           await pushArgumentsChunk(e.args);
+                          argumentChunks.push(e.args);
                         }
                       },
                       error: (err) => {
@@ -340,6 +392,16 @@ export class CopilotResolver {
                         streamingArgumentsStatus.next(new SuccessMessageStatus());
                         stopStreamingArguments();
                         actionExecutionArgumentSubscription?.unsubscribe();
+
+                        outputMessages.push(
+                          plainToInstance(ActionExecutionMessage, {
+                            id: event.actionExecutionId,
+                            createdAt: new Date(),
+                            name: event.actionName,
+                            scope: event.scope!,
+                            arguments: argumentChunks.join(""),
+                          }),
+                        );
                       },
                     });
                   }),
@@ -358,6 +420,16 @@ export class CopilotResolver {
                   actionName: event.actionName,
                   result: event.result,
                 });
+
+                outputMessages.push(
+                  plainToInstance(ResultMessage, {
+                    id: nanoid(),
+                    createdAt: new Date(),
+                    actionExecutionId: event.actionExecutionId,
+                    actionName: event.actionName,
+                    result: event.result,
+                  }),
+                );
                 break;
             }
           },
@@ -370,6 +442,8 @@ export class CopilotResolver {
             );
             eventStreamSubscription?.unsubscribe();
             stopStreamingMessages();
+
+            rejectOutputMessagesPromise(err);
           },
           complete: async () => {
             logger.debug("Event stream completed");
@@ -380,6 +454,8 @@ export class CopilotResolver {
             responseStatus$.next(new SuccessResponseStatus());
             eventStreamSubscription?.unsubscribe();
             stopStreamingMessages();
+
+            resolveOutputMessagesPromise(outputMessages);
           },
         });
       }),

package/src/graphql/types/enums.ts

@@ -11,6 +11,14 @@ export enum ActionExecutionScope {
   client = "client",
 }
 
+export enum CopilotRequestType {
+  Chat = "Chat",
+  Task = "Task",
+  TextareaCompletion = "TextareaCompletion",
+  TextareaPopover = "TextareaPopover",
+  Suggestion = "Suggestion",
+}
+
 registerEnumType(MessageRole, {
   name: "MessageRole",
   description: "The role of the message",
@@ -20,3 +28,8 @@ registerEnumType(ActionExecutionScope, {
   name: "ActionExecutionScope",
   description: "The scope of the action",
 });
+
+registerEnumType(CopilotRequestType, {
+  name: "CopilotRequestType",
+  description: "The type of Copilot request",
+});

package/src/lib/copilot-runtime.ts

@@ -104,41 +104,102 @@ import { MessageInput } from "../graphql/inputs/message.input";
 import { ActionInput } from "../graphql/inputs/action.input";
 import { RuntimeEventSource } from "../service-adapters/events";
 import { convertGqlInputToMessages } from "../service-adapters/conversion";
+import { Message } from "../graphql/types/converted";
 
 interface CopilotRuntimeRequest {
   serviceAdapter: CopilotServiceAdapter;
   messages: MessageInput[];
   actions: ActionInput[];
+  outputMessagesPromise: Promise<Message[]>;
+  properties: any;
   threadId?: string;
   runId?: string;
   publicApiKey?: string;
 }
 
 interface CopilotRuntimeResponse {
-  threadId?: string;
+  threadId: string;
   runId?: string;
   eventSource: RuntimeEventSource;
   actions: Action<any>[];
 }
 
+type ActionsConfiguration<T extends Parameter[] | [] = []> =
+  | Action<T>[]
+  | ((ctx: { properties: any }) => Action<T>[]);
+
+interface OnBeforeRequestOptions {
+  threadId?: string;
+  runId?: string;
+  inputMessages: Message[];
+  properties: any;
+}
+
+type OnBeforeRequestHandler = (options: OnBeforeRequestOptions) => void | Promise<void>;
+
+interface OnAfterRequestOptions {
+  threadId: string;
+  runId?: string;
+  inputMessages: Message[];
+  outputMessages: Message[];
+  properties: any;
+}
+
+type OnAfterRequestHandler = (options: OnAfterRequestOptions) => void | Promise<void>;
+
+interface Middleware {
+  /**
+   * A function that is called before the request is processed.
+   */
+  onBeforeRequest?: OnBeforeRequestHandler;
+
+  /**
+   * A function that is called after the request is processed.
+   */
+  onAfterRequest?: OnAfterRequestHandler;
+}
+
 export interface CopilotRuntimeConstructorParams<T extends Parameter[] | [] = []> {
+  /**
+   * Middleware to be used by the runtime.
+   *
+   * ```ts
+   * onBeforeRequest: (options: {
+   *   threadId?: string;
+   *   runId?: string;
+   *   inputMessages: Message[];
+   *   properties: any;
+   * }) => void | Promise<void>;
+   * ```
+   *
+   * ```ts
+   * onAfterRequest: (options: {
+   *   threadId?: string;
+   *   runId?: string;
+   *   inputMessages: Message[];
+   *   outputMessages: Message[];
+   *   properties: any;
+   * }) => void | Promise<void>;
+   * ```
+   */
+  middleware?: Middleware;
+
   /*
    * A list of server side actions that can be executed.
    */
-  actions?: Action<T>[];
+  actions?: ActionsConfiguration<T>;
 
   /*
    * An array of LangServer URLs.
    */
   langserve?: RemoteChainParameters[];
-
-  debug?: boolean;
 }
 
 export class CopilotRuntime<const T extends Parameter[] | [] = []> {
-  public actions: Action<any>[] = [];
+  public actions: ActionsConfiguration<T>;
   private langserve: Promise<Action<any>>[] = [];
-  private debug: boolean = false;
+  private onBeforeRequest?: OnBeforeRequestHandler;
+  private onAfterRequest?: OnAfterRequestHandler;
 
   constructor(params?: CopilotRuntimeConstructorParams<T>) {
     this.actions = params?.actions || [];
@@ -147,26 +208,21 @@ export class CopilotRuntime<const T extends Parameter[] | [] = []> {
       const remoteChain = new RemoteChain(chain);
       this.langserve.push(remoteChain.toAction());
     }
-    this.debug = params?.debug || false;
-  }
-
-  addAction<const T extends Parameter[] | [] = []>(action: Action<T>): void {
-    this.removeAction(action.name);
-    this.actions.push(action);
-  }
 
-  removeAction(actionName: string): void {
-    this.actions = this.actions.filter((f) => f.name !== actionName);
+    this.onBeforeRequest = params?.middleware?.onBeforeRequest;
+    this.onAfterRequest = params?.middleware?.onAfterRequest;
   }
 
-  async process({
-    serviceAdapter,
-    messages,
-    actions: clientSideActionsInput,
-    threadId,
-    runId,
-    publicApiKey,
-  }: CopilotRuntimeRequest): Promise<CopilotRuntimeResponse> {
+  async process(request: CopilotRuntimeRequest): Promise<CopilotRuntimeResponse> {
+    const {
+      serviceAdapter,
+      messages,
+      actions: clientSideActionsInput,
+      threadId,
+      runId,
+      properties,
+      outputMessagesPromise,
+    } = request;
     const langserveFunctions: Action<any>[] = [];
 
     for (const chainPromise of this.langserve) {
@@ -178,7 +234,10 @@ export class CopilotRuntime<const T extends Parameter[] | [] = []> {
       }
     }
 
-    const actions = [...this.actions, ...langserveFunctions];
+    const configuredActions =
+      typeof this.actions === "function" ? this.actions({ properties }) : this.actions;
+
+    const actions = [...configuredActions, ...langserveFunctions];
 
     const serverSideActionsInput: ActionInput[] = actions.map((action) => ({
       name: action.name,
@@ -190,23 +249,43 @@ export class CopilotRuntime<const T extends Parameter[] | [] = []> {
       ...serverSideActionsInput,
       ...clientSideActionsInput,
     ]);
+    const inputMessages = convertGqlInputToMessages(messages);
+
+    await this.onBeforeRequest?.({
+      threadId,
+      runId,
+      inputMessages,
+      properties,
+    });
 
     try {
       const eventSource = new RuntimeEventSource();
-      // TODO-PROTOCOL: type this and support function calls
+
       const result = await serviceAdapter.process({
-        messages: convertGqlInputToMessages(messages),
+        messages: inputMessages,
         actions: actionInputs,
         threadId,
         runId,
         eventSource,
       });
 
+      outputMessagesPromise
+        .then((outputMessages) => {
+          this.onAfterRequest?.({
+            threadId: result.threadId,
+            runId: result.runId,
+            inputMessages,
+            outputMessages,
+            properties,
+          });
+        })
+        .catch((_error) => {});
+
       return {
         threadId: result.threadId,
         runId: result.runId,
         eventSource,
-        actions,
+        actions: actions,
       };
     } catch (error) {
       console.error("Error getting response:", error);
@@ -226,8 +305,3 @@ export function flattenToolCallsNoDuplicates(toolsByPriority: ActionInput[]): Ac
   }
   return allTools;
 }
-
-/**
- * @deprecated use CopilotRuntime instead
- */
-export class CopilotBackend extends CopilotRuntime {}

package/src/lib/integrations/index.ts

@@ -2,3 +2,5 @@ export * from "./shared";
 export * from "./nextjs/app-router";
 export * from "./nextjs/pages-router";
 export * from "./node-http";
+export * from "./node-express";
+export * from "./nest";

package/src/lib/integrations/nest/index.ts

@@ -0,0 +1,14 @@
+import { CreateCopilotRuntimeServerOptions } from "../shared";
+import { copilotRuntimeNodeHttpEndpoint } from "../node-http";
+import telemetry from "../../telemetry-client";
+
+export function copilotRuntimeNestEndpoint(options: CreateCopilotRuntimeServerOptions) {
+  telemetry.setGlobalProperties({
+    runtime: {
+      framework: "nest",
+    },
+  });
+
+  telemetry.capture("oss.runtime.instance_created", {});
+  return copilotRuntimeNodeHttpEndpoint(options);
+}

package/src/lib/integrations/nextjs/app-router.ts

@@ -1,8 +1,18 @@
 import { createYoga } from "graphql-yoga";
 import { CreateCopilotRuntimeServerOptions, getCommonConfig } from "../shared";
+import telemetry from "../../telemetry-client";
 
 export function copilotRuntimeNextJSAppRouterEndpoint(options: CreateCopilotRuntimeServerOptions) {
   const commonConfig = getCommonConfig(options);
+
+  telemetry.setGlobalProperties({
+    runtime: {
+      framework: "nextjs-app-router",
+    },
+  });
+
+  telemetry.capture("oss.runtime.instance_created", {});
+
   const logger = commonConfig.logging;
   logger.debug("Creating NextJS App Router endpoint");
 

package/src/lib/integrations/nextjs/pages-router.ts

@@ -1,5 +1,6 @@
 import { YogaServerInstance, createYoga } from "graphql-yoga";
 import { CreateCopilotRuntimeServerOptions, GraphQLContext, getCommonConfig } from "../shared";
+import telemetry from "../../telemetry-client";
 
 export const config = {
   api: {
@@ -18,6 +19,15 @@ export function copilotRuntimeNextJSPagesRouterEndpoint(
   options: CreateCopilotRuntimeServerOptions,
 ): CopilotRuntimeServerInstance<GraphQLContext> {
   const commonConfig = getCommonConfig(options);
+
+  telemetry.setGlobalProperties({
+    runtime: {
+      framework: "nextjs-pages-router",
+    },
+  });
+
+  telemetry.capture("oss.runtime.instance_created", {});
+
   const logger = commonConfig.logging;
   logger.debug("Creating NextJS Pages Router endpoint");
 

package/src/lib/integrations/node-express/index.ts

@@ -0,0 +1,14 @@
+import { CreateCopilotRuntimeServerOptions } from "../shared";
+import { copilotRuntimeNodeHttpEndpoint } from "../node-http";
+import telemetry from "../../telemetry-client";
+
+export function copilotRuntimeNodeExpressEndpoint(options: CreateCopilotRuntimeServerOptions) {
+  telemetry.setGlobalProperties({
+    runtime: {
+      framework: "node-express",
+    },
+  });
+
+  telemetry.capture("oss.runtime.instance_created", {});
+  return copilotRuntimeNodeHttpEndpoint(options);
+}

package/src/lib/integrations/node-http/index.ts

@@ -1,8 +1,18 @@
 import { createYoga } from "graphql-yoga";
 import { CreateCopilotRuntimeServerOptions, getCommonConfig } from "../shared";
+import telemetry from "../../telemetry-client";
 
 export function copilotRuntimeNodeHttpEndpoint(options: CreateCopilotRuntimeServerOptions) {
   const commonConfig = getCommonConfig(options);
+
+  telemetry.setGlobalProperties({
+    runtime: {
+      framework: "node-http",
+    },
+  });
+
+  telemetry.capture("oss.runtime.instance_created", {});
+
   const logger = commonConfig.logging;
   logger.debug("Creating Node HTTP endpoint");
 

package/src/lib/integrations/shared.ts

@@ -7,6 +7,7 @@ import { CopilotServiceAdapter } from "../../service-adapters";
 import { CopilotCloudOptions } from "../cloud";
 import { LogLevel, createLogger } from "../../lib/logger";
 import { createYoga } from "graphql-yoga";
+import telemetry from "../telemetry-client";
 
 const logger = createLogger();
 
@@ -23,7 +24,7 @@ export type GraphQLContext = YogaInitialContext & {
 };
 
 export interface CreateCopilotRuntimeServerOptions {
-  runtime: CopilotRuntime;
+  runtime: CopilotRuntime<any>;
   serviceAdapter: CopilotServiceAdapter;
   endpoint: string;
   baseUrl?: string;
@@ -74,10 +75,22 @@ export type CommonConfig = {
 export function getCommonConfig(options: CreateCopilotRuntimeServerOptions): CommonConfig {
   const logLevel = (process.env.LOG_LEVEL as LogLevel) || (options.logLevel as LogLevel) || "error";
   const logger = createLogger({ level: logLevel, component: "getCommonConfig" });
-  logger.debug("Getting common config");
 
   const contextLogger = createLogger({ level: logLevel });
 
+  if (options.cloud) {
+    telemetry.setCloudConfiguration({
+      publicApiKey: options.cloud.publicApiKey,
+      baseUrl: options.cloud.baseUrl,
+    });
+  }
+
+  telemetry.setGlobalProperties({
+    runtime: {
+      serviceAdapter: options.serviceAdapter.constructor.name,
+    },
+  });
+
   return {
     logging: createLogger({ component: "Yoga GraphQL", level: logLevel }),
     schema: buildSchema(),

package/src/lib/telemetry-client.ts

@@ -0,0 +1,9 @@
+import { TelemetryClient } from "@copilotkit/shared";
+const packageJson = require("../../package.json");
+
+const telemetryClient = new TelemetryClient({
+  packageName: packageJson.name,
+  packageVersion: packageJson.version,
+});
+
+export default telemetryClient;

package/src/service-adapters/experimental/groq/groq-adapter.ts

@@ -107,7 +107,9 @@ export class ExperimentalGroqAdapter implements CopilotServiceAdapter {
 
       eventStream$.complete();
     });
-    return {};
+    return {
+      threadId: request.threadId || nanoid(),
+    };
   }
 }
 

package/src/service-adapters/experimental/ollama/ollama-adapter.ts

@@ -70,6 +70,8 @@ export class ExperimentalOllamaAdapter implements CopilotServiceAdapter {
 
       eventStream$.complete();
     });
-    return {};
+    return {
+      threadId: request.threadId || nanoid(),
+    };
   }
 }

package/src/service-adapters/google/google-genai-adapter.ts

@@ -130,7 +130,9 @@ export class GoogleGenerativeAIAdapter implements CopilotServiceAdapter {
       eventStream$.complete();
     });
 
-    return {};
+    return {
+      threadId: request.threadId || nanoid(),
+    };
   }
 }
 

package/src/service-adapters/langchain/langchain-adapter.ts

@@ -35,6 +35,7 @@ import {
 } from "./utils";
 import { DynamicStructuredTool } from "@langchain/core/tools";
 import { LangChainReturnType } from "./types";
+import { nanoid } from "nanoid";
 
 interface ChainFnParameters {
   model: string;
@@ -54,14 +55,10 @@ export class LangChainAdapter implements CopilotServiceAdapter {
    */
   constructor(private options: LangChainAdapterOptions) {}
 
-  async process({
-    eventSource,
-    model,
-    actions,
-    messages,
-    threadId,
-    runId,
-  }: CopilotRuntimeChatCompletionRequest): Promise<CopilotRuntimeChatCompletionResponse> {
+  async process(
+    request: CopilotRuntimeChatCompletionRequest,
+  ): Promise<CopilotRuntimeChatCompletionResponse> {
+    const { eventSource, model, actions, messages, threadId, runId } = request;
     const result = await this.options.chainFn({
       messages: messages.map(convertMessageToLangChainMessage),
       tools: actions.map(convertActionInputToLangChainTool),
@@ -77,6 +74,8 @@ export class LangChainAdapter implements CopilotServiceAdapter {
       });
     });
 
-    return {};
+    return {
+      threadId: threadId || nanoid(),
+    };
   }
 }