@copilotkit/runtime 0.37.0 → 0.38.0-beta.0

package/src/graphql/resolvers/copilot.resolver.ts

@@ -1,5 +1,16 @@
 import { Arg, Ctx, Mutation, Query, Resolver } from "type-graphql";
-import { Subject, firstValueFrom, shareReplay, skipWhile, takeWhile } from "rxjs";
+import {
+  ReplaySubject,
+  Subject,
+  Subscription,
+  finalize,
+  firstValueFrom,
+  shareReplay,
+  skipWhile,
+  take,
+  takeWhile,
+  tap,
+} from "rxjs";
 import { GenerateCopilotResponseInput } from "../inputs/generate-copilot-response.input";
 import { CopilotResponse } from "../types/copilot-response.type";
 import { MessageRole } from "../types/enums";
@@ -7,9 +18,78 @@ import { Repeater } from "graphql-yoga";
 import type { CopilotRequestContextProperties, GraphQLContext } from "../../lib/integrations";
 import { nanoid } from "nanoid";
 import { RuntimeEvent, RuntimeEventTypes } from "../../service-adapters/events";
-import { MessageStatusUnion, SuccessMessageStatus } from "../types/message-status.type";
+import {
+  FailedMessageStatus,
+  MessageStatusUnion,
+  SuccessMessageStatus,
+} from "../types/message-status.type";
 import { ResponseStatusUnion, SuccessResponseStatus } from "../types/response-status.type";
 import { GraphQLJSONObject } from "graphql-scalars";
+import { plainToInstance } from "class-transformer";
+import { GuardrailsResult, GuardrailsResultStatus } from "../types/guardrails-result.type";
+import { GraphQLError } from "graphql";
+import {
+  GuardrailsValidationFailureResponse,
+  MessageStreamInterruptedResponse,
+  UnknownErrorResponse,
+} from "../../utils";
+import { ActionExecutionMessage, Message, ResultMessage, TextMessage } from "../types/converted";
+
+const invokeGuardrails = async ({
+  baseUrl,
+  copilotCloudPublicApiKey,
+  data,
+  onResult,
+  onError,
+}: {
+  baseUrl: string;
+  copilotCloudPublicApiKey: string;
+  data: GenerateCopilotResponseInput;
+  onResult: (result: GuardrailsResult) => void;
+  onError: (err: Error) => void;
+}) => {
+  if (
+    data.messages.length &&
+    data.messages[data.messages.length - 1].textMessage?.role === MessageRole.user
+  ) {
+    const messages = data.messages
+      .filter(
+        (m) =>
+          m.textMessage !== undefined &&
+          (m.textMessage.role === MessageRole.user || m.textMessage.role === MessageRole.assistant),
+      )
+      .map((m) => ({
+        role: m.textMessage!.role,
+        content: m.textMessage.content,
+      }));
+
+    const lastMessage = messages[messages.length - 1];
+    const restOfMessages = messages.slice(0, -1);
+
+    const body = {
+      input: lastMessage.content,
+      validTopics: data.cloud.guardrails.inputValidationRules.allowList,
+      invalidTopics: data.cloud.guardrails.inputValidationRules.denyList,
+      messages: restOfMessages,
+    };
+
+    const guardrailsResult = await fetch(`${baseUrl}/guardrails/validate`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "X-CopilotCloud-Public-API-Key": copilotCloudPublicApiKey,
+      },
+      body: JSON.stringify(body),
+    });
+
+    if (guardrailsResult.ok) {
+      const resultJson: GuardrailsResult = await guardrailsResult.json();
+      onResult(resultJson);
+    } else {
+      onError(await guardrailsResult.json());
+    }
+  }
+};
 
 @Resolver(() => CopilotResponse)
 export class CopilotResolver {
@@ -25,17 +105,62 @@ export class CopilotResolver {
     @Arg("properties", () => GraphQLJSONObject, { nullable: true })
     properties?: CopilotRequestContextProperties,
   ) {
-    if (properties) {
-      ctx._copilotkit.properties = { ...ctx._copilotkit.properties, ...properties };
-    }
+    let logger = ctx.logger.child({ component: "CopilotResolver.generateCopilotResponse" });
+    logger.debug({ data }, "Generating Copilot response");
+
     const copilotRuntime = ctx._copilotkit.runtime;
     const serviceAdapter = ctx._copilotkit.serviceAdapter;
-    const responseStatus = new Subject<typeof ResponseStatusUnion>();
 
+    if (properties) {
+      logger.debug("Properties provided, merging with context properties");
+      ctx.properties = { ...ctx.properties, ...properties };
+    }
+
+    let copilotCloudPublicApiKey: string | null = null;
+    let copilotCloudBaseUrl: string;
+
+    if (data.cloud) {
+      logger = logger.child({ cloud: true });
+      logger.debug("Cloud configuration provided, checking for public API key in headers");
+      const key = ctx.request.headers.get("x-copilotcloud-public-api-key");
+      if (key) {
+        logger.debug("Public API key found in headers");
+        copilotCloudPublicApiKey = key;
+      } else {
+        logger.error("Public API key not found in headers");
+        throw new GraphQLError("X-CopilotCloud-Public-API-Key header is required");
+      }
+
+      if (process.env.COPILOT_CLOUD_BASE_URL) {
+        copilotCloudBaseUrl = process.env.COPILOT_CLOUD_BASE_URL;
+      } else if (ctx._copilotkit.baseUrl) {
+        copilotCloudBaseUrl = ctx._copilotkit.baseUrl;
+      } else {
+        copilotCloudBaseUrl = "https://api.cloud.copilotkit.ai";
+      }
+
+      logger = logger.child({ copilotCloudBaseUrl });
+    }
+    logger.debug("Setting up subjects");
+    const responseStatus$ = new ReplaySubject<typeof ResponseStatusUnion>();
+    const interruptStreaming$ = new ReplaySubject<{ reason: string; messageId?: string }>();
+    const guardrailsResult$ = new ReplaySubject<GuardrailsResult>();
+
+    let outputMessages: Message[] = [];
+    let resolveOutputMessagesPromise: (messages: Message[]) => void;
+    let rejectOutputMessagesPromise: (err: Error) => void;
+
+    const outputMessagesPromise = new Promise<Message[]>((resolve, reject) => {
+      resolveOutputMessagesPromise = resolve;
+      rejectOutputMessagesPromise = reject;
+    });
+
+    logger.debug("Processing");
     const {
       eventSource,
       threadId = nanoid(),
       runId,
+      actions,
     } = await copilotRuntime.process({
       serviceAdapter,
       messages: data.messages,
@@ -43,20 +168,90 @@ export class CopilotResolver {
       threadId: data.threadId,
       runId: data.runId,
       publicApiKey: undefined,
+      properties: ctx.properties || {},
+      outputMessagesPromise,
     });
 
+    logger.debug("Event source created, creating response");
+
     const response = {
       threadId,
       runId,
-      status: firstValueFrom(responseStatus),
+      status: firstValueFrom(responseStatus$),
       messages: new Repeater(async (pushMessage, stopStreamingMessages) => {
+        logger.debug("Messages repeater created");
+
+        if (data.cloud?.guardrails) {
+          logger = logger.child({ guardrails: true });
+          logger.debug("Guardrails is enabled, validating input");
+
+          invokeGuardrails({
+            baseUrl: copilotCloudBaseUrl,
+            copilotCloudPublicApiKey,
+            data,
+            onResult: (result) => {
+              logger.debug({ status: result.status }, "Guardrails validation done");
+              guardrailsResult$.next(result);
+
+              // Guardrails validation failed
+              if (result.status === "denied") {
+                // send the reason to the client and interrupt streaming
+                responseStatus$.next(
+                  new GuardrailsValidationFailureResponse({ guardrailsReason: result.reason }),
+                );
+                interruptStreaming$.next({
+                  reason: `Interrupted due to Guardrails validation failure. Reason: ${result.reason}`,
+                });
+
+                // resolve messages promise to the middleware
+                outputMessages = [
+                  plainToInstance(TextMessage, {
+                    id: nanoid(),
+                    createdAt: new Date(),
+                    content: result.reason,
+                    role: MessageRole.assistant,
+                  }),
+                ];
+                resolveOutputMessagesPromise(outputMessages);
+              }
+            },
+            onError: (err) => {
+              logger.error({ err }, "Error in guardrails validation");
+              responseStatus$.next(
+                new UnknownErrorResponse({
+                  description: `An unknown error has occurred in the guardrails validation`,
+                }),
+              );
+              interruptStreaming$.next({
+                reason: `Interrupted due to unknown error in guardrails validation`,
+              });
+
+              // reject the middleware promise
+              rejectOutputMessagesPromise(err);
+            },
+          });
+        }
+
+        let eventStreamSubscription: Subscription;
+
         // run and process the event stream
-        const eventStream = eventSource.process(copilotRuntime.actions).pipe(
-          // shareReplay() ensures that later subscribers will see the whole stream instead of
-          // just the events that were emitted after the subscriber was added.
-          shareReplay(),
-        );
-        eventStream.subscribe({
+        const eventStream = eventSource
+          .process({
+            serversideActions: actions,
+            guardrailsResult$: data.cloud?.guardrails ? guardrailsResult$ : null,
+          })
+          .pipe(
+            // shareReplay() ensures that later subscribers will see the whole stream instead of
+            // just the events that were emitted after the subscriber was added.
+            shareReplay(),
+            finalize(() => {
+              logger.debug("Event stream finalized");
+            }),
+          );
+
+        logger.debug("Event stream created, subscribing to event stream");
+
+        eventStreamSubscription = eventStream.subscribe({
           next: async (event) => {
             switch (event.type) {
               ////////////////////////////////
@@ -74,21 +269,72 @@ export class CopilotResolver {
                 // signal when we are done streaming
                 const streamingTextStatus = new Subject<typeof MessageStatusUnion>();
 
+                const messageId = nanoid();
+
                 // push the new message
                 pushMessage({
-                  id: nanoid(),
+                  id: messageId,
                   status: firstValueFrom(streamingTextStatus),
                   createdAt: new Date(),
                   role: MessageRole.assistant,
                   content: new Repeater(async (pushTextChunk, stopStreamingText) => {
-                    // push the message content
-                    await textMessageContentStream.forEach(async (e: RuntimeEvent) => {
-                      if (e.type == RuntimeEventTypes.TextMessageContent) {
-                        await pushTextChunk(e.content);
-                      }
+                    logger.debug("Text message content repeater created");
+
+                    const textChunks: string[] = [];
+                    let textSubscription: Subscription;
+
+                    interruptStreaming$
+                      .pipe(
+                        shareReplay(),
+                        take(1),
+                        tap(({ reason, messageId }) => {
+                          logger.debug({ reason, messageId }, "Text streaming interrupted");
+
+                          streamingTextStatus.next(
+                            plainToInstance(FailedMessageStatus, { reason }),
+                          );
+
+                          responseStatus$.next(new MessageStreamInterruptedResponse({ messageId }));
+                          stopStreamingText();
+                          textSubscription?.unsubscribe();
+                        }),
+                      )
+                      .subscribe();
+
+                    logger.debug("Subscribing to text message content stream");
+
+                    textSubscription = textMessageContentStream.subscribe({
+                      next: async (e: RuntimeEvent) => {
+                        if (e.type == RuntimeEventTypes.TextMessageContent) {
+                          await pushTextChunk(e.content);
+                          textChunks.push(e.content);
+                        }
+                      },
+                      error: (err) => {
+                        logger.error({ err }, "Error in text message content stream");
+                        interruptStreaming$.next({
+                          reason: "Error streaming message content",
+                          messageId,
+                        });
+                        stopStreamingText();
+                        textSubscription?.unsubscribe();
+                      },
+                      complete: () => {
+                        logger.debug("Text message content stream completed");
+                        streamingTextStatus.next(new SuccessMessageStatus());
+                        stopStreamingText();
+                        textSubscription?.unsubscribe();
+
+                        outputMessages.push(
+                          plainToInstance(TextMessage, {
+                            id: messageId,
+                            createdAt: new Date(),
+                            content: textChunks.join(""),
+                            role: MessageRole.assistant,
+                          }),
+                        );
+                      },
                     });
-                    stopStreamingText();
-                    streamingTextStatus.next(new SuccessMessageStatus());
                   }),
                 });
                 break;
@@ -96,6 +342,7 @@ export class CopilotResolver {
               // ActionExecutionStart
               ////////////////////////////////
               case RuntimeEventTypes.ActionExecutionStart:
+                logger.debug("Action execution start event received");
                 const actionExecutionArgumentStream = eventStream.pipe(
                   skipWhile((e) => e !== event),
                   takeWhile((e) => e.type != RuntimeEventTypes.ActionExecutionEnd),
@@ -108,13 +355,46 @@ export class CopilotResolver {
                   name: event.actionName,
                   scope: event.scope!,
                   arguments: new Repeater(async (pushArgumentsChunk, stopStreamingArguments) => {
-                    await actionExecutionArgumentStream.forEach(async (e: RuntimeEvent) => {
-                      if (e.type == RuntimeEventTypes.ActionExecutionArgs) {
-                        await pushArgumentsChunk(e.args);
-                      }
+                    logger.debug("Action execution argument stream created");
+
+                    const argumentChunks: string[] = [];
+                    let actionExecutionArgumentSubscription: Subscription;
+
+                    actionExecutionArgumentSubscription = actionExecutionArgumentStream.subscribe({
+                      next: async (e: RuntimeEvent) => {
+                        if (e.type == RuntimeEventTypes.ActionExecutionArgs) {
+                          await pushArgumentsChunk(e.args);
+                          argumentChunks.push(e.args);
+                        }
+                      },
+                      error: (err) => {
+                        logger.error({ err }, "Error in action execution argument stream");
+                        streamingArgumentsStatus.next(
+                          plainToInstance(FailedMessageStatus, {
+                            reason:
+                              "An unknown error has occurred in the action execution argument stream",
+                          }),
+                        );
+                        stopStreamingArguments();
+                        actionExecutionArgumentSubscription?.unsubscribe();
+                      },
+                      complete: () => {
+                        logger.debug("Action execution argument stream completed");
+                        streamingArgumentsStatus.next(new SuccessMessageStatus());
+                        stopStreamingArguments();
+                        actionExecutionArgumentSubscription?.unsubscribe();
+
+                        outputMessages.push(
+                          plainToInstance(ActionExecutionMessage, {
+                            id: event.actionExecutionId,
+                            createdAt: new Date(),
+                            name: event.actionName,
+                            scope: event.scope!,
+                            arguments: argumentChunks.join(""),
+                          }),
+                        );
+                      },
                     });
-                    stopStreamingArguments();
-                    streamingArgumentsStatus.next(new SuccessMessageStatus());
                   }),
                 });
                 break;
@@ -122,6 +402,7 @@ export class CopilotResolver {
               // ActionExecutionResult
               ////////////////////////////////
               case RuntimeEventTypes.ActionExecutionResult:
+                logger.debug({ result: event.result }, "Action execution result event received");
                 pushMessage({
                   id: nanoid(),
                   status: new SuccessMessageStatus(),
@@ -130,13 +411,42 @@ export class CopilotResolver {
                   actionName: event.actionName,
                   result: event.result,
                 });
+
+                outputMessages.push(
+                  plainToInstance(ResultMessage, {
+                    id: nanoid(),
+                    createdAt: new Date(),
+                    actionExecutionId: event.actionExecutionId,
+                    actionName: event.actionName,
+                    result: event.result,
+                  }),
+                );
                 break;
             }
           },
-          error: (err) => console.error("Error in event source", err),
-          complete: () => {
-            responseStatus.next(new SuccessResponseStatus());
+          error: (err) => {
+            logger.error({ err }, "Error in event stream");
+            responseStatus$.next(
+              new UnknownErrorResponse({
+                description: `An unknown error has occurred in the event stream`,
+              }),
+            );
+            eventStreamSubscription?.unsubscribe();
             stopStreamingMessages();
+
+            rejectOutputMessagesPromise(err);
+          },
+          complete: async () => {
+            logger.debug("Event stream completed");
+            if (data.cloud?.guardrails) {
+              logger.debug("Guardrails is enabled, waiting for guardrails result");
+              await firstValueFrom(guardrailsResult$);
+            }
+            responseStatus$.next(new SuccessResponseStatus());
+            eventStreamSubscription?.unsubscribe();
+            stopStreamingMessages();
+
+            resolveOutputMessagesPromise(outputMessages);
           },
         });
       }),

package/src/graphql/types/response-status.type.ts

@@ -1,3 +1,4 @@
+import { GraphQLJSON } from "graphql-scalars";
 import { Field, InterfaceType, ObjectType, createUnionType, registerEnumType } from "type-graphql";
 
 export enum ResponseStatusCode {
@@ -38,12 +39,25 @@ export class SuccessResponseStatus extends BaseResponseStatus {
   code: ResponseStatusCode = ResponseStatusCode.Success;
 }
 
+export enum FailedResponseStatusReason {
+  GUARDRAILS_VALIDATION_FAILED = "GUARDRAILS_VALIDATION_FAILED",
+  MESSAGE_STREAM_INTERRUPTED = "MESSAGE_STREAM_INTERRUPTED",
+  UNKNOWN_ERROR = "UNKNOWN_ERROR",
+}
+
+registerEnumType(FailedResponseStatusReason, {
+  name: "FailedResponseStatusReason",
+});
+
 @ObjectType({ implements: BaseResponseStatus })
 export class FailedResponseStatus extends BaseResponseStatus {
   code: ResponseStatusCode = ResponseStatusCode.Failed;
 
-  @Field(() => String)
-  reason: string;
+  @Field(() => FailedResponseStatusReason)
+  reason: FailedResponseStatusReason;
+
+  @Field(() => GraphQLJSON, { nullable: true })
+  details?: Record<string, any> = null;
 }
 
 export const ResponseStatusUnion = createUnionType({

package/src/index.ts

@@ -1,2 +1,3 @@
 import "reflect-metadata";
 export * from "./lib";
+export * from "./utils";

package/src/lib/cloud/index.ts

@@ -0,0 +1,4 @@
+export interface CopilotCloudOptions {
+  baseUrl?: string;
+  publicApiKey?: string;
+}