@copilotkit/runtime 0.37.0 → 0.38.0-beta.0

package/src/lib/copilot-runtime.ts

@@ -99,48 +99,107 @@
  */
 
 import { Action, actionParametersToJsonSchema, Parameter } from "@copilotkit/shared";
-import { RemoteChain, CopilotServiceAdapter } from "../service-adapters";
-import { CopilotCloud, RemoteCopilotCloud } from "./copilot-cloud";
+import { RemoteChain, RemoteChainParameters, CopilotServiceAdapter } from "../service-adapters";
 import { MessageInput } from "../graphql/inputs/message.input";
 import { ActionInput } from "../graphql/inputs/action.input";
 import { RuntimeEventSource } from "../service-adapters/events";
 import { convertGqlInputToMessages } from "../service-adapters/conversion";
+import { Message } from "../graphql/types/converted";
 
 interface CopilotRuntimeRequest {
   serviceAdapter: CopilotServiceAdapter;
   messages: MessageInput[];
   actions: ActionInput[];
+  outputMessagesPromise: Promise<Message[]>;
+  properties: any;
   threadId?: string;
   runId?: string;
   publicApiKey?: string;
 }
 
 interface CopilotRuntimeResponse {
-  threadId?: string;
+  threadId: string;
   runId?: string;
   eventSource: RuntimeEventSource;
+  actions: Action<any>[];
+}
+
+type ActionsConfiguration<T extends Parameter[] | [] = []> =
+  | Action<T>[]
+  | ((ctx: { properties: any }) => Action<T>[]);
+
+interface OnBeforeRequestOptions {
+  threadId?: string;
+  runId?: string;
+  inputMessages: Message[];
+  properties: any;
+}
+
+type OnBeforeRequestHandler = (options: OnBeforeRequestOptions) => void | Promise<void>;
+
+interface OnAfterRequestOptions {
+  threadId: string;
+  runId?: string;
+  inputMessages: Message[];
+  outputMessages: Message[];
+  properties: any;
+}
+
+type OnAfterRequestHandler = (options: OnAfterRequestOptions) => void | Promise<void>;
+
+interface Middleware {
+  /**
+   * A function that is called before the request is processed.
+   */
+  onBeforeRequest?: OnBeforeRequestHandler;
+
+  /**
+   * A function that is called after the request is processed.
+   */
+  onAfterRequest?: OnAfterRequestHandler;
 }
 
 export interface CopilotRuntimeConstructorParams<T extends Parameter[] | [] = []> {
+  /**
+   * Middleware to be used by the runtime.
+   *
+   * ```ts
+   * onBeforeRequest: (options: {
+   *   threadId?: string;
+   *   runId?: string;
+   *   inputMessages: Message[];
+   *   properties: any;
+   * }) => void | Promise<void>;
+   * ```
+   *
+   * ```ts
+   * onAfterRequest: (options: {
+   *   threadId?: string;
+   *   runId?: string;
+   *   inputMessages: Message[];
+   *   outputMessages: Message[];
+   *   properties: any;
+   * }) => void | Promise<void>;
+   * ```
+   */
+  middleware?: Middleware;
+
   /*
    * A list of server side actions that can be executed.
    */
-  actions?: Action<T>[];
+  actions?: ActionsConfiguration<T>;
 
   /*
    * An array of LangServer URLs.
    */
-  langserve?: RemoteChain[];
-
-  debug?: boolean;
-  copilotCloud?: CopilotCloud;
+  langserve?: RemoteChainParameters[];
 }
 
 export class CopilotRuntime<const T extends Parameter[] | [] = []> {
-  public actions: Action<any>[] = [];
+  public actions: ActionsConfiguration<T>;
   private langserve: Promise<Action<any>>[] = [];
-  private debug: boolean = false;
-  private copilotCloud: CopilotCloud;
+  private onBeforeRequest?: OnBeforeRequestHandler;
+  private onAfterRequest?: OnAfterRequestHandler;
 
   constructor(params?: CopilotRuntimeConstructorParams<T>) {
     this.actions = params?.actions || [];
@@ -149,30 +208,21 @@ export class CopilotRuntime<const T extends Parameter[] | [] = []> {
       const remoteChain = new RemoteChain(chain);
       this.langserve.push(remoteChain.toAction());
     }
-    this.debug = params?.debug || false;
-    this.copilotCloud = params?.copilotCloud || new RemoteCopilotCloud();
-  }
-
-  addAction<const T extends Parameter[] | [] = []>(action: Action<T>): void {
-    this.removeAction(action.name);
-    this.actions.push(action);
-  }
 
-  removeAction(actionName: string): void {
-    this.actions = this.actions.filter((f) => f.name !== actionName);
+    this.onBeforeRequest = params?.middleware?.onBeforeRequest;
+    this.onAfterRequest = params?.middleware?.onAfterRequest;
   }
 
-  async process({
-    serviceAdapter,
-    messages,
-    actions: clientSideActionsInput,
-    threadId,
-    runId,
-    publicApiKey,
-  }: CopilotRuntimeRequest): Promise<CopilotRuntimeResponse> {
-    // TODO-PROTOCOL: cloud configuration
-    // const cloud: CopilotCloudConfig = forwardedProps.cloud;
-
+  async process(request: CopilotRuntimeRequest): Promise<CopilotRuntimeResponse> {
+    const {
+      serviceAdapter,
+      messages,
+      actions: clientSideActionsInput,
+      threadId,
+      runId,
+      properties,
+      outputMessagesPromise,
+    } = request;
     const langserveFunctions: Action<any>[] = [];
 
     for (const chainPromise of this.langserve) {
@@ -184,57 +234,58 @@ export class CopilotRuntime<const T extends Parameter[] | [] = []> {
       }
     }
 
-    const serverSideActionsInput: ActionInput[] = [...this.actions, ...langserveFunctions].map(
-      (action) => ({
-        name: action.name,
-        description: action.description,
-        jsonSchema: JSON.stringify(actionParametersToJsonSchema(action.parameters)),
-      }),
-    );
+    const configuredActions =
+      typeof this.actions === "function" ? this.actions({ properties }) : this.actions;
 
-    const actions = flattenToolCallsNoDuplicates([
+    const actions = [...configuredActions, ...langserveFunctions];
+
+    const serverSideActionsInput: ActionInput[] = actions.map((action) => ({
+      name: action.name,
+      description: action.description,
+      jsonSchema: JSON.stringify(actionParametersToJsonSchema(action.parameters)),
+    }));
+
+    const actionInputs = flattenToolCallsNoDuplicates([
       ...serverSideActionsInput,
       ...clientSideActionsInput,
     ]);
+    const inputMessages = convertGqlInputToMessages(messages);
+
+    await this.onBeforeRequest?.({
+      threadId,
+      runId,
+      inputMessages,
+      properties,
+    });
 
     try {
       const eventSource = new RuntimeEventSource();
-      // TODO-PROTOCOL: type this and support function calls
+
       const result = await serviceAdapter.process({
-        messages: convertGqlInputToMessages(messages),
-        actions,
+        messages: inputMessages,
+        actions: actionInputs,
         threadId,
         runId,
         eventSource,
       });
 
-      // TODO-PROTOCOL add guardrails
-      //
-      // if (publicApiKey !== undefined) {
-      //   // wait for the cloud log chat to finish before streaming back the response
-      //   try {
-      //     const checkGuardrailsInputResult = await this.copilotCloud.checkGuardrailsInput({
-      //       cloud,
-      //       publicApiKey,
-      //       messages: forwardedProps.messages || [],
-      //     });
+      outputMessagesPromise
+        .then((outputMessages) => {
+          this.onAfterRequest?.({
+            threadId: result.threadId,
+            runId: result.runId,
+            inputMessages,
+            outputMessages,
+            properties,
+          });
+        })
+        .catch((_error) => {});
 
-      //     if (checkGuardrailsInputResult.status === "denied") {
-      //       // the chat was denied. instead of streaming back the response,
-      //       // we let the client know...
-      //       return {
-      //         stream: new SingleChunkReadableStream(checkGuardrailsInputResult.reason),
-      //         // headers: result.headers,
-      //       };
-      //     }
-      //   } catch (error) {
-      //     console.error("Error checking guardrails:", error);
-      //   }
-      // }
       return {
         threadId: result.threadId,
         runId: result.runId,
         eventSource,
+        actions: actions,
       };
     } catch (error) {
       console.error("Error getting response:", error);
@@ -254,8 +305,3 @@ export function flattenToolCallsNoDuplicates(toolsByPriority: ActionInput[]): Ac
   }
   return allTools;
 }
-
-/**
- * @deprecated use CopilotRuntime instead
- */
-export class CopilotBackend extends CopilotRuntime {}

package/src/lib/index.ts

@@ -3,5 +3,4 @@ export * from "../service-adapters/openai/openai-adapter";
 export * from "../service-adapters/langchain/langchain-adapter";
 export * from "../service-adapters/google/google-genai-adapter";
 export * from "../service-adapters/openai/openai-assistant-adapter";
-export * from "./copilot-cloud";
 export * from "./integrations";

package/src/lib/integrations/nextjs/app-router.ts

@@ -1,29 +1,21 @@
 import { createYoga } from "graphql-yoga";
-import { getCommonConfig } from "../shared";
-import { CopilotRuntime } from "../../copilot-runtime";
-import { CopilotServiceAdapter } from "../../../service-adapters";
+import { CreateCopilotRuntimeServerOptions, getCommonConfig } from "../shared";
 
-export function copilotRuntimeNextJSAppRouterEndpoint({
-  runtime,
-  endpoint,
-  serviceAdapter,
-}: {
-  runtime: CopilotRuntime;
-  serviceAdapter: CopilotServiceAdapter;
-  endpoint: string;
-}) {
-  const commonConfig = getCommonConfig({ runtime, serviceAdapter });
+export function copilotRuntimeNextJSAppRouterEndpoint(options: CreateCopilotRuntimeServerOptions) {
+  const commonConfig = getCommonConfig(options);
+  const logger = commonConfig.logging;
+  logger.debug("Creating NextJS App Router endpoint");
 
   const yoga = createYoga({
     ...commonConfig,
-    graphqlEndpoint: endpoint,
+    graphqlEndpoint: options.endpoint,
     fetchAPI: { Response: globalThis.Response },
   });
 
   return {
     handleRequest: yoga,
-    GET: yoga,
-    POST: yoga,
-    OPTIONS: yoga,
+    GET: yoga as any,
+    POST: yoga as any,
+    OPTIONS: yoga as any,
   };
 }

package/src/lib/integrations/nextjs/pages-router.ts

@@ -1,7 +1,5 @@
 import { YogaServerInstance, createYoga } from "graphql-yoga";
-import { GraphQLContext, getCommonConfig } from "../shared";
-import { CopilotRuntime } from "../../copilot-runtime";
-import { CopilotServiceAdapter } from "../../../service-adapters";
+import { CreateCopilotRuntimeServerOptions, GraphQLContext, getCommonConfig } from "../shared";
 
 export const config = {
   api: {
@@ -11,25 +9,21 @@ export const config = {
 
 export type CopilotRuntimeServerInstance<T> = YogaServerInstance<T, Partial<GraphQLContext>>;
 
-// Theis import is needed to fix the type error
+// This import is needed to fix the type error
 // Fix is currently in TypeScript 5.5 beta, waiting for stable version
 // https://github.com/microsoft/TypeScript/issues/42873#issuecomment-2066874644
 export type {} from "@whatwg-node/server";
 
-export function copilotRuntimeNextJSPagesRouterEndpoint({
-  runtime,
-  endpoint,
-  serviceAdapter,
-}: {
-  runtime: CopilotRuntime;
-  serviceAdapter: CopilotServiceAdapter;
-  endpoint: string;
-}): CopilotRuntimeServerInstance<GraphQLContext> {
-  const commonConfig = getCommonConfig({ runtime, serviceAdapter });
+export function copilotRuntimeNextJSPagesRouterEndpoint(
+  options: CreateCopilotRuntimeServerOptions,
+): CopilotRuntimeServerInstance<GraphQLContext> {
+  const commonConfig = getCommonConfig(options);
+  const logger = commonConfig.logging;
+  logger.debug("Creating NextJS Pages Router endpoint");
 
   const yoga = createYoga({
     ...commonConfig,
-    graphqlEndpoint: endpoint,
+    graphqlEndpoint: options.endpoint,
   });
 
   return yoga;

package/src/lib/integrations/node-http/index.ts

@@ -1,22 +1,14 @@
 import { createYoga } from "graphql-yoga";
-import { getCommonConfig } from "../shared";
-import { CopilotRuntime } from "../../copilot-runtime";
-import { CopilotServiceAdapter } from "../../../service-adapters";
+import { CreateCopilotRuntimeServerOptions, getCommonConfig } from "../shared";
 
-export function copilotRuntimeNodeHttpEndpoint({
-  runtime,
-  endpoint,
-  serviceAdapter,
-}: {
-  runtime: CopilotRuntime;
-  serviceAdapter: CopilotServiceAdapter;
-  endpoint: string;
-}) {
-  const commonConfig = getCommonConfig({ runtime, serviceAdapter });
+export function copilotRuntimeNodeHttpEndpoint(options: CreateCopilotRuntimeServerOptions) {
+  const commonConfig = getCommonConfig(options);
+  const logger = commonConfig.logging;
+  logger.debug("Creating Node HTTP endpoint");
 
   const yoga = createYoga({
     ...commonConfig,
-    graphqlEndpoint: endpoint,
+    graphqlEndpoint: options.endpoint,
   });
 
   return yoga;

package/src/lib/integrations/shared.ts

@@ -1,10 +1,14 @@
 import { YogaInitialContext } from "graphql-yoga";
-import { GuardrailsOptions } from "../guardrails";
 import { buildSchemaSync } from "type-graphql";
 import { CopilotResolver } from "../../graphql/resolvers/copilot.resolver";
 import { useDeferStream } from "@graphql-yoga/plugin-defer-stream";
 import { CopilotRuntime } from "../copilot-runtime";
 import { CopilotServiceAdapter } from "../../service-adapters";
+import { CopilotCloudOptions } from "../cloud";
+import { LogLevel, createLogger } from "../../lib/logger";
+import { createYoga } from "graphql-yoga";
+
+const logger = createLogger();
 
 type AnyPrimitive = string | boolean | number | null;
 export type CopilotRequestContextProperties = Record<
@@ -12,33 +16,37 @@ export type CopilotRequestContextProperties = Record<
   AnyPrimitive | Record<string, AnyPrimitive>
 >;
 
-type CopilotKitContext = {
-  runtime: CopilotRuntime;
-  serviceAdapter: CopilotServiceAdapter;
-  properties: CopilotRequestContextProperties;
-};
-
 export type GraphQLContext = YogaInitialContext & {
-  _copilotkit: CopilotKitContext;
+  _copilotkit: CreateCopilotRuntimeServerOptions;
+  properties: CopilotRequestContextProperties;
+  logger: typeof logger;
 };
 
 export interface CreateCopilotRuntimeServerOptions {
-  runtime: CopilotRuntime;
+  runtime: CopilotRuntime<any>;
   serviceAdapter: CopilotServiceAdapter;
-  guardrails?: GuardrailsOptions;
+  endpoint: string;
+  baseUrl?: string;
+  cloud?: CopilotCloudOptions;
+  properties?: CopilotRequestContextProperties;
+  logLevel?: LogLevel;
 }
 
 export async function createContext(
   initialContext: YogaInitialContext,
-  copilotKitContext: CopilotKitContext,
+  copilotKitContext: CreateCopilotRuntimeServerOptions,
+  contextLogger: typeof logger,
+  properties: CopilotRequestContextProperties = {},
 ): Promise<Partial<GraphQLContext>> {
+  logger.debug({ copilotKitContext }, "Creating GraphQL context");
   const ctx: GraphQLContext = {
     ...initialContext,
     _copilotkit: {
       ...copilotKitContext,
     },
+    properties: { ...properties },
+    logger: contextLogger,
   };
-
   return ctx;
 }
 
@@ -47,22 +55,34 @@ export function buildSchema(
     emitSchemaFile?: string;
   } = {},
 ) {
+  logger.debug("Building GraphQL schema...");
   const schema = buildSchemaSync({
     resolvers: [CopilotResolver],
     emitSchemaFile: options.emitSchemaFile,
   });
+  logger.debug("GraphQL schema built successfully");
   return schema;
 }
 
-export function getCommonConfig(options?: CreateCopilotRuntimeServerOptions) {
+export type CommonConfig = {
+  logging: typeof logger;
+  schema: ReturnType<typeof buildSchema>;
+  plugins: Parameters<typeof createYoga>[0]["plugins"];
+  context: (ctx: YogaInitialContext) => Promise<Partial<GraphQLContext>>;
+};
+
+export function getCommonConfig(options: CreateCopilotRuntimeServerOptions): CommonConfig {
+  const logLevel = (process.env.LOG_LEVEL as LogLevel) || (options.logLevel as LogLevel) || "error";
+  const logger = createLogger({ level: logLevel, component: "getCommonConfig" });
+  logger.debug("Getting common config");
+
+  const contextLogger = createLogger({ level: logLevel });
+
   return {
+    logging: createLogger({ component: "Yoga GraphQL", level: logLevel }),
     schema: buildSchema(),
     plugins: [useDeferStream()],
     context: (ctx: YogaInitialContext): Promise<Partial<GraphQLContext>> =>
-      createContext(ctx, {
-        runtime: options.runtime,
-        serviceAdapter: options.serviceAdapter,
-        properties: {},
-      }),
+      createContext(ctx, options, contextLogger, options.properties),
   };
 }

package/src/lib/logger.ts

@@ -0,0 +1,28 @@
+import createPinoLogger from "pino";
+import pretty from "pino-pretty";
+
+export type LogLevel = "debug" | "info" | "warn" | "error";
+
+export type CopilotRuntimeLogger = ReturnType<typeof createLogger>;
+
+export function createLogger(options?: { level?: LogLevel; component?: string }) {
+  const { level, component } = options || {};
+  const stream = pretty({ colorize: true });
+
+  const logger = createPinoLogger(
+    {
+      level: process.env.LOG_LEVEL || level || "error",
+      redact: {
+        paths: ["pid", "hostname"],
+        remove: true,
+      },
+    },
+    stream,
+  );
+
+  if (component) {
+    return logger.child({ component });
+  } else {
+    return logger;
+  }
+}

package/src/service-adapters/events.ts

@@ -1,6 +1,7 @@
 import { Action } from "@copilotkit/shared";
-import { of, concat, map, scan, concatMap, ReplaySubject } from "rxjs";
+import { of, concat, map, scan, concatMap, ReplaySubject, Subject, firstValueFrom } from "rxjs";
 import { streamLangChainResponse } from "./langchain/utils";
+import { GuardrailsResult } from "../graphql/types/guardrails-result.type";
 
 export enum RuntimeEventTypes {
   TextMessageStart = "TextMessageStart",
@@ -109,7 +110,13 @@ export class RuntimeEventSource {
     this.callback = callback;
   }
 
-  process(serversideActions: Action<any>[]) {
+  process({
+    serversideActions,
+    guardrailsResult$,
+  }: {
+    serversideActions: Action<any>[];
+    guardrailsResult$?: Subject<GuardrailsResult>;
+  }) {
     this.callback(this.eventStream$).catch((error) => {
       console.error("Error in event source callback", error);
     });
@@ -156,6 +163,7 @@ export class RuntimeEventSource {
           const toolCallEventStream$ = new RuntimeEventSubject();
           executeAction(
             toolCallEventStream$,
+            guardrailsResult$ ? guardrailsResult$ : null,
             eventWithState.action!,
             eventWithState.args,
             eventWithState.actionExecutionId,
@@ -173,10 +181,20 @@ export class RuntimeEventSource {
 
 async function executeAction(
   eventStream$: RuntimeEventSubject,
+  guardrailsResult$: Subject<GuardrailsResult> | null,
   action: Action<any>,
   actionArguments: string,
   actionExecutionId: string,
 ) {
+  if (guardrailsResult$) {
+    const { status } = await firstValueFrom(guardrailsResult$);
+
+    if (status === "denied") {
+      eventStream$.complete();
+      return;
+    }
+  }
+
   // Prepare arguments for function calling
   let args: Record<string, any>[] = [];
   if (actionArguments) {

package/src/service-adapters/experimental/groq/groq-adapter.ts

@@ -107,7 +107,9 @@ export class ExperimentalGroqAdapter implements CopilotServiceAdapter {
 
       eventStream$.complete();
     });
-    return {};
+    return {
+      threadId: request.threadId || nanoid(),
+    };
   }
 }
 

package/src/service-adapters/experimental/ollama/ollama-adapter.ts

@@ -70,6 +70,8 @@ export class ExperimentalOllamaAdapter implements CopilotServiceAdapter {
 
       eventStream$.complete();
     });
-    return {};
+    return {
+      threadId: request.threadId || nanoid(),
+    };
   }
 }

package/src/service-adapters/google/google-genai-adapter.ts

@@ -104,6 +104,9 @@ export class GoogleGenerativeAIAdapter implements CopilotServiceAdapter {
       let isTextMessage = false;
       for await (const chunk of result.stream) {
         const chunkText = chunk.text();
+        if (chunkText === "") {
+          continue;
+        }
         if (!isTextMessage) {
           isTextMessage = true;
           eventStream$.sendTextMessageStart(nanoid());
@@ -127,7 +130,9 @@ export class GoogleGenerativeAIAdapter implements CopilotServiceAdapter {
       eventStream$.complete();
     });
 
-    return {};
+    return {
+      threadId: request.threadId || nanoid(),
+    };
   }
 }
 

package/src/service-adapters/google/utils.ts

@@ -38,7 +38,7 @@ export function convertMessageToGoogleGenAIMessage(message: Message) {
     };
   } else if (message instanceof ResultMessage) {
     return {
-      role: "model",
+      role: "function",
       parts: [
         {
           functionResponse: {

package/src/service-adapters/index.ts

@@ -3,4 +3,4 @@ export { OpenAIAdapter } from "./openai/openai-adapter";
 export { OpenAIAssistantAdapter } from "./openai/openai-assistant-adapter";
 export { GoogleGenerativeAIAdapter } from "./google/google-genai-adapter";
 export { LangChainAdapter } from "./langchain/langchain-adapter";
-export { RemoteChain } from "./langchain/langserve";
+export { RemoteChain, type RemoteChainParameters } from "./langchain/langserve";

package/src/service-adapters/langchain/langchain-adapter.ts

@@ -35,6 +35,7 @@ import {
 } from "./utils";
 import { DynamicStructuredTool } from "@langchain/core/tools";
 import { LangChainReturnType } from "./types";
+import { nanoid } from "nanoid";
 
 interface ChainFnParameters {
   model: string;
@@ -54,14 +55,10 @@ export class LangChainAdapter implements CopilotServiceAdapter {
    */
   constructor(private options: LangChainAdapterOptions) {}
 
-  async process({
-    eventSource,
-    model,
-    actions,
-    messages,
-    threadId,
-    runId,
-  }: CopilotRuntimeChatCompletionRequest): Promise<CopilotRuntimeChatCompletionResponse> {
+  async process(
+    request: CopilotRuntimeChatCompletionRequest,
+  ): Promise<CopilotRuntimeChatCompletionResponse> {
+    const { eventSource, model, actions, messages, threadId, runId } = request;
     const result = await this.options.chainFn({
       messages: messages.map(convertMessageToLangChainMessage),
       tools: actions.map(convertActionInputToLangChainTool),
@@ -77,6 +74,8 @@ export class LangChainAdapter implements CopilotServiceAdapter {
       });
     });
 
-    return {};
+    return {
+      threadId: threadId || nanoid(),
+    };
   }
 }