@copilotkit/runtime 1.2.1 → 1.2.2-feat-runtime-remote-actions.0

package/src/graphql/resolvers/copilot.resolver.ts

@@ -32,7 +32,13 @@ import {
   MessageStreamInterruptedResponse,
   UnknownErrorResponse,
 } from "../../utils";
-import { ActionExecutionMessage, Message, ResultMessage, TextMessage } from "../types/converted";
+import {
+  ActionExecutionMessage,
+  AgentStateMessage,
+  Message,
+  ResultMessage,
+  TextMessage,
+} from "../types/converted";
 import telemetry from "../../lib/telemetry-client";
 import { randomId } from "@copilotkit/shared";
 
@@ -49,8 +55,6 @@ const invokeGuardrails = async ({
   onResult: (result: GuardrailsResult) => void;
   onError: (err: Error) => void;
 }) => {
-  console.log("invokeGuardrails.baseUrl", baseUrl);
-
   if (
     data.messages.length &&
     data.messages[data.messages.length - 1].textMessage?.role === MessageRole.user
@@ -116,14 +120,14 @@ export class CopilotResolver {
     let logger = ctx.logger.child({ component: "CopilotResolver.generateCopilotResponse" });
     logger.debug({ data }, "Generating Copilot response");
 
-    const copilotRuntime = ctx._copilotkit.runtime;
-    const serviceAdapter = ctx._copilotkit.serviceAdapter;
-
     if (properties) {
       logger.debug("Properties provided, merging with context properties");
       ctx.properties = { ...ctx.properties, ...properties };
     }
 
+    const copilotRuntime = ctx._copilotkit.runtime;
+    const serviceAdapter = ctx._copilotkit.serviceAdapter;
+
     let copilotCloudPublicApiKey: string | null = null;
     let copilotCloudBaseUrl: string;
 
@@ -169,17 +173,20 @@ export class CopilotResolver {
       eventSource,
       threadId = randomId(),
       runId,
-      actions,
-    } = await copilotRuntime.process({
+      serverSideActions,
+      actionInputsWithoutAgents,
+    } = await copilotRuntime.processRuntimeRequest({
       serviceAdapter,
       messages: data.messages,
       actions: data.frontend.actions,
       threadId: data.threadId,
       runId: data.runId,
       publicApiKey: undefined,
-      properties: ctx.properties || {},
       outputMessagesPromise,
+      graphqlContext: ctx,
       forwardedParameters: data.forwardedParameters,
+      agentSession: data.agentSession,
+      agentStates: data.agentStates,
       url: data.frontend.url,
     });
 
@@ -247,9 +254,14 @@ export class CopilotResolver {
 
         // run and process the event stream
         const eventStream = eventSource
-          .process({
-            serversideActions: actions,
+          .processRuntimeEvents({
+            serverSideActions,
             guardrailsResult$: data.cloud?.guardrails ? guardrailsResult$ : null,
+            actionInputsWithoutAgents: actionInputsWithoutAgents.filter(
+              // TODO-AGENTS: do not exclude ALL server side actions
+              (action) =>
+                !serverSideActions.find((serverSideAction) => serverSideAction.name == action.name),
+            ),
           })
           .pipe(
             // shareReplay() ensures that later subscribers will see the whole stream instead of
@@ -281,7 +293,6 @@ export class CopilotResolver {
                 const streamingTextStatus = new Subject<typeof MessageStatusUnion>();
 
                 const messageId = randomId();
-
                 // push the new message
                 pushMessage({
                   id: messageId,
@@ -433,6 +444,39 @@ export class CopilotResolver {
                   }),
                 );
                 break;
+              ////////////////////////////////
+              // AgentStateMessage
+              ////////////////////////////////
+              case RuntimeEventTypes.AgentStateMessage:
+                logger.debug({ event }, "Agent message event received");
+                pushMessage({
+                  id: randomId(),
+                  status: new SuccessMessageStatus(),
+                  threadId: event.threadId,
+                  agentName: event.agentName,
+                  nodeName: event.nodeName,
+                  runId: event.runId,
+                  active: event.active,
+                  state: event.state,
+                  running: event.running,
+                  role: MessageRole.assistant,
+                  createdAt: new Date(),
+                });
+                outputMessages.push(
+                  plainToInstance(AgentStateMessage, {
+                    id: randomId(),
+                    threadId: event.threadId,
+                    agentName: event.agentName,
+                    nodeName: event.nodeName,
+                    runId: event.runId,
+                    active: event.active,
+                    state: event.state,
+                    running: event.running,
+                    role: MessageRole.assistant,
+                    createdAt: new Date(),
+                  }),
+                );
+                break;
             }
           },
           error: (err) => {

package/src/graphql/types/converted/index.ts

@@ -2,6 +2,7 @@ import {
   ActionExecutionMessageInput,
   ResultMessageInput,
   TextMessageInput,
+  AgentStateMessageInput,
 } from "../../inputs/message.input";
 import { BaseMessage } from "../base";
 import { ActionExecutionScope, MessageRole } from "../enums";
@@ -27,3 +28,17 @@ export class ResultMessage extends BaseMessage implements ResultMessageInput {
   actionName: string;
   result: string;
 }
+
+export class AgentStateMessage
+  extends BaseMessage
+  implements Omit<AgentStateMessageInput, "state">
+{
+  threadId: string;
+  agentName: string;
+  nodeName: string;
+  runId: string;
+  active: boolean;
+  role: MessageRole;
+  state: any;
+  running: boolean;
+}

package/src/graphql/types/copilot-response.type.ts

@@ -11,6 +11,8 @@ import { ResponseStatusUnion } from "./response-status.type";
       return ActionExecutionMessageOutput;
     } else if (value.hasOwnProperty("result")) {
       return ResultMessageOutput;
+    } else if (value.hasOwnProperty("state")) {
+      return AgentStateMessageOutput;
     }
     return undefined;
   },
@@ -59,6 +61,33 @@ export class ResultMessageOutput {
   result: string;
 }
 
+@ObjectType({ implements: BaseMessageOutput })
+export class AgentStateMessageOutput {
+  @Field(() => String)
+  threadId: string;
+
+  @Field(() => String)
+  agentName: string;
+
+  @Field(() => String)
+  nodeName: string;
+
+  @Field(() => String)
+  runId: string;
+
+  @Field(() => Boolean)
+  active: boolean;
+
+  @Field(() => MessageRole)
+  role: MessageRole;
+
+  @Field(() => String)
+  state: string;
+
+  @Field(() => Boolean)
+  running: boolean;
+}
+
 @ObjectType()
 export class CopilotResponse {
   @Field(() => String)

package/src/graphql/types/enums.ts

@@ -9,6 +9,7 @@ export enum MessageRole {
 export enum ActionExecutionScope {
   server = "server",
   client = "client",
+  passThrough = "passThrough",
 }
 
 export enum CopilotRequestType {

package/src/lib/index.ts

@@ -1,4 +1,4 @@
-export * from "./copilot-runtime";
+export * from "./runtime/copilot-runtime";
 export * from "../service-adapters/openai/openai-adapter";
 export * from "../service-adapters/langchain/langchain-adapter";
 export * from "../service-adapters/google/google-genai-adapter";

package/src/lib/integrations/shared.ts

@@ -2,7 +2,7 @@ import { YogaInitialContext } from "graphql-yoga";
 import { buildSchemaSync } from "type-graphql";
 import { CopilotResolver } from "../../graphql/resolvers/copilot.resolver";
 import { useDeferStream } from "@graphql-yoga/plugin-defer-stream";
-import { CopilotRuntime } from "../copilot-runtime";
+import { CopilotRuntime } from "../runtime/copilot-runtime";
 import { CopilotServiceAdapter } from "../../service-adapters";
 import { CopilotCloudOptions } from "../cloud";
 import { LogLevel, createLogger } from "../../lib/logger";

package/src/lib/runtime/copilot-runtime.ts

@@ -0,0 +1,360 @@
+/**
+ * <Callout type="info">
+ *   This is the reference for the `CopilotRuntime` class. For more information and example code snippets, please see [Concept: Copilot Runtime](/concepts/copilot-runtime).
+ * </Callout>
+ *
+ * ## Usage
+ *
+ * ```tsx
+ * import { CopilotRuntime } from "@copilotkit/runtime";
+ *
+ * const copilotKit = new CopilotRuntime();
+ * ```
+ */
+
+import { Action, actionParametersToJsonSchema, Parameter, randomId } from "@copilotkit/shared";
+import { RemoteChain, RemoteChainParameters, CopilotServiceAdapter } from "../../service-adapters";
+import { MessageInput } from "../../graphql/inputs/message.input";
+import { ActionInput } from "../../graphql/inputs/action.input";
+import { RuntimeEventSource } from "../../service-adapters/events";
+import { convertGqlInputToMessages } from "../../service-adapters/conversion";
+import { AgentStateMessage, Message } from "../../graphql/types/converted";
+import { ForwardedParametersInput } from "../../graphql/inputs/forwarded-parameters.input";
+import {
+  setupRemoteActions,
+  RemoteActionDefinition,
+  LangGraphAgentAction,
+  isLangGraphAgentAction,
+} from "./remote-actions";
+import { GraphQLContext } from "../integrations/shared";
+import { AgentSessionInput } from "../../graphql/inputs/agent-session.input";
+import { from } from "rxjs";
+import { AgentStateInput } from "../../graphql/inputs/agent-state.input";
+
+interface CopilotRuntimeRequest {
+  serviceAdapter: CopilotServiceAdapter;
+  messages: MessageInput[];
+  actions: ActionInput[];
+  agentSession?: AgentSessionInput;
+  agentStates?: AgentStateInput[];
+  outputMessagesPromise: Promise<Message[]>;
+  threadId?: string;
+  runId?: string;
+  publicApiKey?: string;
+  graphqlContext: GraphQLContext;
+  forwardedParameters?: ForwardedParametersInput;
+  url?: string;
+}
+
+interface CopilotRuntimeResponse {
+  threadId: string;
+  runId?: string;
+  eventSource: RuntimeEventSource;
+  serverSideActions: Action<any>[];
+  actionInputsWithoutAgents: ActionInput[];
+}
+
+type ActionsConfiguration<T extends Parameter[] | [] = []> =
+  | Action<T>[]
+  | ((ctx: { properties: any; url?: string }) => Action<T>[]);
+
+interface OnBeforeRequestOptions {
+  threadId?: string;
+  runId?: string;
+  inputMessages: Message[];
+  properties: any;
+  url?: string;
+}
+
+type OnBeforeRequestHandler = (options: OnBeforeRequestOptions) => void | Promise<void>;
+
+interface OnAfterRequestOptions {
+  threadId: string;
+  runId?: string;
+  inputMessages: Message[];
+  outputMessages: Message[];
+  properties: any;
+  url?: string;
+}
+
+type OnAfterRequestHandler = (options: OnAfterRequestOptions) => void | Promise<void>;
+
+interface Middleware {
+  /**
+   * A function that is called before the request is processed.
+   */
+  onBeforeRequest?: OnBeforeRequestHandler;
+
+  /**
+   * A function that is called after the request is processed.
+   */
+  onAfterRequest?: OnAfterRequestHandler;
+}
+
+export interface CopilotRuntimeConstructorParams<T extends Parameter[] | [] = []> {
+  /**
+   * Middleware to be used by the runtime.
+   *
+   * ```ts
+   * onBeforeRequest: (options: {
+   *   threadId?: string;
+   *   runId?: string;
+   *   inputMessages: Message[];
+   *   properties: any;
+   * }) => void | Promise<void>;
+   * ```
+   *
+   * ```ts
+   * onAfterRequest: (options: {
+   *   threadId?: string;
+   *   runId?: string;
+   *   inputMessages: Message[];
+   *   outputMessages: Message[];
+   *   properties: any;
+   * }) => void | Promise<void>;
+   * ```
+   */
+  middleware?: Middleware;
+
+  /*
+   * A list of server side actions that can be executed.
+   */
+  actions?: ActionsConfiguration<T>;
+
+  /*
+   * A list of remote actions that can be executed.
+   */
+  remoteActions?: RemoteActionDefinition[];
+
+  /*
+   * An array of LangServer URLs.
+   */
+  langserve?: RemoteChainParameters[];
+}
+
+export class CopilotRuntime<const T extends Parameter[] | [] = []> {
+  public actions: ActionsConfiguration<T>;
+  private remoteActionDefinitions: RemoteActionDefinition[];
+  private langserve: Promise<Action<any>>[] = [];
+  private onBeforeRequest?: OnBeforeRequestHandler;
+  private onAfterRequest?: OnAfterRequestHandler;
+
+  constructor(params?: CopilotRuntimeConstructorParams<T>) {
+    this.actions = params?.actions || [];
+
+    for (const chain of params?.langserve || []) {
+      const remoteChain = new RemoteChain(chain);
+      this.langserve.push(remoteChain.toAction());
+    }
+
+    this.remoteActionDefinitions = params?.remoteActions || [];
+
+    this.onBeforeRequest = params?.middleware?.onBeforeRequest;
+    this.onAfterRequest = params?.middleware?.onAfterRequest;
+  }
+
+  async processRuntimeRequest(request: CopilotRuntimeRequest): Promise<CopilotRuntimeResponse> {
+    const {
+      serviceAdapter,
+      messages: rawMessages,
+      actions: clientSideActionsInput,
+      threadId,
+      runId,
+      outputMessagesPromise,
+      graphqlContext,
+      forwardedParameters,
+      agentSession,
+      url,
+    } = request;
+
+    if (agentSession) {
+      return this.processAgentRequest(request);
+    }
+
+    const messages = rawMessages.filter((message) => !message.agentStateMessage);
+
+    const inputMessages = convertGqlInputToMessages(messages);
+    const serverSideActions = await this.getServerSideActions(request);
+
+    const serverSideActionsInput: ActionInput[] = serverSideActions.map((action) => ({
+      name: action.name,
+      description: action.description,
+      jsonSchema: JSON.stringify(actionParametersToJsonSchema(action.parameters)),
+    }));
+
+    const actionInputs = flattenToolCallsNoDuplicates([
+      ...serverSideActionsInput,
+      ...clientSideActionsInput,
+    ]);
+
+    await this.onBeforeRequest?.({
+      threadId,
+      runId,
+      inputMessages,
+      properties: graphqlContext.properties,
+      url,
+    });
+
+    try {
+      const eventSource = new RuntimeEventSource();
+
+      const result = await serviceAdapter.process({
+        messages: inputMessages,
+        actions: actionInputs,
+        threadId,
+        runId,
+        eventSource,
+        forwardedParameters,
+      });
+
+      outputMessagesPromise
+        .then((outputMessages) => {
+          this.onAfterRequest?.({
+            threadId: result.threadId,
+            runId: result.runId,
+            inputMessages,
+            outputMessages,
+            properties: graphqlContext.properties,
+            url,
+          });
+        })
+        .catch((_error) => {});
+
+      return {
+        threadId: result.threadId,
+        runId: result.runId,
+        eventSource,
+        serverSideActions,
+        actionInputsWithoutAgents: actionInputs.filter(
+          (action) =>
+            // TODO-AGENTS: do not exclude ALL server side actions
+            !serverSideActions.find((serverSideAction) => serverSideAction.name == action.name),
+          // !isLangGraphAgentAction(
+          //   serverSideActions.find((serverSideAction) => serverSideAction.name == action.name),
+          // ),
+        ),
+      };
+    } catch (error) {
+      console.error("Error getting response:", error);
+      throw error;
+    }
+  }
+
+  private async processAgentRequest(
+    request: CopilotRuntimeRequest,
+  ): Promise<CopilotRuntimeResponse> {
+    const { messages: rawMessages, outputMessagesPromise, graphqlContext, agentSession } = request;
+    const { threadId = randomId(), agentName, nodeName } = agentSession;
+    const serverSideActions = await this.getServerSideActions(request);
+
+    const messages = convertGqlInputToMessages(rawMessages);
+
+    const agent = serverSideActions.find(
+      (action) => action.name === agentName && isLangGraphAgentAction(action),
+    ) as LangGraphAgentAction;
+
+    if (!agent) {
+      throw new Error(`Agent ${agentName} not found`);
+    }
+
+    const serverSideActionsInput: ActionInput[] = serverSideActions
+      .filter((action) => !isLangGraphAgentAction(action))
+      .map((action) => ({
+        name: action.name,
+        description: action.description,
+        jsonSchema: JSON.stringify(actionParametersToJsonSchema(action.parameters)),
+      }));
+
+    const actionInputsWithoutAgents = flattenToolCallsNoDuplicates([
+      ...serverSideActionsInput,
+      ...request.actions,
+    ]);
+
+    await this.onBeforeRequest?.({
+      threadId,
+      runId: undefined,
+      inputMessages: messages,
+      properties: graphqlContext.properties,
+    });
+    try {
+      const eventSource = new RuntimeEventSource();
+      const stream = await agent.langGraphAgentHandler({
+        name: agentName,
+        threadId,
+        nodeName,
+        actionInputsWithoutAgents,
+      });
+
+      eventSource.stream(async (eventStream$) => {
+        from(stream).subscribe({
+          next: (event) => eventStream$.next(event),
+          error: (err) => console.error("Error in stream", err),
+          complete: () => eventStream$.complete(),
+        });
+      });
+
+      outputMessagesPromise
+        .then((outputMessages) => {
+          this.onAfterRequest?.({
+            threadId,
+            runId: undefined,
+            inputMessages: messages,
+            outputMessages,
+            properties: graphqlContext.properties,
+          });
+        })
+        .catch((_error) => {});
+
+      return {
+        threadId,
+        runId: undefined,
+        eventSource,
+        serverSideActions: [],
+        actionInputsWithoutAgents,
+      };
+    } catch (error) {
+      console.error("Error getting response:", error);
+      throw error;
+    }
+  }
+
+  private async getServerSideActions(request: CopilotRuntimeRequest): Promise<Action<any>[]> {
+    const { messages: rawMessages, graphqlContext, agentStates, url } = request;
+    const inputMessages = convertGqlInputToMessages(rawMessages);
+    const langserveFunctions: Action<any>[] = [];
+
+    for (const chainPromise of this.langserve) {
+      try {
+        const chain = await chainPromise;
+        langserveFunctions.push(chain);
+      } catch (error) {
+        console.error("Error loading langserve chain:", error);
+      }
+    }
+    const remoteActions = await setupRemoteActions({
+      remoteActionDefinitions: this.remoteActionDefinitions,
+      graphqlContext,
+      messages: inputMessages,
+      agentStates,
+    });
+
+    const configuredActions =
+      typeof this.actions === "function"
+        ? this.actions({ properties: graphqlContext.properties, url })
+        : this.actions;
+
+    return [...configuredActions, ...langserveFunctions, ...remoteActions];
+  }
+}
+
+export function flattenToolCallsNoDuplicates(toolsByPriority: ActionInput[]): ActionInput[] {
+  let allTools: ActionInput[] = [];
+  const allToolNames: string[] = [];
+  for (const tool of toolsByPriority) {
+    if (!allToolNames.includes(tool.name)) {
+      allTools.push(tool);
+      allToolNames.push(tool.name);
+    }
+  }
+  return allTools;
+}