@copilotkit/runtime 1.1.2 → 1.1.3-feat-runtime-remote-actions.0

package/src/lib/runtime/remote-actions.ts

@@ -0,0 +1,241 @@
+import { Action } from "@copilotkit/shared";
+import { GraphQLContext } from "../integrations/shared";
+import { Logger } from "pino";
+import telemetry from "../../lib/telemetry-client";
+import { Message } from "../../graphql/types/converted";
+import { RuntimeEvent, RuntimeEventSubject } from "../../service-adapters/events";
+import { RemoteLangGraphEventSource } from "../../agents/langgraph/event-source";
+import { Observable } from "rxjs";
+import { ActionInput } from "../../graphql/inputs/action.input";
+import { AgentStateInput } from "../../graphql/inputs/agent-state.input";
+
+export type RemoteActionDefinition = {
+  url: string;
+  onBeforeRequest?: ({ ctx }: { ctx: GraphQLContext }) => {
+    headers?: Record<string, string> | undefined;
+  };
+};
+
+export type LangGraphAgentHandlerParams = {
+  name: string;
+  actionInputsWithoutAgents: ActionInput[];
+  threadId?: string;
+  nodeName?: string;
+};
+
+export type LangGraphAgentAction = Action<any> & {
+  langGraphAgentHandler: (params: LangGraphAgentHandlerParams) => Promise<Observable<RuntimeEvent>>;
+};
+
+export function isLangGraphAgentAction(action: Action<any>): action is LangGraphAgentAction {
+  if (!action) {
+    return false;
+  }
+  return typeof (action as LangGraphAgentAction).langGraphAgentHandler === "function";
+}
+
+function createHeaders(
+  onBeforeRequest: RemoteActionDefinition["onBeforeRequest"],
+  graphqlContext: GraphQLContext,
+) {
+  const headers = {
+    "Content-Type": "application/json",
+  };
+
+  if (onBeforeRequest) {
+    const { headers: additionalHeaders } = onBeforeRequest({ ctx: graphqlContext });
+    if (additionalHeaders) {
+      Object.assign(headers, additionalHeaders);
+    }
+  }
+
+  return headers;
+}
+
+async function fetchRemoteInfo({
+  url,
+  onBeforeRequest,
+  graphqlContext,
+  logger,
+}: {
+  url: string;
+  onBeforeRequest?: RemoteActionDefinition["onBeforeRequest"];
+  graphqlContext: GraphQLContext;
+  logger: Logger;
+}): Promise<any[]> {
+  logger.debug({ url }, "Fetching actions from url");
+  const headers = createHeaders(onBeforeRequest, graphqlContext);
+
+  const response = await fetch(`${url}/info`, {
+    method: "POST",
+    headers,
+    body: JSON.stringify({ properties: graphqlContext.properties }),
+  });
+
+  if (!response.ok) {
+    logger.error(
+      { url, status: response.status, body: await response.text() },
+      "Failed to fetch actions from url",
+    );
+    return [];
+  }
+
+  const json = await response.json();
+  logger.debug({ json }, "Fetched actions from url");
+  return json;
+}
+
+function constructRemoteActions({
+  json,
+  url,
+  onBeforeRequest,
+  graphqlContext,
+  logger,
+  messages,
+  agentStates,
+}: {
+  json: any[];
+  url: string;
+  onBeforeRequest?: RemoteActionDefinition["onBeforeRequest"];
+  graphqlContext: GraphQLContext;
+  logger: Logger;
+  messages: Message[];
+  agentStates?: AgentStateInput[];
+}): Action<any>[] {
+  const actions = json["actions"].map((action) => ({
+    name: action.name,
+    description: action.description,
+    parameters: action.parameters,
+    handler: async (args: any) => {
+      logger.debug({ actionName: action.name, args }, "Executing remote action");
+
+      const headers = createHeaders(onBeforeRequest, graphqlContext);
+      telemetry.capture("oss.runtime.remote_action_executed", {});
+
+      const response = await fetch(`${url}/actions/execute`, {
+        method: "POST",
+        headers,
+        body: JSON.stringify({
+          name: action.name,
+          arguments: args,
+          properties: graphqlContext.properties,
+        }),
+      });
+
+      if (!response.ok) {
+        logger.error(
+          { url, status: response.status, body: await response.text() },
+          "Failed to execute remote action",
+        );
+        return "Failed to execute remote action";
+      }
+
+      const requestResult = await response.json();
+
+      const result = requestResult["result"];
+      logger.debug({ actionName: action.name, result }, "Executed remote action");
+      return result;
+    },
+  }));
+
+  const agents = json["agents"].map((agent) => ({
+    name: agent.name,
+    description: agent.description,
+    parameters: [],
+    handler: async (_args: any) => {},
+
+    langGraphAgentHandler: async ({
+      name,
+      actionInputsWithoutAgents,
+      threadId,
+      nodeName,
+    }: LangGraphAgentHandlerParams): Promise<Observable<RuntimeEvent>> => {
+      logger.debug({ actionName: agent.name }, "Executing remote agent");
+
+      const headers = createHeaders(onBeforeRequest, graphqlContext);
+      telemetry.capture("oss.runtime.remote_action_executed", {});
+
+      let state = {};
+      if (agentStates) {
+        const jsonState = agentStates.find((state) => state.agentName === name)?.state;
+        if (jsonState) {
+          state = JSON.parse(jsonState);
+        }
+      }
+
+      const response = await fetch(`${url}/agents/execute`, {
+        method: "POST",
+        headers,
+        body: JSON.stringify({
+          name,
+          threadId,
+          nodeName,
+          messages,
+          state,
+          properties: graphqlContext.properties,
+          actions: actionInputsWithoutAgents.map((action) => ({
+            name: action.name,
+            description: action.description,
+            parameters: JSON.parse(action.jsonSchema),
+          })),
+        }),
+      });
+
+      if (!response.ok) {
+        logger.error(
+          { url, status: response.status, body: await response.text() },
+          "Failed to execute remote agent",
+        );
+        throw new Error("Failed to execute remote agent");
+      }
+
+      const eventSource = new RemoteLangGraphEventSource();
+      eventSource.streamResponse(response);
+      return eventSource.processLangGraphEvents();
+    },
+  }));
+
+  return [...actions, ...agents];
+}
+
+export async function setupRemoteActions({
+  remoteActionDefinitions,
+  graphqlContext,
+  messages,
+  agentStates,
+}: {
+  remoteActionDefinitions: RemoteActionDefinition[];
+  graphqlContext: GraphQLContext;
+  messages: Message[];
+  agentStates?: AgentStateInput[];
+}): Promise<Action[]> {
+  const logger = graphqlContext.logger.child({ component: "remote-actions.fetchRemoteActions" });
+  logger.debug({ remoteActionDefinitions }, "Fetching remote actions");
+
+  // Remove duplicates of remoteActionDefinitions.url
+  const filtered = remoteActionDefinitions.filter(
+    (value, index, self) => index === self.findIndex((t) => t.url === value.url),
+  );
+
+  const result = await Promise.all(
+    filtered.map(async (actionDefinition) => {
+      const json = await fetchRemoteInfo({
+        url: actionDefinition.url,
+        onBeforeRequest: actionDefinition.onBeforeRequest,
+        graphqlContext,
+        logger: logger.child({ component: "remote-actions.fetchActionsFromUrl", actionDefinition }),
+      });
+      return constructRemoteActions({
+        json,
+        messages,
+        url: actionDefinition.url,
+        onBeforeRequest: actionDefinition.onBeforeRequest,
+        graphqlContext,
+        logger: logger.child({ component: "remote-actions.constructActions", actionDefinition }),
+        agentStates,
+      });
+    }),
+  );
+
+  return result.flat();
+}

package/src/service-adapters/conversion.ts

@@ -3,6 +3,7 @@ import {
   Message,
   ResultMessage,
   TextMessage,
+  AgentStateMessage,
 } from "../graphql/types/converted";
 import { MessageInput } from "../graphql/inputs/message.input";
 import { plainToInstance } from "class-transformer";
@@ -40,6 +41,21 @@ export function convertGqlInputToMessages(inputMessages: MessageInput[]): Messag
           result: message.resultMessage.result,
         }),
       );
+    } else if (message.agentStateMessage) {
+      messages.push(
+        plainToInstance(AgentStateMessage, {
+          id: message.id,
+          threadId: message.agentStateMessage.threadId,
+          createdAt: message.createdAt,
+          agentName: message.agentStateMessage.agentName,
+          nodeName: message.agentStateMessage.nodeName,
+          runId: message.agentStateMessage.runId,
+          active: message.agentStateMessage.active,
+          role: message.agentStateMessage.role,
+          state: JSON.parse(message.agentStateMessage.state),
+          running: message.agentStateMessage.running,
+        }),
+      );
     }
   }
 

package/src/service-adapters/events.ts

@@ -1,7 +1,20 @@
 import { Action } from "@copilotkit/shared";
-import { of, concat, map, scan, concatMap, ReplaySubject, Subject, firstValueFrom } from "rxjs";
+import {
+  of,
+  concat,
+  map,
+  scan,
+  concatMap,
+  ReplaySubject,
+  Subject,
+  firstValueFrom,
+  from,
+} from "rxjs";
 import { streamLangChainResponse } from "./langchain/utils";
 import { GuardrailsResult } from "../graphql/types/guardrails-result.type";
+import telemetry from "../lib/telemetry-client";
+import { isLangGraphAgentAction } from "../lib/runtime/remote-actions";
+import { ActionInput } from "../graphql/inputs/action.input";
 
 export enum RuntimeEventTypes {
   TextMessageStart = "TextMessageStart",
@@ -11,9 +24,10 @@ export enum RuntimeEventTypes {
   ActionExecutionArgs = "ActionExecutionArgs",
   ActionExecutionEnd = "ActionExecutionEnd",
   ActionExecutionResult = "ActionExecutionResult",
+  AgentStateMessage = "AgentStateMessage",
 }
 
-type FunctionCallScope = "client" | "server";
+type FunctionCallScope = "client" | "server" | "passThrough";
 
 export type RuntimeEvent =
   | { type: RuntimeEventTypes.TextMessageStart; messageId: string }
@@ -35,6 +49,17 @@ export type RuntimeEvent =
       actionName: string;
       actionExecutionId: string;
       result: string;
+    }
+  | {
+      type: RuntimeEventTypes.AgentStateMessage;
+      threadId: string;
+      agentName: string;
+      nodeName: string;
+      runId: string;
+      active: boolean;
+      role: string;
+      state: string;
+      running: boolean;
     };
 
 interface RuntimeEventWithState {
@@ -100,6 +125,29 @@ export class RuntimeEventSubject extends ReplaySubject<RuntimeEvent> {
       result,
     });
   }
+
+  sendAgentStateMessage(
+    threadId: string,
+    agentName: string,
+    nodeName: string,
+    runId: string,
+    active: boolean,
+    role: string,
+    state: string,
+    running: boolean,
+  ) {
+    this.next({
+      type: RuntimeEventTypes.AgentStateMessage,
+      threadId,
+      agentName,
+      nodeName,
+      runId,
+      active,
+      role,
+      state,
+      running,
+    });
+  }
 }
 
 export class RuntimeEventSource {
@@ -110,12 +158,14 @@ export class RuntimeEventSource {
     this.callback = callback;
   }
 
-  process({
-    serversideActions,
+  processRuntimeEvents({
+    serverSideActions,
     guardrailsResult$,
+    actionInputsWithoutAgents,
   }: {
-    serversideActions: Action<any>[];
+    serverSideActions: Action<any>[];
     guardrailsResult$?: Subject<GuardrailsResult>;
+    actionInputsWithoutAgents: ActionInput[];
   }) {
     this.callback(this.eventStream$).catch((error) => {
       console.error("Error in event source callback", error);
@@ -124,27 +174,35 @@ export class RuntimeEventSource {
       // mark tools for server side execution
       map((event) => {
         if (event.type === RuntimeEventTypes.ActionExecutionStart) {
-          event.scope = serversideActions.find((action) => action.name === event.actionName)
-            ? "server"
-            : "client";
+          if (event.scope !== "passThrough") {
+            event.scope = serverSideActions.find((action) => action.name === event.actionName)
+              ? "server"
+              : "client";
+          }
         }
         return event;
       }),
       // track state
       scan(
         (acc, event) => {
+          // It seems like this is needed so that rxjs recognizes the object has changed
+          // This fixes an issue where action were executed multiple times
+          // Not investigating further for now (Markus)
+          acc = { ...acc };
+
           if (event.type === RuntimeEventTypes.ActionExecutionStart) {
             acc.callActionServerSide = event.scope === "server";
             acc.args = "";
             acc.actionExecutionId = event.actionExecutionId;
             if (acc.callActionServerSide) {
-              acc.action = serversideActions.find((action) => action.name === event.actionName);
+              acc.action = serverSideActions.find((action) => action.name === event.actionName);
             }
           } else if (event.type === RuntimeEventTypes.ActionExecutionArgs) {
             acc.args += event.args;
           }
 
           acc.event = event;
+
           return acc;
         },
         {
@@ -167,9 +225,12 @@ export class RuntimeEventSource {
             eventWithState.action!,
             eventWithState.args,
             eventWithState.actionExecutionId,
+            actionInputsWithoutAgents,
           ).catch((error) => {
             console.error(error);
           });
+
+          telemetry.capture("oss.runtime.server_action_executed", {});
           return concat(of(eventWithState.event!), toolCallEventStream$);
         } else {
           return of(eventWithState.event!);
@@ -185,6 +246,7 @@ async function executeAction(
   action: Action<any>,
   actionArguments: string,
   actionExecutionId: string,
+  actionInputsWithoutAgents: ActionInput[],
 ) {
   if (guardrailsResult$) {
     const { status } = await firstValueFrom(guardrailsResult$);
@@ -201,15 +263,35 @@ async function executeAction(
     args = JSON.parse(actionArguments);
   }
 
-  // call the function
-  const result = await action.handler(args);
-
-  await streamLangChainResponse({
-    result,
-    eventStream$,
-    actionExecution: {
+  // handle LangGraph agents
+  if (isLangGraphAgentAction(action)) {
+    eventStream$.sendActionExecutionResult(
+      actionExecutionId,
+      action.name,
+      `${action.name} agent started`,
+    );
+    const stream = await action.langGraphAgentHandler({
       name: action.name,
-      id: actionExecutionId,
-    },
-  });
+      actionInputsWithoutAgents,
+    });
+
+    // forward to eventStream$
+    from(stream).subscribe({
+      next: (event) => eventStream$.next(event),
+      error: (err) => console.error("Error in stream", err),
+      complete: () => eventStream$.complete(),
+    });
+  } else {
+    // call the function
+    const result = await action.handler?.(args);
+
+    await streamLangChainResponse({
+      result,
+      eventStream$,
+      actionExecution: {
+        name: action.name,
+        id: actionExecutionId,
+      },
+    });
+  }
 }

package/src/service-adapters/groq/groq-adapter.ts

@@ -41,11 +41,22 @@ export interface GroqAdapterParams {
    * The model to use.
    */
   model?: string;
+
+  /**
+   * Whether to disable parallel tool calls.
+   * You can disable parallel tool calls to force the model to execute tool calls sequentially.
+   * This is useful if you want to execute tool calls in a specific order so that the state changes
+   * introduced by one tool call are visible to the next tool call. (i.e. new actions or readables)
+   *
+   * @default false
+   */
+  disableParallelToolCalls?: boolean;
 }
 
 export class GroqAdapter implements CopilotServiceAdapter {
   private model: string = DEFAULT_MODEL;
 
+  private disableParallelToolCalls: boolean = false;
   private _groq: Groq;
   public get groq(): Groq {
     return this._groq;
@@ -56,6 +67,7 @@ export class GroqAdapter implements CopilotServiceAdapter {
     if (params?.model) {
       this.model = params.model;
     }
+    this.disableParallelToolCalls = params?.disableParallelToolCalls || false;
   }
 
   async process(
@@ -91,6 +103,7 @@ export class GroqAdapter implements CopilotServiceAdapter {
       }),
       ...(forwardedParameters?.stop && { stop: forwardedParameters.stop }),
       ...(toolChoice && { tool_choice: toolChoice }),
+      ...(this.disableParallelToolCalls && { parallel_tool_calls: false }),
     });
 
     eventSource.stream(async (eventStream$) => {

package/src/service-adapters/openai/openai-adapter.ts

@@ -45,11 +45,22 @@ export interface OpenAIAdapterParams {
    * The model to use.
    */
   model?: string;
+
+  /**
+   * Whether to disable parallel tool calls.
+   * You can disable parallel tool calls to force the model to execute tool calls sequentially.
+   * This is useful if you want to execute tool calls in a specific order so that the state changes
+   * introduced by one tool call are visible to the next tool call. (i.e. new actions or readables)
+   *
+   * @default false
+   */
+  disableParallelToolCalls?: boolean;
 }
 
 export class OpenAIAdapter implements CopilotServiceAdapter {
   private model: string = DEFAULT_MODEL;
 
+  private disableParallelToolCalls: boolean = false;
   private _openai: OpenAI;
   public get openai(): OpenAI {
     return this._openai;
@@ -60,6 +71,7 @@ export class OpenAIAdapter implements CopilotServiceAdapter {
     if (params?.model) {
       this.model = params.model;
     }
+    this.disableParallelToolCalls = params?.disableParallelToolCalls || false;
   }
 
   async process(
@@ -94,6 +106,7 @@ export class OpenAIAdapter implements CopilotServiceAdapter {
       ...(forwardedParameters?.maxTokens && { max_tokens: forwardedParameters.maxTokens }),
       ...(forwardedParameters?.stop && { stop: forwardedParameters.stop }),
       ...(toolChoice && { tool_choice: toolChoice }),
+      ...(this.disableParallelToolCalls && { parallel_tool_calls: false }),
     });
 
     eventSource.stream(async (eventStream$) => {

package/src/service-adapters/openai/openai-assistant-adapter.ts

@@ -65,6 +65,16 @@ export interface OpenAIAssistantAdapterParams {
    * @default true
    */
   fileSearchEnabled?: boolean;
+
+  /**
+   * Whether to disable parallel tool calls.
+   * You can disable parallel tool calls to force the model to execute tool calls sequentially.
+   * This is useful if you want to execute tool calls in a specific order so that the state changes
+   * introduced by one tool call are visible to the next tool call. (i.e. new actions or readables)
+   *
+   * @default false
+   */
+  disableParallelToolCalls?: boolean;
 }
 
 export class OpenAIAssistantAdapter implements CopilotServiceAdapter {
@@ -72,12 +82,14 @@ export class OpenAIAssistantAdapter implements CopilotServiceAdapter {
   private codeInterpreterEnabled: boolean;
   private assistantId: string;
   private fileSearchEnabled: boolean;
+  private disableParallelToolCalls: boolean;
 
   constructor(params: OpenAIAssistantAdapterParams) {
     this.openai = params.openai || new OpenAI({});
     this.codeInterpreterEnabled = params.codeInterpreterEnabled === false || true;
     this.fileSearchEnabled = params.fileSearchEnabled === false || true;
     this.assistantId = params.assistantId;
+    this.disableParallelToolCalls = params?.disableParallelToolCalls || false;
   }
 
   async process(
@@ -154,6 +166,7 @@ export class OpenAIAssistantAdapter implements CopilotServiceAdapter {
 
     const stream = this.openai.beta.threads.runs.submitToolOutputsStream(threadId, runId, {
       tool_outputs: toolOutputs,
+      ...(this.disableParallelToolCalls && { parallel_tool_calls: false }),
     });
 
     await this.streamResponse(stream, eventSource);
@@ -206,6 +219,7 @@ export class OpenAIAssistantAdapter implements CopilotServiceAdapter {
       ...(forwardedParameters?.maxTokens && {
         max_completion_tokens: forwardedParameters.maxTokens,
       }),
+      ...(this.disableParallelToolCalls && { parallel_tool_calls: false }),
     });
 
     await this.streamResponse(stream, eventSource);