@copilotkit/runtime 1.2.1 → 1.2.2-feat-runtime-remote-actions.0

package/src/agents/langgraph/event-source.ts

@@ -0,0 +1,222 @@
+import { ReplaySubject, scan, mergeMap } from "rxjs";
+import { LangGraphEvent, LangGraphEventTypes } from "./events";
+import { RuntimeEvent, RuntimeEventTypes } from "../../service-adapters/events";
+
+interface LangGraphEventWithState {
+  event: LangGraphEvent | null;
+  toolCallName: string | null;
+  toolCallId: string | null;
+  prevToolCallId: string | null;
+  messageId: string | null;
+  prevMessageId: string | null;
+}
+
+export class RemoteLangGraphEventSource {
+  private eventStream$ = new ReplaySubject<LangGraphEvent>();
+
+  async streamResponse(response: Response) {
+    const reader = response.body!.getReader();
+    const decoder = new TextDecoder();
+    let buffer = [];
+    const eventStream$ = this.eventStream$;
+
+    function flushBuffer() {
+      const currentBuffer = buffer.join("");
+      if (currentBuffer.trim().length === 0) {
+        return;
+      }
+      const parts = currentBuffer.split("\n");
+      if (parts.length === 0) {
+        return;
+      }
+
+      const lastPartIsComplete = currentBuffer.endsWith("\n");
+
+      // truncate buffer
+      buffer = [];
+
+      if (!lastPartIsComplete) {
+        // put back the last part
+        buffer.push(parts.pop());
+      }
+
+      parts
+        .map((part) => part.trim())
+        .filter((part) => part != "")
+        .forEach((part) => {
+          eventStream$.next(JSON.parse(part));
+        });
+    }
+
+    while (true) {
+      const { done, value } = await reader.read();
+
+      if (!done) {
+        buffer.push(decoder.decode(value, { stream: true }));
+      }
+
+      flushBuffer();
+
+      if (done) {
+        break;
+      }
+    }
+    eventStream$.complete();
+  }
+
+  processLangGraphEvents() {
+    return this.eventStream$.pipe(
+      scan(
+        (acc, event) => {
+          if (event.event === LangGraphEventTypes.OnChatModelStream) {
+            if (event.data?.chunk?.kwargs?.tool_call_chunks) {
+              acc.prevToolCallId = acc.toolCallId;
+              acc.toolCallId = event.data.chunk.kwargs?.id;
+              if (event.data.chunk.kwargs.tool_call_chunks[0]?.name) {
+                acc.toolCallName = event.data.chunk.kwargs.tool_call_chunks[0].name;
+              }
+            }
+            acc.prevMessageId = acc.messageId;
+            acc.messageId = event.data?.chunk?.kwargs?.id;
+          } else {
+            acc.prevToolCallId = acc.toolCallId;
+            acc.toolCallId = null;
+            acc.prevMessageId = acc.messageId;
+            acc.messageId = null;
+            acc.toolCallName = null;
+          }
+
+          acc.event = event;
+          return acc;
+        },
+        {
+          event: null,
+          toolCallId: null,
+          prevToolCallId: null,
+          messageId: null,
+          toolCallName: null,
+          prevMessageId: null,
+        } as LangGraphEventWithState,
+      ),
+      mergeMap((eventWithState): RuntimeEvent[] => {
+        const events: RuntimeEvent[] = [];
+
+        let shouldEmitMessages = false;
+        let shouldEmitToolCalls = false;
+
+        if (eventWithState.event.event == LangGraphEventTypes.OnChatModelStream) {
+          if (eventWithState.event.tags?.includes("copilotkit:emit-tool-calls")) {
+            shouldEmitToolCalls = true;
+          }
+          if (eventWithState.event.tags?.includes("copilotkit:emit-messages")) {
+            shouldEmitMessages = true;
+          }
+        }
+
+        // Tool call ended: emit ActionExecutionEnd
+        if (
+          eventWithState.prevToolCallId !== null &&
+          eventWithState.prevToolCallId !== eventWithState.toolCallId &&
+          shouldEmitToolCalls
+        ) {
+          events.push({
+            type: RuntimeEventTypes.ActionExecutionEnd,
+          });
+        }
+
+        // Message ended: emit TextMessageEnd
+        if (
+          eventWithState.prevMessageId !== null &&
+          eventWithState.prevMessageId !== eventWithState.messageId &&
+          shouldEmitMessages
+        ) {
+          events.push({
+            type: RuntimeEventTypes.TextMessageEnd,
+          });
+        }
+
+        switch (eventWithState.event!.event) {
+          case LangGraphEventTypes.OnCopilotKitStateSync:
+            events.push({
+              type: RuntimeEventTypes.AgentStateMessage,
+              threadId: eventWithState.event.thread_id,
+              role: eventWithState.event.role,
+              agentName: eventWithState.event.agent_name,
+              nodeName: eventWithState.event.node_name,
+              runId: eventWithState.event.run_id,
+              active: eventWithState.event.active,
+              state: JSON.stringify(eventWithState.event.state),
+              running: eventWithState.event.running,
+            });
+            break;
+          case LangGraphEventTypes.OnToolEnd:
+            // TODO-AGENTS: emit ActionExecutionResult when needed
+            // Need a special tool node for that?
+
+            // const result = eventWithState.event.data?.output?.kwargs?.content?.[0];
+            // const toolCallId = eventWithState.event.data?.output?.kwargs?.tool_call_id;
+            // const toolCallName = eventWithState.event.data?.output?.kwargs?.name;
+            // if (result && toolCallId && toolCallName) {
+            //   events.push({
+            //     type: RuntimeEventTypes.ActionExecutionResult,
+            //     actionExecutionId: toolCallId,
+            //     actionName: toolCallName,
+            //     result,
+            //   });
+            // }
+            break;
+          case LangGraphEventTypes.OnChatModelStream:
+            if (
+              eventWithState.toolCallId !== null &&
+              eventWithState.prevToolCallId !== eventWithState.toolCallId
+            ) {
+              if (shouldEmitToolCalls) {
+                events.push({
+                  type: RuntimeEventTypes.ActionExecutionStart,
+                  actionExecutionId: eventWithState.toolCallId,
+                  actionName: eventWithState.toolCallName,
+                  scope: "client",
+                });
+              }
+            }
+            // Message started: emit TextMessageStart
+            else if (
+              eventWithState.messageId !== null &&
+              eventWithState.prevMessageId !== eventWithState.messageId
+            ) {
+              if (shouldEmitMessages) {
+                events.push({
+                  type: RuntimeEventTypes.TextMessageStart,
+                  messageId: eventWithState.messageId,
+                });
+              }
+            }
+
+            const args = eventWithState.event.data?.chunk?.kwargs?.tool_call_chunks?.[0]?.args;
+            const content = eventWithState.event.data?.chunk?.kwargs?.content;
+
+            // Tool call args: emit ActionExecutionArgs
+            if (args) {
+              if (shouldEmitToolCalls) {
+                events.push({
+                  type: RuntimeEventTypes.ActionExecutionArgs,
+                  args,
+                });
+              }
+            }
+            // Message content: emit TextMessageContent
+            else if (eventWithState.messageId !== null && content) {
+              if (shouldEmitMessages) {
+                events.push({
+                  type: RuntimeEventTypes.TextMessageContent,
+                  content,
+                });
+              }
+            }
+            break;
+        }
+        return events;
+      }),
+    );
+  }
+}

package/src/agents/langgraph/events.ts

@@ -0,0 +1,309 @@
+export enum LangGraphEventTypes {
+  OnChainStart = "on_chain_start",
+  OnChainStream = "on_chain_stream",
+  OnChainEnd = "on_chain_end",
+  OnChatModelStart = "on_chat_model_start",
+  OnChatModelStream = "on_chat_model_stream",
+  OnChatModelEnd = "on_chat_model_end",
+  OnToolStart = "on_tool_start",
+  OnToolEnd = "on_tool_end",
+  OnCopilotKitStateSync = "on_copilotkit_state_sync",
+}
+
+type LangGraphOnCopilotKitStateSyncEvent = {
+  event: LangGraphEventTypes.OnCopilotKitStateSync;
+  thread_id: string;
+  agent_name: string;
+  node_name: string;
+  run_id: string;
+  active: boolean;
+  role: string;
+  state: any;
+  running: boolean;
+};
+
+type LangGraphOnChainStartEvent = {
+  event: LangGraphEventTypes.OnChainStart;
+  run_id: string;
+  name: string;
+  tags: string[];
+  metadata: { thread_id: string };
+  data: {
+    input: any;
+  };
+  parent_ids: string[];
+};
+
+type LangGraphOnChainEndEvent = {
+  event: LangGraphEventTypes.OnChainEnd;
+  name: string;
+  run_id: string;
+  tags: string[];
+  metadata: {
+    thread_id: string;
+    langgraph_step: number;
+    langgraph_node: string;
+    langgraph_triggers: string[];
+    langgraph_task_idx: number;
+    thread_ts: string;
+  };
+  data: {
+    input: any;
+    output: any;
+  };
+  parent_ids: string[];
+};
+
+type LangGraphOnChatModelStartEvent = {
+  event: LangGraphEventTypes.OnChatModelStart;
+  name: string;
+  run_id: string;
+  tags: string[];
+  metadata: {
+    thread_id: string;
+    langgraph_step: number;
+    langgraph_node: string;
+    langgraph_triggers: string[];
+    langgraph_task_idx: number;
+    thread_ts: string;
+    ls_provider: string;
+    ls_model_name: string;
+    ls_model_type: string;
+    ls_temperature: number;
+  };
+  data: {
+    input: {
+      messages: {
+        lc: number;
+        type: string;
+        id: string[];
+        kwargs: {
+          content: string;
+          type: string;
+          id: string;
+        };
+      }[][];
+    };
+  };
+  parent_ids: string[];
+};
+
+type LangGraphOnChatModelStreamEvent = {
+  event: LangGraphEventTypes.OnChatModelStream;
+  name: string;
+  run_id: string;
+  tags: string[];
+  metadata: {
+    thread_id: string;
+    langgraph_step: number;
+    langgraph_node: string;
+    langgraph_triggers: string[];
+    langgraph_task_idx: number;
+    thread_ts: string;
+    ls_provider: string;
+    ls_model_name: string;
+    ls_model_type: string;
+    ls_temperature: number;
+  };
+  data: {
+    chunk: {
+      lc: number;
+      type: string;
+      id: string[];
+      kwargs: {
+        content: string;
+        additional_kwargs: {
+          tool_calls: {
+            index: number;
+            id: string;
+            function: { arguments: string; name: string };
+            type: string;
+          }[];
+        };
+        type: string;
+        id: string;
+        tool_calls: { name: string; args: {}; id: string; type: string }[];
+        tool_call_chunks: {
+          name: string;
+          args: string;
+          id: string;
+          index: number;
+          type: string;
+        }[];
+        invalid_tool_calls: any[];
+      };
+    };
+  };
+  parent_ids: string[];
+};
+
+type LangGraphOnChatModelEndEvent = {
+  event: LangGraphEventTypes.OnChatModelEnd;
+  name: string;
+  run_id: string;
+  tags: string[];
+  metadata: {
+    thread_id: string;
+    langgraph_step: number;
+    langgraph_node: string;
+    langgraph_triggers: string[];
+    langgraph_task_idx: number;
+    thread_ts: string;
+    ls_provider: string;
+    ls_model_name: string;
+    ls_model_type: string;
+    ls_temperature: number;
+  };
+  data: {
+    input: any;
+    output: {
+      generations: {
+        text: string;
+        generation_info: {
+          finish_reason: string;
+          model_name: string;
+          system_fingerprint: string;
+        };
+        type: string;
+        message: {
+          lc: number;
+          type: string;
+          id: string[];
+          kwargs: {
+            content: string;
+            additional_kwargs: {
+              tool_calls: {
+                index: number;
+                id: string;
+                function: { arguments: string; name: string };
+                type: string;
+              }[];
+            };
+            response_metadata: {
+              finish_reason: string;
+              model_name: string;
+              system_fingerprint: string;
+            };
+            type: string;
+            id: string;
+            tool_calls: { name: string; args: { query: string }; id: string; type: string }[];
+            invalid_tool_calls: any[];
+          };
+        };
+      }[][];
+      llm_output: any;
+      run: any;
+    };
+  };
+  parent_ids: string[];
+};
+
+type LangGraphOnChainStreamEvent = {
+  event: LangGraphEventTypes.OnChainStream;
+  name: string;
+  run_id: string;
+  tags: string[];
+  metadata: {
+    thread_id: string;
+    langgraph_step?: number;
+    langgraph_node?: string;
+    langgraph_triggers?: string[];
+    langgraph_task_idx?: number;
+    thread_ts?: string;
+  };
+  data: {
+    chunk: {
+      messages: {
+        lc: number;
+        type: string;
+        id: string[];
+        kwargs: {
+          content: string;
+          additional_kwargs?: {
+            tool_calls?: {
+              index: number;
+              id: string;
+              function: { arguments: string; name: string };
+              type: string;
+            }[];
+          };
+          response_metadata?: {
+            finish_reason: string;
+            model_name: string;
+            system_fingerprint: string;
+          };
+          type: string;
+          id: string;
+          tool_calls?: { name: string; args: { query: string }; id: string; type: string }[];
+          invalid_tool_calls?: any[];
+        };
+      }[];
+    };
+  };
+  parent_ids: string[];
+};
+
+type LangGraphOnToolStartEvent = {
+  event: LangGraphEventTypes.OnToolStart;
+  name: string;
+  run_id: string;
+  tags: string[];
+  metadata: {
+    thread_id: string;
+    langgraph_step: number;
+    langgraph_node: string;
+    langgraph_triggers: string[];
+    langgraph_task_idx: number;
+    thread_ts: string;
+  };
+  data: {
+    input: {
+      query: string;
+    };
+  };
+  parent_ids: string[];
+};
+
+type LangGraphOnToolEndEvent = {
+  event: LangGraphEventTypes.OnToolEnd;
+  name: string;
+  run_id: string;
+  tags: string[];
+  metadata: {
+    thread_id: string;
+    langgraph_step: number;
+    langgraph_node: string;
+    langgraph_triggers: string[];
+    langgraph_task_idx: number;
+    thread_ts: string;
+  };
+  data: {
+    input: {
+      query: string;
+    };
+    output: {
+      lc: number;
+      type: string;
+      id: string[];
+      kwargs: {
+        content: string[];
+        type: string;
+        name: string;
+        tool_call_id: string;
+        status: string;
+      };
+    };
+  };
+  parent_ids: string[];
+};
+
+export type LangGraphEvent =
+  | LangGraphOnChainStartEvent
+  | LangGraphOnChainStreamEvent
+  | LangGraphOnChainEndEvent
+  | LangGraphOnChatModelStartEvent
+  | LangGraphOnChatModelStreamEvent
+  | LangGraphOnChatModelEndEvent
+  | LangGraphOnToolStartEvent
+  | LangGraphOnToolEndEvent
+  | LangGraphOnCopilotKitStateSyncEvent;

package/src/graphql/inputs/agent-session.input.ts

@@ -0,0 +1,13 @@
+import { Field, InputType } from "type-graphql";
+
+@InputType()
+export class AgentSessionInput {
+  @Field(() => String)
+  agentName: string;
+
+  @Field(() => String, { nullable: true })
+  threadId?: string;
+
+  @Field(() => String, { nullable: true })
+  nodeName?: string;
+}

package/src/graphql/inputs/agent-state.input.ts

@@ -0,0 +1,10 @@
+import { Field, InputType } from "type-graphql";
+
+@InputType()
+export class AgentStateInput {
+  @Field(() => String)
+  agentName: string;
+
+  @Field(() => String)
+  state: string;
+}

package/src/graphql/inputs/generate-copilot-response.input.ts

@@ -4,6 +4,8 @@ import { FrontendInput } from "./frontend.input";
 import { CloudInput } from "./cloud.input";
 import { CopilotRequestType } from "../types/enums";
 import { ForwardedParametersInput } from "./forwarded-parameters.input";
+import { AgentSessionInput } from "./agent-session.input";
+import { AgentStateInput } from "./agent-state.input";
 
 @InputType()
 export class GenerateCopilotResponseMetadataInput {
@@ -33,4 +35,13 @@ export class GenerateCopilotResponseInput {
 
   @Field(() => ForwardedParametersInput, { nullable: true })
   forwardedParameters?: ForwardedParametersInput;
+
+  @Field(() => AgentSessionInput, { nullable: true })
+  agentSession?: AgentSessionInput;
+
+  @Field(() => AgentStateInput, { nullable: true })
+  agentState?: AgentStateInput;
+
+  @Field(() => [AgentStateInput], { nullable: true })
+  agentStates?: AgentStateInput[];
 }

package/src/graphql/inputs/message.input.ts

@@ -14,6 +14,9 @@ export class MessageInput extends BaseMessage {
 
   @Field(() => ResultMessageInput, { nullable: true })
   resultMessage?: ResultMessageInput;
+
+  @Field(() => AgentStateMessageInput, { nullable: true })
+  agentStateMessage?: AgentStateMessageInput;
 }
 
 @InputType()
@@ -48,3 +51,30 @@ export class ResultMessageInput {
   @Field(() => String)
   result: string;
 }
+
+@InputType()
+export class AgentStateMessageInput {
+  @Field(() => String)
+  threadId: string;
+
+  @Field(() => String)
+  agentName: string;
+
+  @Field(() => MessageRole)
+  role: MessageRole;
+
+  @Field(() => String)
+  state: string;
+
+  @Field(() => Boolean)
+  running: boolean;
+
+  @Field(() => String)
+  nodeName: string;
+
+  @Field(() => String)
+  runId: string;
+
+  @Field(() => Boolean)
+  active: boolean;
+}