@copilotkit/runtime 1.4.6 → 1.4.8-coagents-v0-3.1

package/src/service-adapters/events.ts

@@ -15,6 +15,8 @@ import { GuardrailsResult } from "../graphql/types/guardrails-result.type";
 import telemetry from "../lib/telemetry-client";
 import { isLangGraphAgentAction } from "../lib/runtime/remote-actions";
 import { ActionInput } from "../graphql/inputs/action.input";
+import { ActionExecutionMessage, ResultMessage } from "../graphql/types/converted";
+import { plainToInstance } from "class-transformer";
 
 export enum RuntimeEventTypes {
   TextMessageStart = "TextMessageStart",
@@ -27,23 +29,22 @@ export enum RuntimeEventTypes {
   AgentStateMessage = "AgentStateMessage",
 }
 
-type FunctionCallScope = "client" | "server" | "passThrough";
-
 export type RuntimeEvent =
-  | { type: RuntimeEventTypes.TextMessageStart; messageId: string }
+  | { type: RuntimeEventTypes.TextMessageStart; messageId: string; parentMessageId?: string }
   | {
       type: RuntimeEventTypes.TextMessageContent;
+      messageId: string;
       content: string;
     }
-  | { type: RuntimeEventTypes.TextMessageEnd }
+  | { type: RuntimeEventTypes.TextMessageEnd; messageId: string }
   | {
       type: RuntimeEventTypes.ActionExecutionStart;
       actionExecutionId: string;
       actionName: string;
-      scope?: FunctionCallScope;
+      parentMessageId?: string;
     }
-  | { type: RuntimeEventTypes.ActionExecutionArgs; args: string }
-  | { type: RuntimeEventTypes.ActionExecutionEnd }
+  | { type: RuntimeEventTypes.ActionExecutionArgs; actionExecutionId: string; args: string }
+  | { type: RuntimeEventTypes.ActionExecutionEnd; actionExecutionId: string }
   | {
       type: RuntimeEventTypes.ActionExecutionResult;
       actionName: string;
@@ -77,47 +78,86 @@ export class RuntimeEventSubject extends ReplaySubject<RuntimeEvent> {
     super();
   }
 
-  sendTextMessageStart(messageId: string) {
-    this.next({ type: RuntimeEventTypes.TextMessageStart, messageId });
+  sendTextMessageStart({
+    messageId,
+    parentMessageId,
+  }: {
+    messageId: string;
+    parentMessageId?: string;
+  }) {
+    this.next({ type: RuntimeEventTypes.TextMessageStart, messageId, parentMessageId });
   }
 
-  sendTextMessageContent(content: string) {
-    this.next({ type: RuntimeEventTypes.TextMessageContent, content });
+  sendTextMessageContent({ messageId, content }: { messageId: string; content: string }) {
+    this.next({ type: RuntimeEventTypes.TextMessageContent, content, messageId });
   }
 
-  sendTextMessageEnd() {
-    this.next({ type: RuntimeEventTypes.TextMessageEnd });
+  sendTextMessageEnd({ messageId }: { messageId: string }) {
+    this.next({ type: RuntimeEventTypes.TextMessageEnd, messageId });
   }
 
   sendTextMessage(messageId: string, content: string) {
-    this.sendTextMessageStart(messageId);
-    this.sendTextMessageContent(content);
-    this.sendTextMessageEnd();
+    this.sendTextMessageStart({ messageId });
+    this.sendTextMessageContent({ messageId, content });
+    this.sendTextMessageEnd({ messageId });
   }
 
-  sendActionExecutionStart(actionExecutionId: string, actionName: string) {
+  sendActionExecutionStart({
+    actionExecutionId,
+    actionName,
+    parentMessageId,
+  }: {
+    actionExecutionId: string;
+    actionName: string;
+    parentMessageId?: string;
+  }) {
     this.next({
       type: RuntimeEventTypes.ActionExecutionStart,
       actionExecutionId,
       actionName,
+      parentMessageId,
     });
   }
 
-  sendActionExecutionArgs(args: string) {
-    this.next({ type: RuntimeEventTypes.ActionExecutionArgs, args });
+  sendActionExecutionArgs({
+    actionExecutionId,
+    args,
+  }: {
+    actionExecutionId: string;
+    args: string;
+  }) {
+    this.next({ type: RuntimeEventTypes.ActionExecutionArgs, args, actionExecutionId });
   }
 
-  sendActionExecutionEnd() {
-    this.next({ type: RuntimeEventTypes.ActionExecutionEnd });
+  sendActionExecutionEnd({ actionExecutionId }: { actionExecutionId: string }) {
+    this.next({ type: RuntimeEventTypes.ActionExecutionEnd, actionExecutionId });
   }
 
-  sendActionExecution(actionExecutionId: string, toolName: string, args: string) {
-    this.sendActionExecutionStart(actionExecutionId, toolName);
-    this.sendActionExecutionArgs(args);
-    this.sendActionExecutionEnd();
+  sendActionExecution({
+    actionExecutionId,
+    actionName,
+    args,
+    parentMessageId,
+  }: {
+    actionExecutionId: string;
+    actionName: string;
+    args: string;
+    parentMessageId?: string;
+  }) {
+    this.sendActionExecutionStart({ actionExecutionId, actionName, parentMessageId });
+    this.sendActionExecutionArgs({ actionExecutionId, args });
+    this.sendActionExecutionEnd({ actionExecutionId });
   }
 
-  sendActionExecutionResult(actionExecutionId: string, actionName: string, result: string) {
+  sendActionExecutionResult({
+    actionExecutionId,
+    actionName,
+    result,
+  }: {
+    actionExecutionId: string;
+    actionName: string;
+    result: string;
+  }) {
     this.next({
       type: RuntimeEventTypes.ActionExecutionResult,
       actionName,
@@ -126,16 +166,25 @@ export class RuntimeEventSubject extends ReplaySubject<RuntimeEvent> {
     });
   }
 
-  sendAgentStateMessage(
-    threadId: string,
-    agentName: string,
-    nodeName: string,
-    runId: string,
-    active: boolean,
-    role: string,
-    state: string,
-    running: boolean,
-  ) {
+  sendAgentStateMessage({
+    threadId,
+    agentName,
+    nodeName,
+    runId,
+    active,
+    role,
+    state,
+    running,
+  }: {
+    threadId: string;
+    agentName: string;
+    nodeName: string;
+    runId: string;
+    active: boolean;
+    role: string;
+    state: string;
+    running: boolean;
+  }) {
     this.next({
       type: RuntimeEventTypes.AgentStateMessage,
       threadId,
@@ -183,17 +232,6 @@ export class RuntimeEventSource {
       this.sendErrorMessageToChat();
     });
     return this.eventStream$.pipe(
-      // mark tools for server side execution
-      map((event) => {
-        if (event.type === RuntimeEventTypes.ActionExecutionStart) {
-          if (event.scope !== "passThrough") {
-            event.scope = serverSideActions.find((action) => action.name === event.actionName)
-              ? "server"
-              : "client";
-          }
-        }
-        return event;
-      }),
       // track state
       scan(
         (acc, event) => {
@@ -203,7 +241,8 @@ export class RuntimeEventSource {
           acc = { ...acc };
 
           if (event.type === RuntimeEventTypes.ActionExecutionStart) {
-            acc.callActionServerSide = event.scope === "server";
+            acc.callActionServerSide =
+              serverSideActions.find((action) => action.name === event.actionName) !== undefined;
             acc.args = "";
             acc.actionExecutionId = event.actionExecutionId;
             if (acc.callActionServerSide) {
@@ -281,14 +320,34 @@ async function executeAction(
 
   // handle LangGraph agents
   if (isLangGraphAgentAction(action)) {
-    eventStream$.sendActionExecutionResult(
+    const result = `${action.name} agent started`;
+
+    const agentExecution = plainToInstance(ActionExecutionMessage, {
+      id: actionExecutionId,
+      createdAt: new Date(),
+      name: action.name,
+      arguments: JSON.parse(actionArguments),
+      parentMessageId: actionExecutionId,
+    });
+
+    const agentExecutionResult = plainToInstance(ResultMessage, {
+      id: "result-" + actionExecutionId,
+      createdAt: new Date(),
       actionExecutionId,
-      action.name,
-      `${action.name} agent started`,
-    );
+      actionName: action.name,
+      result,
+    });
+
+    eventStream$.sendActionExecutionResult({
+      actionExecutionId,
+      actionName: action.name,
+      result,
+    });
+
     const stream = await action.langGraphAgentHandler({
       name: action.name,
       actionInputsWithoutAgents,
+      additionalMessages: [agentExecution, agentExecutionResult],
     });
 
     // forward to eventStream$

package/src/service-adapters/experimental/ollama/ollama-adapter.ts

@@ -58,11 +58,15 @@ export class ExperimentalOllamaAdapter implements CopilotServiceAdapter {
     const _stream = await ollama.stream(contents); // [TODO] role info is dropped...
 
     eventSource.stream(async (eventStream$) => {
-      eventStream$.sendTextMessageStart(randomId());
+      const currentMessageId = randomId();
+      eventStream$.sendTextMessageStart({ messageId: currentMessageId });
       for await (const chunkText of _stream) {
-        eventStream$.sendTextMessageContent(chunkText);
+        eventStream$.sendTextMessageContent({
+          messageId: currentMessageId,
+          content: chunkText,
+        });
       }
-      eventStream$.sendTextMessageEnd();
+      eventStream$.sendTextMessageEnd({ messageId: currentMessageId });
       // we may need to add this later.. [nc]
       // let calls = (await result.response).functionCalls();
 

package/src/service-adapters/groq/groq-adapter.ts

@@ -106,6 +106,9 @@ export class GroqAdapter implements CopilotServiceAdapter {
 
     eventSource.stream(async (eventStream$) => {
       let mode: "function" | "message" | null = null;
+      let currentMessageId: string;
+      let currentToolCallId: string;
+
       for await (const chunk of stream) {
         const toolCall = chunk.choices[0].delta.tool_calls?.[0];
         const content = chunk.choices[0].delta.content;
@@ -115,36 +118,48 @@ export class GroqAdapter implements CopilotServiceAdapter {
         // If toolCall?.id is defined, it means a new tool call starts.
         if (mode === "message" && toolCall?.id) {
           mode = null;
-          eventStream$.sendTextMessageEnd();
+          eventStream$.sendTextMessageEnd({ messageId: currentMessageId });
         } else if (mode === "function" && (toolCall === undefined || toolCall?.id)) {
           mode = null;
-          eventStream$.sendActionExecutionEnd();
+          eventStream$.sendActionExecutionEnd({ actionExecutionId: currentToolCallId });
         }
 
         // If we send a new message type, send the appropriate start event.
         if (mode === null) {
           if (toolCall?.id) {
             mode = "function";
-            eventStream$.sendActionExecutionStart(toolCall!.id, toolCall!.function!.name);
+            currentToolCallId = toolCall!.id;
+            eventStream$.sendActionExecutionStart({
+              actionExecutionId: currentToolCallId,
+              actionName: toolCall!.function!.name,
+              parentMessageId: chunk.id,
+            });
           } else if (content) {
             mode = "message";
-            eventStream$.sendTextMessageStart(chunk.id);
+            currentMessageId = chunk.id;
+            eventStream$.sendTextMessageStart({ messageId: currentMessageId });
           }
         }
 
         // send the content events
         if (mode === "message" && content) {
-          eventStream$.sendTextMessageContent(content);
+          eventStream$.sendTextMessageContent({
+            messageId: currentMessageId,
+            content,
+          });
         } else if (mode === "function" && toolCall?.function?.arguments) {
-          eventStream$.sendActionExecutionArgs(toolCall.function.arguments);
+          eventStream$.sendActionExecutionArgs({
+            actionExecutionId: currentToolCallId,
+            args: toolCall.function.arguments,
+          });
         }
       }
 
       // send the end events
       if (mode === "message") {
-        eventStream$.sendTextMessageEnd();
+        eventStream$.sendTextMessageEnd({ messageId: currentMessageId });
       } else if (mode === "function") {
-        eventStream$.sendActionExecutionEnd();
+        eventStream$.sendActionExecutionEnd({ actionExecutionId: currentToolCallId });
       }
 
       eventStream$.complete();

package/src/service-adapters/langchain/utils.ts

@@ -97,11 +97,11 @@ function maybeSendActionExecutionResultIsMessage(
   // language models need a result after the function call
   // we simply let them know that we are sending a message
   if (actionExecution) {
-    eventStream$.sendActionExecutionResult(
-      actionExecution.id,
-      actionExecution.name,
-      "Sending a message",
-    );
+    eventStream$.sendActionExecutionResult({
+      actionExecutionId: actionExecution.id,
+      actionName: actionExecution.name,
+      result: "Sending a message",
+    });
   }
 }
 
@@ -120,7 +120,11 @@ export async function streamLangChainResponse({
       eventStream$.sendTextMessage(randomId(), result);
     } else {
       // Send as a result
-      eventStream$.sendActionExecutionResult(actionExecution.id, actionExecution.name, result);
+      eventStream$.sendActionExecutionResult({
+        actionExecutionId: actionExecution.id,
+        actionName: actionExecution.name,
+        result: result,
+      });
     }
   }
 
@@ -133,11 +137,11 @@ export async function streamLangChainResponse({
       eventStream$.sendTextMessage(randomId(), result.content as string);
     }
     for (const toolCall of result.tool_calls) {
-      eventStream$.sendActionExecution(
-        toolCall.id || randomId(),
-        toolCall.name,
-        JSON.stringify(toolCall.args),
-      );
+      eventStream$.sendActionExecution({
+        actionExecutionId: toolCall.id || randomId(),
+        actionName: toolCall.name,
+        args: JSON.stringify(toolCall.args),
+      });
     }
   }
 
@@ -151,11 +155,11 @@ export async function streamLangChainResponse({
     }
     if (result.lc_kwargs?.tool_calls) {
       for (const toolCall of result.lc_kwargs?.tool_calls) {
-        eventStream$.sendActionExecution(
-          toolCall.id || randomId(),
-          toolCall.name,
-          JSON.stringify(toolCall.args),
-        );
+        eventStream$.sendActionExecution({
+          actionExecutionId: toolCall.id || randomId(),
+          actionName: toolCall.name,
+          args: JSON.stringify(toolCall.args),
+        });
       }
     }
   }
@@ -168,6 +172,7 @@ export async function streamLangChainResponse({
     let reader = result.getReader();
 
     let mode: "function" | "message" | null = null;
+    let currentMessageId: string;
 
     const toolCallDetails = {
       name: null,
@@ -216,10 +221,10 @@ export async function streamLangChainResponse({
         // If toolCallName is defined, it means a new tool call starts.
         if (mode === "message" && (toolCallId || done)) {
           mode = null;
-          eventStream$.sendTextMessageEnd();
+          eventStream$.sendTextMessageEnd({ messageId: currentMessageId });
         } else if (mode === "function" && (!hasToolCall || done)) {
           mode = null;
-          eventStream$.sendActionExecutionEnd();
+          eventStream$.sendActionExecutionEnd({ actionExecutionId: toolCallId });
         }
 
         if (done) {
@@ -230,26 +235,39 @@ export async function streamLangChainResponse({
         if (mode === null) {
           if (hasToolCall && toolCallId && toolCallName) {
             mode = "function";
-            eventStream$.sendActionExecutionStart(toolCallId, toolCallName);
+            eventStream$.sendActionExecutionStart({
+              actionExecutionId: toolCallId,
+              actionName: toolCallName,
+              parentMessageId: value.lc_kwargs?.id,
+            });
           } else if (content) {
             mode = "message";
-            eventStream$.sendTextMessageStart(randomId());
+            currentMessageId = value.lc_kwargs?.id || randomId();
+            eventStream$.sendTextMessageStart({ messageId: currentMessageId });
           }
         }
 
         // send the content events
         if (mode === "message" && content) {
-          eventStream$.sendTextMessageContent(
-            Array.isArray(content) ? (content[0]?.text ?? "") : content,
-          );
+          eventStream$.sendTextMessageContent({
+            messageId: currentMessageId,
+            content: Array.isArray(content) ? (content[0]?.text ?? "") : content,
+          });
         } else if (mode === "function" && toolCallArgs) {
           // For calls of the same tool with different index, we seal last tool call and register a new one
           if (toolCallDetails.index !== toolCallDetails.prevIndex) {
-            eventStream$.sendActionExecutionEnd();
-            eventStream$.sendActionExecutionStart(toolCallId, toolCallName);
+            eventStream$.sendActionExecutionEnd({ actionExecutionId: toolCallId });
+            eventStream$.sendActionExecutionStart({
+              actionExecutionId: toolCallId,
+              actionName: toolCallName,
+              parentMessageId: value.lc_kwargs?.id,
+            });
             toolCallDetails.prevIndex = toolCallDetails.index;
           }
-          eventStream$.sendActionExecutionArgs(toolCallArgs);
+          eventStream$.sendActionExecutionArgs({
+            actionExecutionId: toolCallId,
+            args: toolCallArgs,
+          });
         }
       } catch (error) {
         console.error("Error reading from stream", error);
@@ -257,11 +275,11 @@ export async function streamLangChainResponse({
       }
     }
   } else if (actionExecution) {
-    eventStream$.sendActionExecutionResult(
-      actionExecution.id,
-      actionExecution.name,
-      encodeResult(result),
-    );
+    eventStream$.sendActionExecutionResult({
+      actionExecutionId: actionExecution.id,
+      actionName: actionExecution.name,
+      result: encodeResult(result),
+    });
   }
 
   // unsupported type

package/src/service-adapters/openai/openai-adapter.ts

@@ -140,7 +140,8 @@ export class OpenAIAdapter implements CopilotServiceAdapter {
 
     eventSource.stream(async (eventStream$) => {
       let mode: "function" | "message" | null = null;
-
+      let currentMessageId: string;
+      let currentToolCallId: string;
       for await (const chunk of stream) {
         if (chunk.choices.length === 0) {
           continue;
@@ -154,36 +155,48 @@ export class OpenAIAdapter implements CopilotServiceAdapter {
         // If toolCall?.id is defined, it means a new tool call starts.
         if (mode === "message" && toolCall?.id) {
           mode = null;
-          eventStream$.sendTextMessageEnd();
+          eventStream$.sendTextMessageEnd({ messageId: currentMessageId });
         } else if (mode === "function" && (toolCall === undefined || toolCall?.id)) {
           mode = null;
-          eventStream$.sendActionExecutionEnd();
+          eventStream$.sendActionExecutionEnd({ actionExecutionId: currentToolCallId });
         }
 
         // If we send a new message type, send the appropriate start event.
         if (mode === null) {
           if (toolCall?.id) {
             mode = "function";
-            eventStream$.sendActionExecutionStart(toolCall!.id, toolCall!.function!.name);
+            currentToolCallId = toolCall!.id;
+            eventStream$.sendActionExecutionStart({
+              actionExecutionId: currentToolCallId,
+              parentMessageId: chunk.id,
+              actionName: toolCall!.function!.name,
+            });
           } else if (content) {
             mode = "message";
-            eventStream$.sendTextMessageStart(chunk.id);
+            currentMessageId = chunk.id;
+            eventStream$.sendTextMessageStart({ messageId: currentMessageId });
           }
         }
 
         // send the content events
         if (mode === "message" && content) {
-          eventStream$.sendTextMessageContent(content);
+          eventStream$.sendTextMessageContent({
+            messageId: currentMessageId,
+            content: content,
+          });
         } else if (mode === "function" && toolCall?.function?.arguments) {
-          eventStream$.sendActionExecutionArgs(toolCall.function.arguments);
+          eventStream$.sendActionExecutionArgs({
+            actionExecutionId: currentToolCallId,
+            args: toolCall.function.arguments,
+          });
         }
       }
 
       // send the end events
       if (mode === "message") {
-        eventStream$.sendTextMessageEnd();
+        eventStream$.sendTextMessageEnd({ messageId: currentMessageId });
       } else if (mode === "function") {
-        eventStream$.sendActionExecutionEnd();
+        eventStream$.sendActionExecutionEnd({ actionExecutionId: currentToolCallId });
       }
 
       eventStream$.complete();

package/src/service-adapters/openai/openai-assistant-adapter.ts

@@ -226,22 +226,28 @@ export class OpenAIAssistantAdapter implements CopilotServiceAdapter {
   private async streamResponse(stream: AssistantStream, eventSource: RuntimeEventSource) {
     eventSource.stream(async (eventStream$) => {
       let inFunctionCall = false;
+      let currentMessageId: string;
+      let currentToolCallId: string;
 
       for await (const chunk of stream) {
         switch (chunk.event) {
           case "thread.message.created":
             if (inFunctionCall) {
-              eventStream$.sendActionExecutionEnd();
+              eventStream$.sendActionExecutionEnd({ actionExecutionId: currentToolCallId });
             }
-            eventStream$.sendTextMessageStart(chunk.data.id);
+            currentMessageId = chunk.data.id;
+            eventStream$.sendTextMessageStart({ messageId: currentMessageId });
             break;
           case "thread.message.delta":
             if (chunk.data.delta.content?.[0].type === "text") {
-              eventStream$.sendTextMessageContent(chunk.data.delta.content?.[0].text.value);
+              eventStream$.sendTextMessageContent({
+                messageId: currentMessageId,
+                content: chunk.data.delta.content?.[0].text.value,
+              });
             }
             break;
           case "thread.message.completed":
-            eventStream$.sendTextMessageEnd();
+            eventStream$.sendTextMessageEnd({ messageId: currentMessageId });
             break;
           case "thread.run.step.delta":
             let toolCallId: string | undefined;
@@ -258,18 +264,26 @@ export class OpenAIAssistantAdapter implements CopilotServiceAdapter {
 
             if (toolCallName && toolCallId) {
               if (inFunctionCall) {
-                eventStream$.sendActionExecutionEnd();
+                eventStream$.sendActionExecutionEnd({ actionExecutionId: currentToolCallId });
               }
               inFunctionCall = true;
-              eventStream$.sendActionExecutionStart(toolCallId, toolCallName);
+              currentToolCallId = toolCallId;
+              eventStream$.sendActionExecutionStart({
+                actionExecutionId: currentToolCallId,
+                parentMessageId: chunk.data.id,
+                actionName: toolCallName,
+              });
             } else if (toolCallArgs) {
-              eventStream$.sendActionExecutionArgs(toolCallArgs);
+              eventStream$.sendActionExecutionArgs({
+                actionExecutionId: currentToolCallId,
+                args: toolCallArgs,
+              });
             }
             break;
         }
       }
       if (inFunctionCall) {
-        eventStream$.sendActionExecutionEnd();
+        eventStream$.sendActionExecutionEnd({ actionExecutionId: currentToolCallId });
       }
       eventStream$.complete();
     });