@copilotkit/runtime 1.3.16-mme-improve-langsmith.0 → 1.4.0-pre-1-4-0.0

package/src/lib/runtime/remote-lg-cloud-action.ts

@@ -0,0 +1,624 @@
+import { Client } from "@langchain/langgraph-sdk";
+import { randomUUID } from "node:crypto";
+import { parse as parsePartialJson } from "partial-json";
+import { Logger } from "pino";
+import { ActionInput } from "../../graphql/inputs/action.input";
+import { LangGraphCloudAgent, LangGraphCloudEndpoint } from "./remote-actions";
+import { CopilotRequestContextProperties } from "../integrations";
+import { Message, MessageType } from "../../graphql/types/converted";
+import { MessageRole } from "../../graphql/types/enums";
+
+type State = Record<string, any>;
+
+type ExecutionAction = Pick<ActionInput, "name" | "description"> & { parameters: string };
+
+interface ExecutionArgs extends Omit<LangGraphCloudEndpoint, "agents"> {
+  agent: LangGraphCloudAgent;
+  threadId: string;
+  nodeName: string;
+  messages: Message[];
+  state: State;
+  properties: CopilotRequestContextProperties;
+  actions: ExecutionAction[];
+  logger: Logger;
+}
+
+// The following types are our own definition to the messages accepted by LangGraph cloud, enhanced with some of our extra data.
+interface ToolCall {
+  id: string;
+  name: string;
+  args: Record<string, unknown>;
+}
+
+type BaseLangGraphCloudMessage = Omit<
+  Message,
+  | "isResultMessage"
+  | "isTextMessage"
+  | "isActionExecutionMessage"
+  | "isAgentStateMessage"
+  | "type"
+  | "createdAt"
+> & {
+  content: string;
+  role: MessageRole;
+  additional_kwargs?: Record<string, unknown>;
+  type: MessageType;
+};
+
+interface LangGraphCloudResultMessage extends BaseLangGraphCloudMessage {
+  tool_call_id: string;
+  name: string;
+}
+
+interface LangGraphCloudActionExecutionMessage extends BaseLangGraphCloudMessage {
+  tool_calls: ToolCall[];
+}
+
+type LangGraphCloudMessage =
+  | LangGraphCloudActionExecutionMessage
+  | LangGraphCloudResultMessage
+  | BaseLangGraphCloudMessage;
+
+export async function execute(args: ExecutionArgs): Promise<ReadableStream<Uint8Array>> {
+  return new ReadableStream({
+    async start(controller) {
+      try {
+        await streamEvents(controller, args);
+        controller.close();
+      } catch (err) {}
+    },
+  });
+}
+
+async function streamEvents(controller: ReadableStreamDefaultController, args: ExecutionArgs) {
+  const {
+    deploymentUrl,
+    langsmithApiKey,
+    threadId: agrsInitialThreadId,
+    agent,
+    nodeName: initialNodeName,
+    state: initialState,
+    messages,
+    actions,
+    logger,
+  } = args;
+
+  let nodeName = initialNodeName;
+  let state = initialState;
+  const { name, assistantId: initialAssistantId } = agent;
+
+  // TODO: deploymentUrl is not required in local development
+  const client = new Client({ apiUrl: deploymentUrl, apiKey: langsmithApiKey });
+  let initialThreadId = agrsInitialThreadId;
+  const wasInitiatedWithExistingThread = !!initialThreadId;
+  if (initialThreadId && initialThreadId.startsWith("ck-")) {
+    initialThreadId = initialThreadId.substring(3);
+  }
+
+  const assistants = await client.assistants.search();
+  const retrievedAssistant = assistants.find((a) => a.name === name);
+  const threadId = initialThreadId ?? randomUUID();
+  if (initialThreadId === threadId) {
+    await client.threads.get(threadId);
+  } else {
+    await client.threads.create({ threadId: threadId });
+  }
+
+  let agentState = { values: {} };
+  if (wasInitiatedWithExistingThread) {
+    agentState = await client.threads.getState(threadId);
+  }
+  const agentStateValues = agentState.values as State;
+  state.messages = agentStateValues.messages;
+  const mode = wasInitiatedWithExistingThread && nodeName != "__end__" ? "continue" : "start";
+  state = langGraphDefaultMergeState(state, formatMessages(messages), actions, name);
+
+  if (mode === "continue") {
+    await client.threads.updateState(threadId, { values: state, asNode: nodeName });
+  }
+
+  const assistantId = initialAssistantId ?? retrievedAssistant?.assistant_id;
+  if (!assistantId) {
+    console.error(`
+      No agent found for the agent name specified in CopilotKit provider
+      Please check your available agents or provide an agent ID in the LangGraph Cloud endpoint definition.\n
+      
+      These are the available agents: [${assistants.map((a) => `${a.name} (ID: ${a.assistant_id})`).join(", ")}]
+      `);
+    throw new Error("No agent id found");
+  }
+  const graphInfo = await client.assistants.getGraph(assistantId);
+  const streamInput = mode === "start" ? state : null;
+
+  let streamingStateExtractor = new StreamingStateExtractor([]);
+  let prevNodeName = null;
+  let emitIntermediateStateUntilEnd = null;
+  let shouldExit = null;
+  let externalRunId = null;
+
+  const streamResponse = client.runs.stream(threadId, assistantId, {
+    input: streamInput,
+    streamMode: ["events", "values"],
+  });
+
+  const emit = (message: string) => controller.enqueue(new TextEncoder().encode(message));
+
+  let latestStateValues = {};
+
+  for await (const chunk of streamResponse) {
+    if (!["events", "values", "error"].includes(chunk.event)) continue;
+
+    if (chunk.event === "error") {
+      logger.error(chunk, `Error event thrown: ${chunk.data.message}`);
+      throw new Error(`Error event thrown: ${chunk.data.message}`);
+    }
+
+    if (chunk.event === "values") {
+      latestStateValues = chunk.data;
+      continue;
+    }
+
+    const event = chunk.data;
+    const currentNodeName = event.name;
+    const eventType = event.event;
+    const runId = event.metadata.run_id;
+    externalRunId = runId;
+    const metadata = event.metadata;
+
+    shouldExit = shouldExit != null ? shouldExit : metadata["copilotkit:exit"];
+    const emitIntermediateState = metadata["copilotkit:emit-intermediate-state"];
+    const forceEmitIntermediateState = metadata["copilotkit:force-emit-intermediate-state"];
+    const manuallyEmitMessage = metadata["copilotkit:manually-emit-messages"];
+    const manuallyEmitToolCall = metadata["copilotkit:manually-emit-tool-calls"];
+    // we only want to update the node name under certain conditions
+    // since we don't need any internal node names to be sent to the frontend
+    if (graphInfo["nodes"].some((node) => node.id === currentNodeName)) {
+      nodeName = currentNodeName;
+    }
+
+    if (!nodeName) {
+      continue;
+    }
+
+    if (forceEmitIntermediateState) {
+      if (eventType === "on_chain_end") {
+        state = event.data.output;
+        emit(
+          getStateSyncEvent({
+            threadId,
+            runId,
+            agentName: agent.name,
+            nodeName,
+            state: event.data.output,
+            running: true,
+            active: true,
+          }),
+        );
+      }
+      continue;
+    }
+
+    if (manuallyEmitMessage) {
+      if (eventType === "on_chain_end") {
+        state = event.data.output;
+        emit(
+          JSON.stringify({
+            event: "on_copilotkit_emit_message",
+            message: event.data.output,
+            messageId: randomUUID(),
+            role: MessageRole.assistant,
+          }) + "\n",
+        );
+      }
+      continue;
+    }
+
+    if (manuallyEmitToolCall) {
+      if (eventType === "on_chain_end") {
+        state = event.data.output;
+        emit(
+          JSON.stringify({
+            event: "on_copilotkit_emit_tool_call",
+            name: event.data.output.name,
+            args: event.data.output.args,
+            id: event.data.output.id,
+          }) + "\n",
+        );
+      }
+      continue;
+    }
+
+    if (emitIntermediateState && emitIntermediateStateUntilEnd == null) {
+      emitIntermediateStateUntilEnd = nodeName;
+    }
+
+    if (emitIntermediateState && eventType === "on_chat_model_start") {
+      // reset the streaming state extractor
+      streamingStateExtractor = new StreamingStateExtractor(emitIntermediateState);
+    }
+
+    let updatedState = latestStateValues;
+
+    if (emitIntermediateState && eventType === "on_chat_model_stream") {
+      streamingStateExtractor.bufferToolCalls(event);
+    }
+
+    if (emitIntermediateStateUntilEnd !== null) {
+      updatedState = {
+        ...updatedState,
+        ...streamingStateExtractor.extractState(),
+      };
+    }
+
+    if (
+      !emitIntermediateState &&
+      currentNodeName === emitIntermediateStateUntilEnd &&
+      eventType === "on_chain_end"
+    ) {
+      // stop emitting function call state
+      emitIntermediateStateUntilEnd = null;
+    }
+
+    const exitingNode = nodeName === currentNodeName && eventType === "on_chain_end";
+
+    if (
+      JSON.stringify(updatedState) !== JSON.stringify(state) ||
+      prevNodeName != nodeName ||
+      exitingNode
+    ) {
+      state = updatedState;
+      prevNodeName = nodeName;
+      emit(
+        getStateSyncEvent({
+          threadId,
+          runId,
+          agentName: agent.name,
+          nodeName,
+          state,
+          running: true,
+          active: !exitingNode,
+        }),
+      );
+    }
+
+    emit(JSON.stringify(event) + "\n");
+  }
+
+  state = await client.threads.getState(threadId);
+  const isEndNode = state.next.length === 0;
+  nodeName = Object.keys(state.metadata.writes)[0];
+
+  emit(
+    getStateSyncEvent({
+      threadId,
+      runId: externalRunId,
+      agentName: agent.name,
+      nodeName: isEndNode ? "__end__" : nodeName,
+      state: state.values,
+      running: !shouldExit,
+      active: false,
+    }),
+  );
+
+  return Promise.resolve();
+}
+
+function getStateSyncEvent({
+  threadId,
+  runId,
+  agentName,
+  nodeName,
+  state,
+  running,
+  active,
+}: {
+  threadId: string;
+  runId: string;
+  agentName: string;
+  nodeName: string;
+  state: State;
+  running: boolean;
+  active: boolean;
+}): string {
+  const stateWithoutMessages = Object.keys(state).reduce((acc, key) => {
+    if (key !== "messages") {
+      acc[key] = state[key];
+    }
+    return acc;
+  }, {} as State);
+
+  return (
+    JSON.stringify({
+      event: "on_copilotkit_state_sync",
+      thread_id: threadId,
+      run_id: runId,
+      agent_name: agentName,
+      node_name: nodeName,
+      active: active,
+      state: stateWithoutMessages,
+      running: running,
+      role: "assistant",
+    }) + "\n"
+  );
+}
+
+class StreamingStateExtractor {
+  private emitIntermediateState: { [key: string]: any }[];
+  private toolCallBuffer: { [key: string]: string };
+  private currentToolCall: string | null;
+  private previouslyParsableState: { [key: string]: any };
+
+  constructor(emitIntermediateState: { [key: string]: any }[]) {
+    this.emitIntermediateState = emitIntermediateState;
+    this.toolCallBuffer = {};
+    this.currentToolCall = null;
+    this.previouslyParsableState = {};
+  }
+
+  bufferToolCalls(event: {
+    data: { chunk: { tool_call_chunks: { name: string | null; args: string }[] } };
+  }) {
+    if (event.data.chunk.tool_call_chunks.length > 0) {
+      const chunk = event.data.chunk.tool_call_chunks[0];
+
+      if (chunk.name !== null) {
+        this.currentToolCall = chunk.name;
+        this.toolCallBuffer[this.currentToolCall] = chunk.args;
+      } else if (this.currentToolCall !== null) {
+        this.toolCallBuffer[this.currentToolCall] += chunk.args;
+      }
+    }
+  }
+
+  getEmitStateConfig(currentToolName: string): [string | null, string | null] {
+    for (const config of this.emitIntermediateState) {
+      const stateKey = config["state_key"];
+      const tool = config["tool"];
+      const toolArgument = config["tool_argument"];
+
+      if (currentToolName === tool) {
+        return [toolArgument, stateKey];
+      }
+    }
+    return [null, null];
+  }
+
+  extractState(): State {
+    const state: State = {};
+
+    for (const [key, value] of Object.entries(this.toolCallBuffer)) {
+      const [argumentName, stateKey] = this.getEmitStateConfig(key);
+
+      if (stateKey === null) {
+        continue;
+      }
+
+      let parsedValue;
+      try {
+        parsedValue = parsePartialJson(value);
+      } catch (error) {
+        if (key in this.previouslyParsableState) {
+          parsedValue = this.previouslyParsableState[key];
+        } else {
+          continue;
+        }
+      }
+
+      this.previouslyParsableState[key] = parsedValue;
+
+      if (!argumentName) {
+        state[stateKey] = parsedValue;
+      } else {
+        state[stateKey] = parsedValue[argumentName];
+      }
+    }
+
+    return state;
+  }
+}
+
+// Start of Selection
+function langGraphDefaultMergeState(
+  state: State,
+  messages: LangGraphCloudMessage[],
+  actions: ExecutionAction[],
+  agentName: string,
+): State {
+  if (messages.length > 0 && "role" in messages[0] && messages[0].role === "system") {
+    // remove system message
+    messages = messages.slice(1);
+  }
+
+  // merge with existing messages
+  const mergedMessages: LangGraphCloudMessage[] = state.messages || [];
+  const existingMessageIds = new Set(mergedMessages.map((message) => message.id));
+  const existingToolCallResults = new Set<string>();
+
+  for (const message of mergedMessages) {
+    if ("tool_call_id" in message) {
+      existingToolCallResults.add(message.tool_call_id);
+    }
+  }
+
+  for (const message of messages) {
+    // filter tool calls to activate the agent itself
+    if (
+      "tool_calls" in message &&
+      message.tool_calls.length > 0 &&
+      message.tool_calls[0].name === agentName
+    ) {
+      continue;
+    }
+
+    // filter results from activating the agent
+    if ("name" in message && message.name === agentName) {
+      continue;
+    }
+
+    if (!existingMessageIds.has(message.id)) {
+      // skip duplicate tool call results
+      if ("tool_call_id" in message && existingToolCallResults.has(message.tool_call_id)) {
+        console.warn("Warning: Duplicate tool call result, skipping:", message.tool_call_id);
+        continue;
+      }
+
+      mergedMessages.push(message);
+    } else {
+      // Replace the message with the existing one
+      for (let i = 0; i < mergedMessages.length; i++) {
+        if (mergedMessages[i].id === message.id) {
+          if ("tool_calls" in message) {
+            if (
+              ("tool_calls" in mergedMessages[i] || "additional_kwargs" in mergedMessages[i]) &&
+              mergedMessages[i].content
+            ) {
+              message.tool_calls = mergedMessages[i]["tool_calls"];
+              message.additional_kwargs = mergedMessages[i].additional_kwargs;
+            }
+          }
+          mergedMessages[i] = message;
+        }
+      }
+    }
+  }
+
+  // fix wrong tool call ids
+  for (let i = 0; i < mergedMessages.length - 1; i++) {
+    const currentMessage = mergedMessages[i];
+    const nextMessage = mergedMessages[i + 1];
+
+    if (
+      "tool_calls" in currentMessage &&
+      currentMessage.tool_calls.length > 0 &&
+      "tool_call_id" in nextMessage
+    ) {
+      nextMessage.tool_call_id = currentMessage.tool_calls[0].id;
+    }
+  }
+
+  // try to auto-correct and log alignment issues
+  const correctedMessages: LangGraphCloudMessage[] = [];
+
+  for (let i = 0; i < mergedMessages.length; i++) {
+    const currentMessage = mergedMessages[i];
+    const nextMessage = mergedMessages[i + 1] || null;
+    const prevMessage = mergedMessages[i - 1] || null;
+
+    if ("tool_calls" in currentMessage && currentMessage.tool_calls.length > 0) {
+      if (!nextMessage) {
+        console.warn(
+          "No next message to auto-correct tool call, skipping:",
+          currentMessage.tool_calls[0].id,
+        );
+        continue;
+      }
+
+      if (
+        !("tool_call_id" in nextMessage) ||
+        nextMessage.tool_call_id !== currentMessage.tool_calls[0].id
+      ) {
+        const toolMessage = mergedMessages.find(
+          (m) => "tool_call_id" in m && m.tool_call_id === currentMessage.tool_calls[0].id,
+        );
+
+        if (toolMessage) {
+          console.warn(
+            "Auto-corrected tool call alignment issue:",
+            currentMessage.tool_calls[0].id,
+          );
+          correctedMessages.push(currentMessage, toolMessage);
+          continue;
+        } else {
+          console.warn(
+            "No corresponding tool call result found for tool call, skipping:",
+            currentMessage.tool_calls[0].id,
+          );
+          continue;
+        }
+      }
+
+      correctedMessages.push(currentMessage);
+      continue;
+    }
+
+    if ("tool_call_id" in currentMessage) {
+      if (!prevMessage || !("tool_calls" in prevMessage)) {
+        console.warn("No previous tool call, skipping tool call result:", currentMessage.id);
+        continue;
+      }
+
+      if (prevMessage.tool_calls && prevMessage.tool_calls[0].id !== currentMessage.tool_call_id) {
+        console.warn("Tool call id is incorrect, skipping tool call result:", currentMessage.id);
+        continue;
+      }
+
+      correctedMessages.push(currentMessage);
+      continue;
+    }
+
+    correctedMessages.push(currentMessage);
+  }
+
+  return deepMerge(state, {
+    messages: correctedMessages,
+    copilotkit: {
+      actions,
+    },
+  });
+}
+
+function deepMerge(obj1: State, obj2: State) {
+  let result = { ...obj1 };
+  for (let key in obj2) {
+    if (typeof obj2[key] === "object" && !Array.isArray(obj2[key])) {
+      if (obj1[key]) {
+        result[key] = deepMerge(obj1[key], obj2[key]);
+      } else {
+        result[key] = { ...obj2[key] };
+      }
+    } else {
+      result[key] = obj2[key];
+    }
+  }
+  return result;
+}
+
+function formatMessages(messages: Message[]): LangGraphCloudMessage[] {
+  return messages.map((message) => {
+    if (message.isTextMessage() && message.role === "assistant") {
+      return message;
+    }
+    if (message.isTextMessage() && message.role === "system") {
+      return message;
+    }
+    if (message.isTextMessage() && message.role === "user") {
+      return message;
+    }
+    if (message.isActionExecutionMessage()) {
+      const toolCall: ToolCall = {
+        name: message.name,
+        args: message.arguments,
+        id: message.id,
+      };
+      return {
+        type: message.type,
+        content: "",
+        tool_calls: [toolCall],
+        role: MessageRole.assistant,
+        id: message.id,
+      } satisfies LangGraphCloudActionExecutionMessage;
+    }
+    if (message.isResultMessage()) {
+      return {
+        type: message.type,
+        content: message.result,
+        id: message.id,
+        tool_call_id: message.actionExecutionId,
+        name: message.actionName,
+        role: MessageRole.assistant,
+      } satisfies LangGraphCloudResultMessage;
+    }
+
+    throw new Error(`Unknown message type ${message.type}`);
+  });
+}

package/src/service-adapters/google/google-genai-adapter.ts

@@ -29,12 +29,12 @@ interface GoogleGenerativeAIAdapterOptions {
 export class GoogleGenerativeAIAdapter extends LangChainAdapter {
   constructor(options?: GoogleGenerativeAIAdapterOptions) {
     super({
-      chainFn: async ({ messages, tools, threadId }) => {
+      chainFn: async ({ messages, tools }) => {
         const model = new ChatGoogle({
           modelName: options?.model ?? "gemini-1.5-pro",
           apiVersion: "v1beta",
         }).bindTools(tools);
-        return model.stream(messages, { metadata: { conversation_id: threadId } });
+        return model.stream(messages);
       },
     });
   }

package/src/service-adapters/langchain/langchain-adapter.ts

@@ -16,7 +16,9 @@
  *
  * const serviceAdapter = new LangChainAdapter({
  *   chainFn: async ({ messages, tools }) => {
- *     return model.stream(messages, { tools });
+ *     return model.bindTools(tools).stream(messages);
+ *     // or optionally enable strict mode
+ *     // return model.bindTools(tools, { strict: true }).stream(messages);
  *   }
  * });
  *
@@ -70,8 +72,7 @@ export class LangChainAdapter implements CopilotServiceAdapter {
   async process(
     request: CopilotRuntimeChatCompletionRequest,
   ): Promise<CopilotRuntimeChatCompletionResponse> {
-    const { eventSource, model, actions, messages, runId } = request;
-    const threadId = request.threadId ?? randomId();
+    const { eventSource, model, actions, messages, threadId, runId } = request;
     const result = await this.options.chainFn({
       messages: messages.map(convertMessageToLangChainMessage),
       tools: actions.map(convertActionInputToLangChainTool),
@@ -88,7 +89,7 @@ export class LangChainAdapter implements CopilotServiceAdapter {
     });
 
     return {
-      threadId,
+      threadId: threadId || randomId(),
     };
   }
 }