@copilotkit/runtime 1.9.2-next.1 → 1.9.2-next.10

package/src/lib/runtime/copilot-runtime.ts

@@ -25,6 +25,10 @@ import {
   CopilotKitMisuseError,
   CopilotKitErrorCode,
   CopilotKitLowLevelError,
+  CopilotTraceHandler,
+  CopilotTraceEvent,
+  CopilotRequestContext,
+  ensureStructuredError,
 } from "@copilotkit/shared";
 import {
   CopilotServiceAdapter,
@@ -62,7 +66,7 @@ import { ExtensionsInput } from "../../graphql/inputs/extensions.input";
 import { ExtensionsResponse } from "../../graphql/types/extensions-response.type";
 import { LoadAgentStateResponse } from "../../graphql/types/load-agent-state-response.type";
 import { Client as LangGraphClient } from "@langchain/langgraph-sdk";
-import { langchainMessagesToCopilotKit } from "./remote-lg-action";
+import { langchainMessagesToCopilotKit, isUserConfigurationError } from "./remote-lg-action";
 import { MetaEventInput } from "../../graphql/inputs/meta-event.input";
 import {
   CopilotObservabilityConfig,
@@ -81,10 +85,13 @@ import {
   convertMCPToolsToActions,
   generateMcpToolInstructions,
 } from "./mcp-tools-utils";
+import { LangGraphAgent } from "./langgraph/langgraph-agent";
 // Define the function type alias here or import if defined elsewhere
 type CreateMCPClientFunction = (config: MCPEndpointConfig) => Promise<MCPClient>;
 // --- MCP Imports ---
 
+import { generateHelpfulErrorMessage } from "../streaming";
+
 export interface CopilotRuntimeRequest {
   serviceAdapter: CopilotServiceAdapter;
   messages: MessageInput[];
@@ -274,6 +281,25 @@ export interface CopilotRuntimeConstructorParams<T extends Parameter[] | [] = []
    * ```
    */
   createMCPClient?: CreateMCPClientFunction;
+
+  /**
+   * Optional trace handler for comprehensive debugging and observability.
+   *
+   * **Requires publicApiKey**: Tracing only works when requests include a valid publicApiKey.
+   * This is a premium CopilotKit Cloud feature.
+   *
+   * @param traceEvent - Structured trace event with rich debugging context
+   *
+   * @example
+   * ```typescript
+   * const runtime = new CopilotRuntime({
+   *   onTrace: (traceEvent) => {
+   *     debugDashboard.capture(traceEvent);
+   *   }
+   * });
+   * ```
+   */
+  onTrace?: CopilotTraceHandler;
 }
 
 export class CopilotRuntime<const T extends Parameter[] | [] = []> {
@@ -286,6 +312,8 @@ export class CopilotRuntime<const T extends Parameter[] | [] = []> {
   private delegateAgentProcessingToServiceAdapter: boolean;
   private observability?: CopilotObservabilityConfig;
   private availableAgents: Pick<AgentWithEndpoint, "name" | "id">[];
+  private onTrace?: CopilotTraceHandler;
+  private hasWarnedAboutTracing = false;
 
   // +++ MCP Properties +++
   private readonly mcpServersConfig?: MCPEndpointConfig[];
@@ -303,7 +331,21 @@ export class CopilotRuntime<const T extends Parameter[] | [] = []> {
       params?.remoteEndpoints.some((e) => e.type === EndpointType.LangGraphPlatform)
     ) {
       console.warn("Actions set in runtime instance will not be available for the agent");
+      console.warn(
+        `LangGraph Platform remote endpoints are deprecated in favor of the "agents" property`,
+      );
     }
+
+    // TODO: finalize
+    // if (
+    //   params?.agents &&
+    //   Object.values(params.agents).some((agent) => {
+    //     return agent instanceof AguiLangGraphAgent && !(agent instanceof LangGraphAgent);
+    //   })
+    // ) {
+    //   console.warn('LangGraph Agent class should be imported from @copilotkit/runtime. ')
+    // }
+
     this.actions = params?.actions || [];
     this.availableAgents = [];
 
@@ -320,6 +362,7 @@ export class CopilotRuntime<const T extends Parameter[] | [] = []> {
       params?.delegateAgentProcessingToServiceAdapter || false;
     this.observability = params?.observability_c;
     this.agents = params?.agents ?? {};
+    this.onTrace = params?.onTrace;
     // +++ MCP Initialization +++
     this.mcpServersConfig = params?.mcpServers;
     this.createMCPClientImpl = params?.createMCPClient;
@@ -452,6 +495,32 @@ export class CopilotRuntime<const T extends Parameter[] | [] = []> {
     // For storing streamed chunks if progressive logging is enabled
     const streamedChunks: any[] = [];
 
+    // Trace request start
+    await this.trace(
+      "request",
+      {
+        threadId,
+        runId,
+        source: "runtime",
+        request: {
+          operation: "processRuntimeRequest",
+          method: "POST",
+          url: url,
+          startTime: requestStartTime,
+        },
+        agent: agentSession ? { name: agentSession.agentName } : undefined,
+        messages: {
+          input: rawMessages,
+          messageCount: rawMessages.length,
+        },
+        technical: {
+          environment: process.env.NODE_ENV,
+        },
+      },
+      undefined,
+      publicApiKey,
+    );
+
     try {
       if (
         Object.keys(this.agents).length &&
@@ -675,17 +744,62 @@ please use an LLM adapter instead.`,
         }
       }
 
+      let structuredError: CopilotKitError;
+
       if (error instanceof CopilotKitError) {
-        throw error;
+        structuredError = error;
+      } else {
+        // Convert non-CopilotKitErrors to structured errors, but preserve already structured ones
+        structuredError = ensureStructuredError(error, (err) =>
+          this.convertStreamingErrorToStructured(err),
+        );
       }
 
-      // Convert non-CopilotKitErrors to structured errors
-      console.error("Error getting response:", error);
-      const structuredError = this.convertStreamingErrorToStructured(error);
+      // Trace the error
+      await this.trace(
+        "error",
+        {
+          threadId,
+          runId,
+          source: "runtime",
+          request: {
+            operation: "processRuntimeRequest",
+            method: "POST",
+            url: url,
+            startTime: requestStartTime,
+          },
+          response: {
+            endTime: Date.now(),
+            latency: Date.now() - requestStartTime,
+          },
+          agent: agentSession ? { name: agentSession.agentName } : undefined,
+          technical: {
+            environment: process.env.NODE_ENV,
+            stackTrace: error instanceof Error ? error.stack : undefined,
+          },
+        },
+        structuredError,
+        publicApiKey,
+      );
+
       throw structuredError;
     }
   }
 
+  async getAllAgents(graphqlContext: GraphQLContext): Promise<(AgentWithEndpoint | Agent)[]> {
+    const [agentsWithEndpoints, aguiAgents] = await Promise.all([
+      this.discoverAgentsFromEndpoints(graphqlContext),
+      this.discoverAgentsFromAgui(),
+    ]);
+
+    this.availableAgents = [...agentsWithEndpoints, ...aguiAgents].map((a) => ({
+      name: a.name,
+      id: a.id,
+    }));
+
+    return [...agentsWithEndpoints, ...aguiAgents];
+  }
+
   async discoverAgentsFromEndpoints(graphqlContext: GraphQLContext): Promise<AgentWithEndpoint[]> {
     const agents: Promise<AgentWithEndpoint[]> = this.remoteEndpointDefinitions.reduce(
       async (acc: Promise<Agent[]>, endpoint) => {
@@ -767,7 +881,41 @@ please use an LLM adapter instead.`,
       },
       Promise.resolve([]),
     );
-    this.availableAgents = ((await agents) ?? []).map((a) => ({ name: a.name, id: a.id }));
+
+    return agents;
+  }
+
+  async discoverAgentsFromAgui(): Promise<AgentWithEndpoint[]> {
+    const agents: Promise<AgentWithEndpoint[]> = Object.values(this.agents ?? []).reduce(
+      async (acc: Promise<Agent[]>, agent: LangGraphAgent) => {
+        const agents = await acc;
+
+        const client = agent.client;
+        let data: Array<{ assistant_id: string; graph_id: string }> | { detail: string } = [];
+        try {
+          data = await client.assistants.search();
+
+          if (data && "detail" in data && (data.detail as string).toLowerCase() === "not found") {
+            throw new CopilotKitAgentDiscoveryError({ availableAgents: this.availableAgents });
+          }
+        } catch (e) {
+          throw new CopilotKitMisuseError({
+            message: `
+              Failed to find or contact agent ${agent.graphId}.
+              Make sure the LangGraph API is running and the agent is defined in langgraph.json
+              
+              See more: https://docs.copilotkit.ai/troubleshooting/common-issues`,
+          });
+        }
+        const endpointAgents = data.map((entry) => ({
+          name: entry.graph_id,
+          id: entry.assistant_id,
+          description: "",
+        }));
+        return [...agents, ...endpointAgents];
+      },
+      Promise.resolve([]),
+    );
 
     return agents;
   }
@@ -777,50 +925,18 @@ please use an LLM adapter instead.`,
     threadId: string,
     agentName: string,
   ): Promise<LoadAgentStateResponse> {
-    const agentsWithEndpoints = await this.discoverAgentsFromEndpoints(graphqlContext);
+    const agents = await this.getAllAgents(graphqlContext);
 
-    const agentWithEndpoint = agentsWithEndpoints.find((agent) => agent.name === agentName);
-    if (!agentWithEndpoint) {
+    const agent = agents.find((agent) => agent.name === agentName);
+    if (!agent) {
       throw new Error("Agent not found");
     }
 
-    if (agentWithEndpoint.endpoint.type === EndpointType.LangGraphPlatform) {
-      const propertyHeaders = graphqlContext.properties.authorization
-        ? { authorization: `Bearer ${graphqlContext.properties.authorization}` }
-        : null;
-
-      const client = new LangGraphClient({
-        apiUrl: agentWithEndpoint.endpoint.deploymentUrl,
-        apiKey: agentWithEndpoint.endpoint.langsmithApiKey,
-        defaultHeaders: { ...propertyHeaders },
-      });
-      let state: any = {};
-      try {
-        state = (await client.threads.getState(threadId)).values as any;
-      } catch (error) {}
-
-      if (Object.keys(state).length === 0) {
-        return {
-          threadId: threadId || "",
-          threadExists: false,
-          state: JSON.stringify({}),
-          messages: JSON.stringify([]),
-        };
-      } else {
-        const { messages, ...stateWithoutMessages } = state;
-        const copilotkitMessages = langchainMessagesToCopilotKit(messages);
-        return {
-          threadId: threadId || "",
-          threadExists: true,
-          state: JSON.stringify(stateWithoutMessages),
-          messages: JSON.stringify(copilotkitMessages),
-        };
-      }
-    } else if (
-      agentWithEndpoint.endpoint.type === EndpointType.CopilotKit ||
-      !("type" in agentWithEndpoint.endpoint)
+    if (
+      "endpoint" in agent &&
+      (agent.endpoint.type === EndpointType.CopilotKit || !("type" in agent.endpoint))
     ) {
-      const cpkEndpoint = agentWithEndpoint.endpoint as CopilotKitEndpoint;
+      const cpkEndpoint = agent.endpoint as CopilotKitEndpoint;
       const fetchUrl = `${cpkEndpoint.url}/agents/state`;
       try {
         const response = await fetchWithRetry(fetchUrl, {
@@ -836,10 +952,24 @@ please use an LLM adapter instead.`,
           if (response.status === 404) {
             throw new CopilotKitApiDiscoveryError({ url: fetchUrl });
           }
+
+          // Extract semantic error information from response body
+          let errorMessage = `HTTP ${response.status} error`;
+          try {
+            const errorBody = await response.text();
+            const parsedError = JSON.parse(errorBody);
+            if (parsedError.error && typeof parsedError.error === "string") {
+              errorMessage = parsedError.error;
+            }
+          } catch {
+            // If parsing fails, fall back to generic message
+          }
+
           throw new ResolvedCopilotKitError({
             status: response.status,
             url: fetchUrl,
             isRemoteEndpoint: true,
+            message: errorMessage,
           });
         }
 
@@ -856,9 +986,64 @@ please use an LLM adapter instead.`,
         }
         throw new CopilotKitLowLevelError({ error, url: fetchUrl });
       }
+    }
+
+    const propertyHeaders = graphqlContext.properties.authorization
+      ? { authorization: `Bearer ${graphqlContext.properties.authorization}` }
+      : null;
+
+    let client: LangGraphClient;
+    if ("endpoint" in agent && agent.endpoint.type === EndpointType.LangGraphPlatform) {
+      client = new LangGraphClient({
+        apiUrl: agent.endpoint.deploymentUrl,
+        apiKey: agent.endpoint.langsmithApiKey,
+        defaultHeaders: { ...propertyHeaders },
+      });
     } else {
-      throw new Error(`Unknown endpoint type: ${(agentWithEndpoint.endpoint as any).type}`);
+      const aguiAgent = graphqlContext._copilotkit.runtime.agents[agent.name] as LangGraphAgent;
+      if (!aguiAgent) {
+        throw new Error(`Agent: ${agent.name} could not be resolved`);
+      }
+      // @ts-expect-error -- both clients are the same
+      client = aguiAgent.client;
     }
+    let state: any = {};
+    try {
+      state = (await client.threads.getState(threadId)).values as any;
+    } catch (error) {
+      // All errors from agent state loading are user configuration issues
+      const errorMessage = error instanceof Error ? error.message : String(error);
+
+      // Log user configuration errors at debug level to reduce noise
+      console.debug(`Agent '${agentName}' configuration issue: ${errorMessage}`);
+
+      // Throw a configuration error - all agent state loading failures are user setup issues
+      throw new ResolvedCopilotKitError({
+        status: 400,
+        message: `Agent '${agentName}' failed to execute: ${errorMessage}`,
+        code: CopilotKitErrorCode.CONFIGURATION_ERROR,
+      });
+    }
+
+    if (Object.keys(state).length === 0) {
+      return {
+        threadId: threadId || "",
+        threadExists: false,
+        state: JSON.stringify({}),
+        messages: JSON.stringify([]),
+      };
+    } else {
+      const { messages, ...stateWithoutMessages } = state;
+      const copilotkitMessages = langchainMessagesToCopilotKit(messages);
+      return {
+        threadId: threadId || "",
+        threadExists: true,
+        state: JSON.stringify(stateWithoutMessages),
+        messages: JSON.stringify(copilotkitMessages),
+      };
+    }
+
+    throw new Error(`Agent: ${agent.name} could not be resolved`);
   }
 
   private async processAgentRequest(
@@ -884,6 +1069,33 @@ please use an LLM adapter instead.`,
     // for backwards compatibility, deal with the case when no threadId is provided
     const threadId = threadIdFromRequest ?? agentSession.threadId;
 
+    // Trace agent request start
+    await this.trace(
+      "agent_state",
+      {
+        threadId,
+        source: "agent",
+        request: {
+          operation: "processAgentRequest",
+          method: "POST",
+          startTime: requestStartTime,
+        },
+        agent: {
+          name: agentName,
+          nodeName: nodeName,
+        },
+        messages: {
+          input: rawMessages,
+          messageCount: rawMessages.length,
+        },
+        technical: {
+          environment: process.env.NODE_ENV,
+        },
+      },
+      undefined,
+      publicApiKey,
+    );
+
     const serverSideActions = await this.getServerSideActions(request);
 
     const messages = convertGqlInputToMessages(rawMessages);
@@ -1011,9 +1223,7 @@ please use an LLM adapter instead.`,
       eventSource.stream(async (eventStream$) => {
         from(stream).subscribe({
           next: (event) => eventStream$.next(event),
-          error: (err) => {
-            console.error("Error in stream", err);
-
+          error: async (err) => {
             // Log error with observability if enabled
             if (this.observability?.enabled && publicApiKey) {
               try {
@@ -1035,8 +1245,39 @@ please use an LLM adapter instead.`,
               }
             }
 
-            // Convert network termination errors to structured errors
-            const structuredError = this.convertStreamingErrorToStructured(err);
+            // Preserve structured CopilotKit errors, only convert unstructured errors
+            const structuredError = ensureStructuredError(err, (error) =>
+              this.convertStreamingErrorToStructured(error),
+            );
+
+            // Trace streaming errors
+            await this.trace(
+              "error",
+              {
+                threadId,
+                source: "agent",
+                request: {
+                  operation: "processAgentRequest",
+                  method: "POST",
+                  startTime: requestStartTime,
+                },
+                response: {
+                  endTime: Date.now(),
+                  latency: Date.now() - requestStartTime,
+                },
+                agent: {
+                  name: agentName,
+                  nodeName: nodeName,
+                },
+                technical: {
+                  environment: process.env.NODE_ENV,
+                  stackTrace: err instanceof Error ? err.stack : undefined,
+                },
+              },
+              structuredError,
+              publicApiKey,
+            );
+
             eventStream$.error(structuredError);
             eventStream$.complete();
           },
@@ -1114,8 +1355,41 @@ please use an LLM adapter instead.`,
         }
       }
 
+      // Ensure error is structured
+      const structuredError = ensureStructuredError(error, (err) =>
+        this.convertStreamingErrorToStructured(err),
+      );
+
+      // Trace the agent error
+      await this.trace(
+        "error",
+        {
+          threadId,
+          source: "agent",
+          request: {
+            operation: "processAgentRequest",
+            method: "POST",
+            startTime: requestStartTime,
+          },
+          response: {
+            endTime: Date.now(),
+            latency: Date.now() - requestStartTime,
+          },
+          agent: {
+            name: agentName,
+            nodeName: nodeName,
+          },
+          technical: {
+            environment: process.env.NODE_ENV,
+            stackTrace: error instanceof Error ? error.stack : undefined,
+          },
+        },
+        structuredError,
+        publicApiKey,
+      );
+
       console.error("Error getting response:", error);
-      throw error;
+      throw structuredError;
     }
   }
 
@@ -1231,53 +1505,108 @@ please use an LLM adapter instead.`,
   }
 
   private convertStreamingErrorToStructured(error: any): CopilotKitError {
-    // Handle network termination errors
-    if (
-      error?.message?.includes("terminated") ||
-      error?.cause?.code === "UND_ERR_SOCKET" ||
-      error?.message?.includes("other side closed") ||
-      error?.code === "UND_ERR_SOCKET"
-    ) {
-      return new CopilotKitError({
-        message:
-          "Connection to agent was unexpectedly terminated. This may be due to the agent service being restarted or network issues. Please try again.",
-        code: CopilotKitErrorCode.NETWORK_ERROR,
-      });
-    }
+    // Determine a more helpful error message based on context
+    let helpfulMessage = generateHelpfulErrorMessage(error, "agent streaming connection");
 
-    // Handle other network-related errors
+    // For network-related errors, use CopilotKitLowLevelError to preserve the original error
     if (
       error?.message?.includes("fetch failed") ||
       error?.message?.includes("ECONNREFUSED") ||
       error?.message?.includes("ENOTFOUND") ||
-      error?.message?.includes("ETIMEDOUT")
+      error?.message?.includes("ETIMEDOUT") ||
+      error?.message?.includes("terminated") ||
+      error?.cause?.code === "UND_ERR_SOCKET" ||
+      error?.message?.includes("other side closed") ||
+      error?.code === "UND_ERR_SOCKET"
     ) {
       return new CopilotKitLowLevelError({
         error: error instanceof Error ? error : new Error(String(error)),
         url: "agent streaming connection",
-        message:
-          "Network error occurred during agent streaming. Please check your connection and try again.",
-      });
-    }
-
-    // Handle abort/cancellation errors (these are usually normal)
-    if (
-      error?.message?.includes("aborted") ||
-      error?.message?.includes("canceled") ||
-      error?.message?.includes("signal is aborted")
-    ) {
-      return new CopilotKitError({
-        message: "Agent request was cancelled",
-        code: CopilotKitErrorCode.UNKNOWN,
+        message: helpfulMessage,
       });
     }
 
-    // Default: convert unknown streaming errors
+    // For all other errors, preserve the raw error in a basic CopilotKitError
     return new CopilotKitError({
-      message: `Agent streaming error: ${error?.message || String(error)}`,
+      message: helpfulMessage,
       code: CopilotKitErrorCode.UNKNOWN,
     });
   }
+
+  private async trace(
+    type: CopilotTraceEvent["type"],
+    context: CopilotRequestContext,
+    error?: any,
+    publicApiKey?: string,
+  ): Promise<void> {
+    if (!this.onTrace) return;
+
+    // Just check if publicApiKey is defined (regardless of validity)
+    if (!publicApiKey) {
+      if (!this.hasWarnedAboutTracing) {
+        console.warn(
+          "CopilotKit: onTrace handler provided but requires publicApiKey to be defined for tracing to work.",
+        );
+        this.hasWarnedAboutTracing = true;
+      }
+      return;
+    }
+
+    try {
+      const traceEvent: CopilotTraceEvent = {
+        type,
+        timestamp: Date.now(),
+        context,
+        ...(error && { error }),
+      };
+
+      await this.onTrace(traceEvent);
+    } catch (traceError) {
+      // Don't let trace errors break the main flow
+      console.error("Error in onTrace handler:", traceError);
+    }
+  }
+
+  /**
+   * Public method to trace GraphQL validation errors
+   * This allows the GraphQL resolver to send validation errors through the trace system
+   */
+  public async traceGraphQLError(
+    error: { message: string; code: string; type: string },
+    context: {
+      operation: string;
+      cloudConfigPresent: boolean;
+      guardrailsEnabled: boolean;
+    },
+  ): Promise<void> {
+    if (!this.onTrace) return;
+
+    try {
+      await this.onTrace({
+        type: "error",
+        timestamp: Date.now(),
+        context: {
+          source: "runtime",
+          request: {
+            operation: context.operation,
+            startTime: Date.now(),
+          },
+          technical: {
+            environment: process.env.NODE_ENV,
+          },
+          metadata: {
+            errorType: "GraphQLValidationError",
+            cloudConfigPresent: context.cloudConfigPresent,
+            guardrailsEnabled: context.guardrailsEnabled,
+          },
+        },
+        error,
+      });
+    } catch (traceError) {
+      // Don't let trace errors break the main flow
+      console.error("Error in onTrace handler:", traceError);
+    }
+  }
 }
 
 export function flattenToolCallsNoDuplicates(toolsByPriority: ActionInput[]): ActionInput[] {

package/src/lib/runtime/langgraph/langgraph-agent.ts

@@ -16,6 +16,7 @@ import {
   LangGraphAgent as AGUILangGraphAgent,
   type LangGraphAgentConfig,
   ProcessedEvents,
+  SchemaKeys,
 } from "@ag-ui/langgraph";
 import { Message as LangGraphMessage } from "@langchain/langgraph-sdk/dist/types.messages";
 
@@ -180,6 +181,7 @@ export class LangGraphAgent extends AGUILangGraphAgent {
       messages,
       tools,
     );
+
     return {
       ...rest,
       copilotkit: {
@@ -187,4 +189,14 @@ export class LangGraphAgent extends AGUILangGraphAgent {
       },
     };
   }
+
+  async getSchemaKeys(): Promise<SchemaKeys> {
+    const CONSTANT_KEYS = ["copilotkit"];
+    const schemaKeys = await super.getSchemaKeys();
+    return {
+      config: schemaKeys.config,
+      input: schemaKeys.input ? [...schemaKeys.input, ...CONSTANT_KEYS] : null,
+      output: schemaKeys.output ? [...schemaKeys.output, ...CONSTANT_KEYS] : null,
+    };
+  }
 }