@copilotkit/runtime 1.5.12-next.5 → 1.5.12-next.7

package/src/graphql/resolvers/copilot.resolver.ts

@@ -42,7 +42,6 @@ import {
 } from "../types/converted";
 import telemetry from "../../lib/telemetry-client";
 import { randomId } from "@copilotkit/shared";
-import { EndpointType, LangGraphPlatformAgent } from "../../lib/runtime/remote-actions";
 import { AgentsResponse } from "../types/agents-response.type";
 
 const invokeGuardrails = async ({
@@ -113,12 +112,14 @@ export class CopilotResolver {
     let logger = ctx.logger.child({ component: "CopilotResolver.availableAgents" });
 
     logger.debug("Processing");
-    const agents = await ctx._copilotkit.runtime.discoverAgentsFromEndpoints(ctx);
+    const agentsWithEndpoints = await ctx._copilotkit.runtime.discoverAgentsFromEndpoints(ctx);
 
     logger.debug("Event source created, creating response");
 
     return {
-      agents,
+      agents: agentsWithEndpoints.map(
+        ({ endpoint, ...agentWithoutEndpoint }) => agentWithoutEndpoint,
+      ),
     };
   }
 
@@ -192,6 +193,7 @@ export class CopilotResolver {
       runId,
       serverSideActions,
       actionInputsWithoutAgents,
+      extensions,
     } = await copilotRuntime.processRuntimeRequest({
       serviceAdapter,
       messages: data.messages,
@@ -207,6 +209,7 @@ export class CopilotResolver {
       agentSession: data.agentSession,
       agentStates: data.agentStates,
       url: data.frontend.url,
+      extensions: data.extensions,
     });
 
     logger.debug("Event source created, creating response");
@@ -215,6 +218,7 @@ export class CopilotResolver {
       threadId,
       runId,
       status: firstValueFrom(responseStatus$),
+      extensions,
       messages: new Repeater(async (pushMessage, stopStreamingMessages) => {
         logger.debug("Messages repeater created");
 
@@ -281,6 +285,7 @@ export class CopilotResolver {
               (action) =>
                 !serverSideActions.find((serverSideAction) => serverSideAction.name == action.name),
             ),
+            threadId,
           })
           .pipe(
             // shareReplay() ensures that later subscribers will see the whole stream instead of

package/src/graphql/resolvers/state.resolver.ts

@@ -0,0 +1,23 @@
+import { Arg, Resolver } from "type-graphql";
+import { Ctx } from "type-graphql";
+import { Query } from "type-graphql";
+import { LoadAgentStateResponse } from "../types/load-agent-state-response.type";
+import type { GraphQLContext } from "../../lib/integrations";
+import { LoadAgentStateInput } from "../inputs/load-agent-state.input";
+
+@Resolver(() => LoadAgentStateResponse)
+export class StateResolver {
+  @Query(() => LoadAgentStateResponse)
+  async loadAgentState(@Ctx() ctx: GraphQLContext, @Arg("data") data: LoadAgentStateInput) {
+    const agents = await ctx._copilotkit.runtime.discoverAgentsFromEndpoints(ctx);
+    const agent = agents.find((agent) => agent.name === data.agentName);
+
+    if (!agent) {
+      throw new Error("Agent not found");
+    }
+
+    const state = await ctx._copilotkit.runtime.loadAgentState(ctx, data.threadId, data.agentName);
+
+    return state;
+  }
+}

package/src/graphql/types/agents-response.type.ts

@@ -1,7 +1,4 @@
-import { Field, InterfaceType, ObjectType } from "type-graphql";
-import { MessageRole } from "./enums";
-import { MessageStatusUnion } from "./message-status.type";
-import { ResponseStatusUnion } from "./response-status.type";
+import { Field, ObjectType } from "type-graphql";
 
 @ObjectType()
 export class Agent {

package/src/graphql/types/copilot-response.type.ts

@@ -2,6 +2,7 @@ import { Field, InterfaceType, ObjectType } from "type-graphql";
 import { MessageRole } from "./enums";
 import { MessageStatusUnion } from "./message-status.type";
 import { ResponseStatusUnion } from "./response-status.type";
+import { ExtensionsResponse } from "./extensions-response.type";
 
 @InterfaceType({
   resolveType(value) {
@@ -17,7 +18,7 @@ import { ResponseStatusUnion } from "./response-status.type";
     return undefined;
   },
 })
-abstract class BaseMessageOutput {
+export abstract class BaseMessageOutput {
   @Field(() => String)
   id: string;
 
@@ -110,4 +111,7 @@ export class CopilotResponse {
 
   @Field(() => [BaseMessageOutput])
   messages: (typeof BaseMessageOutput)[];
+
+  @Field(() => ExtensionsResponse, { nullable: true })
+  extensions?: ExtensionsResponse;
 }

package/src/graphql/types/extensions-response.type.ts

@@ -0,0 +1,23 @@
+import { Field, ObjectType } from "type-graphql";
+
+/**
+ * The extensions response is used to receive additional information from the copilot runtime, specific to a
+ * service adapter or agent framework.
+ *
+ * Next time a request to the runtime is made, the extensions response will be included in the request as input.
+ */
+
+@ObjectType()
+export class ExtensionsResponse {
+  @Field(() => OpenAIApiAssistantAPIResponse, { nullable: true })
+  openaiAssistantAPI?: OpenAIApiAssistantAPIResponse;
+}
+
+@ObjectType()
+export class OpenAIApiAssistantAPIResponse {
+  @Field(() => String, { nullable: true })
+  runId?: string;
+
+  @Field(() => String, { nullable: true })
+  threadId?: string;
+}

package/src/graphql/types/load-agent-state-response.type.ts

@@ -0,0 +1,17 @@
+import { Field, ObjectType } from "type-graphql";
+import { BaseMessageOutput } from "./copilot-response.type";
+
+@ObjectType()
+export class LoadAgentStateResponse {
+  @Field(() => String)
+  threadId: string;
+
+  @Field(() => Boolean)
+  threadExists: boolean;
+
+  @Field(() => String)
+  state: string;
+
+  @Field(() => String)
+  messages: string;
+}

package/src/lib/integrations/shared.ts

@@ -8,6 +8,7 @@ import { CopilotCloudOptions } from "../cloud";
 import { LogLevel, createLogger } from "../../lib/logger";
 import { createYoga } from "graphql-yoga";
 import telemetry from "../telemetry-client";
+import { StateResolver } from "../../graphql/resolvers/state.resolver";
 
 const logger = createLogger();
 
@@ -58,7 +59,7 @@ export function buildSchema(
 ) {
   logger.debug("Building GraphQL schema...");
   const schema = buildSchemaSync({
-    resolvers: [CopilotResolver],
+    resolvers: [CopilotResolver, StateResolver],
     emitSchemaFile: options.emitSchemaFile,
   });
   logger.debug("GraphQL schema built successfully");

package/src/lib/runtime/copilot-runtime.ts

@@ -12,7 +12,18 @@
  * ```
  */
 
-import { Action, actionParametersToJsonSchema, Parameter, randomId } from "@copilotkit/shared";
+import {
+  Action,
+  actionParametersToJsonSchema,
+  Parameter,
+  ResolvedCopilotKitError,
+  CopilotKitApiDiscoveryError,
+  randomId,
+  CopilotKitError,
+  CopilotKitLowLevelError,
+  CopilotKitAgentDiscoveryError,
+  CopilotKitMisuseError,
+} from "@copilotkit/shared";
 import {
   CopilotServiceAdapter,
   EmptyAdapter,
@@ -41,6 +52,11 @@ import { AgentStateInput } from "../../graphql/inputs/agent-state.input";
 import { ActionInputAvailability } from "../../graphql/types/enums";
 import { createHeaders } from "./remote-action-constructors";
 import { Agent } from "../../graphql/types/agents-response.type";
+import { ExtensionsInput } from "../../graphql/inputs/extensions.input";
+import { ExtensionsResponse } from "../../graphql/types/extensions-response.type";
+import { LoadAgentStateResponse } from "../../graphql/types/load-agent-state-response.type";
+import { Client as LangGraphClient } from "@langchain/langgraph-sdk";
+import { langchainMessagesToCopilotKit } from "./remote-lg-action";
 
 interface CopilotRuntimeRequest {
   serviceAdapter: CopilotServiceAdapter;
@@ -55,6 +71,7 @@ interface CopilotRuntimeRequest {
   graphqlContext: GraphQLContext;
   forwardedParameters?: ForwardedParametersInput;
   url?: string;
+  extensions?: ExtensionsInput;
 }
 
 interface CopilotRuntimeResponse {
@@ -63,6 +80,7 @@ interface CopilotRuntimeResponse {
   eventSource: RuntimeEventSource;
   serverSideActions: Action<any>[];
   actionInputsWithoutAgents: ActionInput[];
+  extensions?: ExtensionsResponse;
 }
 
 type ActionsConfiguration<T extends Parameter[] | [] = []> =
@@ -102,6 +120,8 @@ interface Middleware {
   onAfterRequest?: OnAfterRequestHandler;
 }
 
+type AgentWithEndpoint = Agent & { endpoint: EndpointDefinition };
+
 export interface CopilotRuntimeConstructorParams<T extends Parameter[] | [] = []> {
   /**
    * Middleware to be used by the runtime.
@@ -181,6 +201,7 @@ export class CopilotRuntime<const T extends Parameter[] | [] = []> {
       forwardedParameters,
       agentSession,
       url,
+      extensions,
     } = request;
 
     const eventSource = new RuntimeEventSource();
@@ -190,10 +211,11 @@ export class CopilotRuntime<const T extends Parameter[] | [] = []> {
         return await this.processAgentRequest(request);
       }
       if (serviceAdapter instanceof EmptyAdapter) {
-        // TODO: use CPK error here
-        throw new Error(`Invalid adapter configuration: EmptyAdapter is only meant to be used with agent lock mode. 
+        throw new CopilotKitMisuseError({
+          message: `Invalid adapter configuration: EmptyAdapter is only meant to be used with agent lock mode. 
 For non-agent components like useCopilotChatSuggestions, CopilotTextarea, or CopilotTask, 
-please use an LLM adapter instead.`);
+please use an LLM adapter instead.`,
+        });
       }
 
       const messages = rawMessages.filter((message) => !message.agentStateMessage);
@@ -230,12 +252,17 @@ please use an LLM adapter instead.`);
         runId,
         eventSource,
         forwardedParameters,
+        extensions,
       });
 
+      // for backwards compatibility, we deal with the case that no threadId is provided
+      // by the frontend, by using the threadId from the response
+      const nonEmptyThreadId = threadId ?? result.threadId;
+
       outputMessagesPromise
         .then((outputMessages) => {
           this.onAfterRequest?.({
-            threadId: result.threadId,
+            threadId: nonEmptyThreadId,
             runId: result.runId,
             inputMessages,
             outputMessages,
@@ -246,7 +273,7 @@ please use an LLM adapter instead.`);
         .catch((_error) => {});
 
       return {
-        threadId: result.threadId,
+        threadId: nonEmptyThreadId,
         runId: result.runId,
         eventSource,
         serverSideActions,
@@ -258,32 +285,37 @@ please use an LLM adapter instead.`);
           //   serverSideActions.find((serverSideAction) => serverSideAction.name == action.name),
           // ),
         ),
+        extensions: result.extensions,
       };
     } catch (error) {
+      if (error instanceof CopilotKitError) {
+        throw error;
+      }
       console.error("Error getting response:", error);
       eventSource.sendErrorMessageToChat();
       throw error;
     }
   }
 
-  async discoverAgentsFromEndpoints(graphqlContext: GraphQLContext): Promise<Agent[]> {
+  async discoverAgentsFromEndpoints(graphqlContext: GraphQLContext): Promise<AgentWithEndpoint[]> {
     const headers = createHeaders(null, graphqlContext);
     const agents = this.remoteEndpointDefinitions.reduce(
       async (acc: Promise<Agent[]>, endpoint) => {
         const agents = await acc;
         if (endpoint.type === EndpointType.LangGraphPlatform) {
-          const response = await fetch(
-            `${(endpoint as LangGraphPlatformEndpoint).deploymentUrl}/assistants/search`,
-            {
-              method: "POST",
-              headers,
-            },
-          );
-
-          const data: Array<{ assistant_id: string; graph_id: string }> = await response.json();
+          const client = new LangGraphClient({
+            apiUrl: endpoint.deploymentUrl,
+            apiKey: endpoint.langsmithApiKey,
+          });
+
+          const data: Array<{ assistant_id: string; graph_id: string }> =
+            await client.assistants.search();
+
           const endpointAgents = (data ?? []).map((entry) => ({
             name: entry.graph_id,
             id: entry.assistant_id,
+            description: "",
+            endpoint,
           }));
           return [...agents, ...endpointAgents];
         }
@@ -295,18 +327,34 @@ please use an LLM adapter instead.`);
           }>;
         }
 
-        const response = await fetch(`${(endpoint as CopilotKitEndpoint).url}/info`, {
-          method: "POST",
-          headers,
-          body: JSON.stringify({ properties: graphqlContext.properties }),
-        });
-        const data: InfoResponse = await response.json();
-        const endpointAgents = (data?.agents ?? []).map((agent) => ({
-          name: agent.name,
-          description: agent.description,
-          id: randomId(), // Required by Agent type
-        }));
-        return [...agents, ...endpointAgents];
+        const fetchUrl = `${(endpoint as CopilotKitEndpoint).url}/info`;
+        try {
+          const response = await fetch(fetchUrl, {
+            method: "POST",
+            headers,
+            body: JSON.stringify({ properties: graphqlContext.properties }),
+          });
+          if (!response.ok) {
+            if (response.status === 404) {
+              throw new CopilotKitApiDiscoveryError();
+            }
+            throw new ResolvedCopilotKitError({ status: response.status, isRemoteEndpoint: true });
+          }
+
+          const data: InfoResponse = await response.json();
+          const endpointAgents = (data?.agents ?? []).map((agent) => ({
+            name: agent.name,
+            description: agent.description ?? "" ?? "",
+            id: randomId(), // Required by Agent type
+            endpoint,
+          }));
+          return [...agents, ...endpointAgents];
+        } catch (error) {
+          if (error instanceof CopilotKitError) {
+            throw error;
+          }
+          throw new CopilotKitLowLevelError({ error: error as Error, url: fetchUrl });
+        }
       },
       Promise.resolve([]),
     );
@@ -314,11 +362,87 @@ please use an LLM adapter instead.`);
     return agents;
   }
 
+  async loadAgentState(
+    graphqlContext: GraphQLContext,
+    threadId: string,
+    agentName: string,
+  ): Promise<LoadAgentStateResponse> {
+    const agentsWithEndpoints = await this.discoverAgentsFromEndpoints(graphqlContext);
+
+    const agentWithEndpoint = agentsWithEndpoints.find((agent) => agent.name === agentName);
+    if (!agentWithEndpoint) {
+      throw new Error("Agent not found");
+    }
+    const headers = createHeaders(null, graphqlContext);
+
+    if (agentWithEndpoint.endpoint.type === EndpointType.LangGraphPlatform) {
+      const client = new LangGraphClient({
+        apiUrl: agentWithEndpoint.endpoint.deploymentUrl,
+        apiKey: agentWithEndpoint.endpoint.langsmithApiKey,
+      });
+      const state = (await client.threads.getState(threadId)).values as any;
+
+      if (Object.keys(state).length === 0) {
+        return {
+          threadId,
+          threadExists: false,
+          state: JSON.stringify({}),
+          messages: JSON.stringify([]),
+        };
+      } else {
+        console.log(state);
+        const { messages, ...stateWithoutMessages } = state;
+        const copilotkitMessages = langchainMessagesToCopilotKit(messages);
+        return {
+          threadId,
+          threadExists: true,
+          state: JSON.stringify(stateWithoutMessages),
+          messages: JSON.stringify(copilotkitMessages),
+        };
+      }
+    } else if (
+      agentWithEndpoint.endpoint.type === EndpointType.CopilotKit ||
+      !("type" in agentWithEndpoint.endpoint)
+    ) {
+      const response = await fetch(
+        `${(agentWithEndpoint.endpoint as CopilotKitEndpoint).url}/agents/state`,
+        {
+          method: "POST",
+          headers,
+          body: JSON.stringify({
+            properties: graphqlContext.properties,
+            threadId,
+            name: agentName,
+          }),
+        },
+      );
+      const data: LoadAgentStateResponse = await response.json();
+
+      return {
+        ...data,
+        state: JSON.stringify(data.state),
+        messages: JSON.stringify(data.messages),
+      };
+    } else {
+      throw new Error(`Unknown endpoint type: ${(agentWithEndpoint.endpoint as any).type}`);
+    }
+  }
+
   private async processAgentRequest(
     request: CopilotRuntimeRequest,
   ): Promise<CopilotRuntimeResponse> {
-    const { messages: rawMessages, outputMessagesPromise, graphqlContext, agentSession } = request;
-    const { threadId, agentName, nodeName } = agentSession;
+    const {
+      messages: rawMessages,
+      outputMessagesPromise,
+      graphqlContext,
+      agentSession,
+      threadId: threadIdFromRequest,
+    } = request;
+    const { agentName, nodeName } = agentSession;
+
+    // for backwards compatibility, deal with the case when no threadId is provided
+    const threadId = threadIdFromRequest ?? agentSession.threadId;
+
     const serverSideActions = await this.getServerSideActions(request);
 
     const messages = convertGqlInputToMessages(rawMessages);
@@ -328,7 +452,7 @@ please use an LLM adapter instead.`);
     ) as LangGraphAgentAction;
 
     if (!agent) {
-      throw new Error(`Agent ${agentName} not found`);
+      throw new CopilotKitAgentDiscoveryError({ agentName });
     }
 
     const serverSideActionsInput: ActionInput[] = serverSideActions

package/src/lib/runtime/remote-action-constructors.ts

@@ -16,6 +16,7 @@ import { RemoteLangGraphEventSource } from "../../agents/langgraph/event-source"
 import { Action } from "@copilotkit/shared";
 import { LangGraphEvent } from "../../agents/langgraph/events";
 import { execute } from "./remote-lg-action";
+import { CopilotKitError, CopilotKitLowLevelError } from "@copilotkit/shared";
 
 export function constructLGCRemoteAction({
   endpoint,
@@ -126,8 +127,9 @@ export function constructRemoteActions({
         agentsAmount: totalAgents,
       });
 
+      const fetchUrl = `${url}/actions/execute`;
       try {
-        const response = await fetch(`${url}/actions/execute`, {
+        const response = await fetch(fetchUrl, {
           method: "POST",
           headers,
           body: JSON.stringify({
@@ -151,11 +153,10 @@ export function constructRemoteActions({
         logger.debug({ actionName: action.name, result }, "Executed remote action");
         return result;
       } catch (error) {
-        logger.error(
-          { error: error.message ? error.message : error + "" },
-          "Failed to execute remote action",
-        );
-        return "Failed to execute remote action";
+        if (error instanceof CopilotKitError) {
+          throw error;
+        }
+        throw new CopilotKitLowLevelError({ error, url: fetchUrl });
       }
     },
   }));
@@ -191,35 +192,43 @@ export function constructRemoteActions({
             }
           }
 
-          const response = await fetch(`${url}/agents/execute`, {
-            method: "POST",
-            headers,
-            body: JSON.stringify({
-              name,
-              threadId,
-              nodeName,
-              messages: [...messages, ...additionalMessages],
-              state,
-              properties: graphqlContext.properties,
-              actions: actionInputsWithoutAgents.map((action) => ({
-                name: action.name,
-                description: action.description,
-                parameters: JSON.parse(action.jsonSchema),
-              })),
-            }),
-          });
+          const fetchUrl = `${url}/agents/execute`;
+          try {
+            const response = await fetch(fetchUrl, {
+              method: "POST",
+              headers,
+              body: JSON.stringify({
+                name,
+                threadId,
+                nodeName,
+                messages: [...messages, ...additionalMessages],
+                state,
+                properties: graphqlContext.properties,
+                actions: actionInputsWithoutAgents.map((action) => ({
+                  name: action.name,
+                  description: action.description,
+                  parameters: JSON.parse(action.jsonSchema),
+                })),
+              }),
+            });
+
+            if (!response.ok) {
+              logger.error(
+                { url, status: response.status, body: await response.text() },
+                "Failed to execute remote agent",
+              );
+              throw new Error("Failed to execute remote agent");
+            }
 
-          if (!response.ok) {
-            logger.error(
-              { url, status: response.status, body: await response.text() },
-              "Failed to execute remote agent",
-            );
-            throw new Error("Failed to execute remote agent");
+            const eventSource = new RemoteLangGraphEventSource();
+            streamResponse(response.body!, eventSource.eventStream$);
+            return eventSource.processLangGraphEvents();
+          } catch (error) {
+            if (error instanceof CopilotKitError) {
+              throw error;
+            }
+            throw new CopilotKitLowLevelError({ error, url: fetchUrl });
           }
-
-          const eventSource = new RemoteLangGraphEventSource();
-          streamResponse(response.body!, eventSource.eventStream$);
-          return eventSource.processLangGraphEvents();
         },
       }))
     : [];

package/src/lib/runtime/remote-actions.ts

@@ -11,6 +11,11 @@ import {
   constructRemoteActions,
   createHeaders,
 } from "./remote-action-constructors";
+import {
+  CopilotKitLowLevelError,
+  ResolvedCopilotKitError,
+  CopilotKitError,
+} from "@copilotkit/shared";
 
 export type EndpointDefinition = CopilotKitEndpoint | LangGraphPlatformEndpoint;
 
@@ -83,8 +88,9 @@ async function fetchRemoteInfo({
   logger.debug({ url }, "Fetching actions from url");
   const headers = createHeaders(onBeforeRequest, graphqlContext);
 
+  const fetchUrl = `${url}/info`;
   try {
-    const response = await fetch(`${url}/info`, {
+    const response = await fetch(fetchUrl, {
       method: "POST",
       headers,
       body: JSON.stringify({ properties: graphqlContext.properties, frontendUrl }),
@@ -95,18 +101,17 @@ async function fetchRemoteInfo({
         { url, status: response.status, body: await response.text() },
         "Failed to fetch actions from url",
       );
-      return { actions: [], agents: [] };
+      throw new ResolvedCopilotKitError({ status: response.status, isRemoteEndpoint: true });
     }
 
     const json = await response.json();
     logger.debug({ json }, "Fetched actions from url");
     return json;
   } catch (error) {
-    logger.error(
-      { error: error.message ? error.message : error + "" },
-      "Failed to fetch actions from url",
-    );
-    return { actions: [], agents: [] };
+    if (error instanceof CopilotKitError) {
+      throw error;
+    }
+    throw new CopilotKitLowLevelError({ error, url: fetchUrl });
   }
 }