@copilotkit/runtime 1.55.0-next.9 → 1.55.1-next.0

package/package.json

@@ -1,15 +1,25 @@
 {
   "name": "@copilotkit/runtime",
+  "version": "1.55.1-next.0",
   "private": false,
+  "keywords": [
+    "ai",
+    "assistant",
+    "automation",
+    "copilot",
+    "copilotkit",
+    "javascript",
+    "nextjs",
+    "nodejs",
+    "react",
+    "textarea"
+  ],
   "homepage": "https://github.com/CopilotKit/CopilotKit",
+  "license": "MIT",
   "repository": {
     "type": "git",
     "url": "https://github.com/CopilotKit/CopilotKit.git"
   },
-  "publishConfig": {
-    "access": "public"
-  },
-  "version": "1.55.0-next.9",
   "sideEffects": [
     "./dist/index.mjs",
     "./dist/index.cjs",
@@ -18,6 +28,23 @@
   ],
   "main": "./dist/index.cjs",
   "module": "./dist/index.mjs",
+  "types": "./dist/index.d.cts",
+  "typesVersions": {
+    "*": {
+      "v2": [
+        "./dist/v2/index.d.cts"
+      ],
+      "v2/express": [
+        "./dist/v2/express.d.cts"
+      ],
+      "v2/hono": [
+        "./dist/v2/hono.d.cts"
+      ],
+      "v2/node": [
+        "./dist/v2/node.d.cts"
+      ]
+    }
+  },
   "exports": {
     ".": {
       "import": "./dist/index.mjs",
@@ -31,47 +58,44 @@
       "import": "./dist/v2/index.mjs",
       "require": "./dist/v2/index.cjs"
     },
+    "./v2/express": {
+      "import": "./dist/v2/express.mjs",
+      "require": "./dist/v2/express.cjs"
+    },
+    "./v2/hono": {
+      "import": "./dist/v2/hono.mjs",
+      "require": "./dist/v2/hono.cjs"
+    },
+    "./v2/node": {
+      "import": "./dist/v2/node.mjs",
+      "require": "./dist/v2/node.cjs"
+    },
     "./package.json": "./package.json"
   },
-  "types": "./dist/index.d.cts",
-  "license": "MIT",
-  "devDependencies": {
-    "@swc/core": "1.5.28",
-    "@types/cors": "^2.8.17",
-    "@types/express": "^4.17.21",
-    "@types/node": "^18.11.17",
-    "@types/phoenix": "^1.6.6",
-    "@valibot/to-json-schema": "^1.5.0",
-    "@vitest/coverage-v8": "^3.2.4",
-    "@whatwg-node/server": "^0.9.34",
-    "arktype": "^2.1.29",
-    "eslint": "^8.56.0",
-    "nodemon": "^3.1.3",
-    "supertest": "^7.1.1",
-    "ts-node": "^10.9.2",
-    "tsdown": "^0.20.3",
-    "typescript": "^5.2.3",
-    "valibot": "^1.2.0",
-    "vitest": "^3.2.4",
-    "eslint-config-custom": "1.4.12",
-    "tsconfig": "1.4.12"
+  "publishConfig": {
+    "access": "public"
   },
   "dependencies": {
-    "@ag-ui/a2ui-middleware": "0.0.2",
-    "@ag-ui/client": "^0.0.48",
-    "@ag-ui/core": "^0.0.48",
-    "@ag-ui/encoder": "0.0.47",
-    "@ag-ui/langgraph": "^0.0.25",
-    "@ag-ui/mcp-apps-middleware": "0.0.2",
+    "@ag-ui/a2ui-middleware": "0.0.3",
+    "@ag-ui/client": "0.0.52",
+    "@ag-ui/core": "0.0.52",
+    "@ag-ui/encoder": "0.0.52",
+    "@ag-ui/langgraph": "0.0.27",
+    "@ag-ui/mcp-apps-middleware": "0.0.3",
     "@ai-sdk/anthropic": "^3.0.49",
     "@ai-sdk/google": "^3.0.33",
     "@ai-sdk/google-vertex": "^3.0.97",
     "@ai-sdk/mcp": "^1.0.21",
     "@ai-sdk/openai": "^3.0.36",
-    "@modelcontextprotocol/sdk": "^1.18.2",
+    "@copilotkit/license-verifier": "0.0.1-a1",
     "@graphql-yoga/plugin-defer-stream": "^3.3.1",
     "@hono/node-server": "^1.13.5",
+    "@modelcontextprotocol/sdk": "^1.18.2",
+    "@remix-run/node-fetch-server": "^0.13.0",
     "@scarf/scarf": "^1.3.0",
+    "@segment/analytics-node": "^2.1.2",
+    "ai": "^6.0.104",
+    "clarinet": "^0.12.4",
     "class-transformer": "^0.5.1",
     "class-validator": "^0.14.1",
     "cors": "^2.8.5",
@@ -79,7 +103,6 @@
     "graphql": "^16.8.1",
     "graphql-scalars": "^1.23.0",
     "graphql-yoga": "^5.3.1",
-    "ai": "^6.0.104",
     "hono": "^4.11.4",
     "openai": "^4.85.1",
     "partial-json": "^0.1.7",
@@ -89,15 +112,37 @@
     "reflect-metadata": "^0.2.2",
     "rxjs": "7.8.1",
     "type-graphql": "2.0.0-rc.1",
+    "uuid": "^10.0.0",
     "ws": "^8.18.0",
     "zod": "^3.23.3",
-    "@copilotkit/shared": "1.55.0-next.9"
+    "@copilotkit/shared": "1.55.1-next.0"
+  },
+  "devDependencies": {
+    "@swc/core": "1.5.28",
+    "@types/cors": "^2.8.17",
+    "@types/express": "^4.17.21",
+    "@types/node": "^18.11.17",
+    "@types/phoenix": "^1.6.6",
+    "@valibot/to-json-schema": "^1.5.0",
+    "@vitest/coverage-v8": "^3.2.4",
+    "@whatwg-node/server": "^0.9.34",
+    "arktype": "^2.1.29",
+    "elysia": "^1.2.25",
+    "eslint": "^8.56.0",
+    "nodemon": "^3.1.3",
+    "supertest": "^7.1.1",
+    "ts-node": "^10.9.2",
+    "tsdown": "^0.20.3",
+    "typescript": "^5.2.3",
+    "valibot": "^1.2.0",
+    "vitest": "^3.2.4",
+    "tsconfig": "1.4.12"
   },
   "peerDependencies": {
     "@anthropic-ai/sdk": "^0.57.0",
     "@langchain/aws": ">=0.1.9",
-    "@langchain/core": ">=0.3.66",
     "@langchain/community": ">=0.3.58",
+    "@langchain/core": ">=0.3.66",
     "@langchain/google-gauth": ">=0.1.0",
     "@langchain/langgraph-sdk": ">=0.1.2",
     "@langchain/openai": ">=0.4.2",
@@ -143,18 +188,6 @@
       }
     }
   },
-  "keywords": [
-    "copilotkit",
-    "copilot",
-    "react",
-    "nextjs",
-    "nodejs",
-    "ai",
-    "assistant",
-    "javascript",
-    "automation",
-    "textarea"
-  ],
   "scripts": {
     "build": "tsdown",
     "dev": "tsdown --watch",

package/src/agent/__tests__/multimodal.test.ts

@@ -0,0 +1,176 @@
+import { describe, it, expect } from "vitest";
+import { convertMessagesToVercelAISDKMessages } from "../index";
+import type { Message, InputContent } from "@ag-ui/client";
+import type { UserModelMessage } from "ai";
+
+/**
+ * Helper: build a user message with the given content parts and convert it.
+ * Returns the converted UserModelMessage for assertion.
+ */
+function convertUserContent(content: string | InputContent[]) {
+  const messages: Message[] = [{ id: "1", role: "user", content }];
+  const result = convertMessagesToVercelAISDKMessages(messages);
+  return result[0] as UserModelMessage;
+}
+
+/** Shorthand factories for AG-UI input parts */
+function dataSource(value: string, mimeType: string) {
+  return { type: "data" as const, value, mimeType };
+}
+
+function urlSource(value: string, mimeType?: string) {
+  return { type: "url" as const, value, ...(mimeType ? { mimeType } : {}) };
+}
+
+describe("convertMessagesToVercelAISDKMessages — multimodal", () => {
+  it("passes through plain string user content", () => {
+    const result = convertUserContent("Hello");
+    expect(result).toEqual({ role: "user", content: "Hello" });
+  });
+
+  it("converts text-only InputContent[] to parts array", () => {
+    const result = convertUserContent([{ type: "text", text: "Hello world" }]);
+    expect(result.role).toBe("user");
+    expect(result.content).toEqual([{ type: "text", text: "Hello world" }]);
+  });
+
+  it("converts ImageInputPart with data source to ImagePart", () => {
+    const result = convertUserContent([
+      { type: "text", text: "What is this?" },
+      { type: "image", source: dataSource("iVBORw0KGgo=", "image/png") },
+    ]);
+    expect(result.content).toEqual([
+      { type: "text", text: "What is this?" },
+      { type: "image", image: "iVBORw0KGgo=", mediaType: "image/png" },
+    ]);
+  });
+
+  it("converts ImageInputPart with url source to ImagePart with URL", () => {
+    const result = convertUserContent([
+      {
+        type: "image",
+        source: urlSource("https://example.com/photo.jpg", "image/jpeg"),
+      },
+    ]);
+    expect(result.content).toEqual([
+      {
+        type: "image",
+        image: new URL("https://example.com/photo.jpg"),
+        mediaType: "image/jpeg",
+      },
+    ]);
+  });
+
+  it("converts AudioInputPart to FilePart", () => {
+    const result = convertUserContent([
+      { type: "audio", source: dataSource("base64audiodata", "audio/mp3") },
+    ]);
+    expect(result.content).toEqual([
+      { type: "file", data: "base64audiodata", mediaType: "audio/mp3" },
+    ]);
+  });
+
+  it("converts VideoInputPart with url source to FilePart", () => {
+    const result = convertUserContent([
+      {
+        type: "video",
+        source: urlSource("https://example.com/video.mp4", "video/mp4"),
+      },
+    ]);
+    expect(result.content).toEqual([
+      {
+        type: "file",
+        data: new URL("https://example.com/video.mp4"),
+        mediaType: "video/mp4",
+      },
+    ]);
+  });
+
+  it("converts DocumentInputPart to FilePart", () => {
+    const result = convertUserContent([
+      {
+        type: "document",
+        source: dataSource("base64pdfdata", "application/pdf"),
+      },
+    ]);
+    expect(result.content).toEqual([
+      { type: "file", data: "base64pdfdata", mediaType: "application/pdf" },
+    ]);
+  });
+
+  it("handles mixed text and multimodal parts", () => {
+    const result = convertUserContent([
+      { type: "text", text: "Analyze these:" },
+      { type: "image", source: dataSource("imgdata", "image/png") },
+      { type: "document", source: dataSource("docdata", "application/pdf") },
+    ]);
+    expect(result.content).toEqual([
+      { type: "text", text: "Analyze these:" },
+      { type: "image", image: "imgdata", mediaType: "image/png" },
+      { type: "file", data: "docdata", mediaType: "application/pdf" },
+    ]);
+  });
+
+  it("returns empty string for empty content array", () => {
+    const result = convertUserContent([]);
+    expect(result.content).toBe("");
+  });
+
+  it("skips image parts with malformed URLs without crashing", () => {
+    const result = convertUserContent([
+      { type: "text", text: "check this" },
+      { type: "image", source: { type: "url", value: "not-a-url" } },
+    ]);
+    // Malformed URL part is skipped, text part preserved
+    expect(result.content).toEqual([{ type: "text", text: "check this" }]);
+  });
+
+  // Legacy backward compat — BinaryInputContent is not in the current schema
+  // but older clients may still send it. We intentionally construct untyped
+  // objects here to simulate that scenario.
+  describe("legacy BinaryInputContent backward compat", () => {
+    it("converts binary with image mimeType and data to ImagePart", () => {
+      const legacyPart = {
+        type: "binary",
+        mimeType: "image/jpeg",
+        data: "legacybase64",
+      };
+      const messages: Message[] = [
+        {
+          id: "1",
+          role: "user",
+          content: [legacyPart] as unknown as InputContent[],
+        },
+      ];
+      const result = convertMessagesToVercelAISDKMessages(messages);
+      const userMsg = result[0] as UserModelMessage;
+      expect(userMsg.content).toEqual([
+        { type: "image", image: "legacybase64", mediaType: "image/jpeg" },
+      ]);
+    });
+
+    it("converts binary with non-image mimeType and url to FilePart", () => {
+      const legacyPart = {
+        type: "binary",
+        mimeType: "application/pdf",
+        url: "https://example.com/doc.pdf",
+      };
+      const messages: Message[] = [
+        {
+          id: "1",
+          role: "user",
+          content: [legacyPart] as unknown as InputContent[],
+        },
+      ];
+      const result = convertMessagesToVercelAISDKMessages(messages);
+      const userMsg = result[0] as UserModelMessage;
+      expect(userMsg.content).toEqual([
+        {
+          type: "file",
+          data: new URL("https://example.com/doc.pdf"),
+          mediaType: "application/pdf",
+        },
+      ]);
+    });
+  });
+});

package/src/agent/index.ts

@@ -31,6 +31,8 @@ import {
   ToolCallPart,
   ToolResultPart,
   TextPart,
+  ImagePart,
+  FilePart,
   tool as createVercelAISDKTool,
   ToolChoice,
   ToolSet,
@@ -42,7 +44,6 @@ import { createOpenAI } from "@ai-sdk/openai";
 import { createAnthropic } from "@ai-sdk/anthropic";
 import { createGoogleGenerativeAI } from "@ai-sdk/google";
 import { createVertex } from "@ai-sdk/google-vertex";
-import { randomUUID } from "crypto";
 import { safeParseToolArgs } from "@copilotkit/shared";
 import { z } from "zod";
 import type { StandardSchemaV1, InferSchemaOutput } from "@copilotkit/shared";
@@ -53,6 +54,7 @@ import {
   StreamableHTTPClientTransportOptions,
 } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
 import { SSEClientTransport } from "@modelcontextprotocol/sdk/client/sse.js";
+import { randomUUID } from "@copilotkit/shared";
 
 /**
  * Properties that can be overridden by forwardedProps
@@ -279,9 +281,14 @@ export function defineTool<TParameters extends StandardSchemaV1>(config: {
 
 type AGUIUserMessage = Extract<Message, { role: "user" }>;
 
-function flattenUserMessageContent(
-  content?: AGUIUserMessage["content"],
-): string {
+/**
+ * Converts AG-UI user message content to Vercel AI SDK UserContent format.
+ * Handles plain strings, new modality-specific parts (image/audio/video/document),
+ * and legacy BinaryInputContent for backward compatibility.
+ */
+function convertUserMessageContent(
+  content: AGUIUserMessage["content"],
+): string | Array<TextPart | ImagePart | FilePart> {
   if (!content) {
     return "";
   }
@@ -290,21 +297,125 @@ function flattenUserMessageContent(
     return content;
   }
 
-  return content
-    .map((part) => {
-      if (
-        part &&
-        typeof part === "object" &&
-        "type" in part &&
-        (part as { type?: unknown }).type === "text" &&
-        typeof (part as { text?: unknown }).text === "string"
-      ) {
-        return (part as { text: string }).text;
+  const parts: Array<TextPart | ImagePart | FilePart> = [];
+
+  for (const part of content) {
+    if (!part || typeof part !== "object" || !("type" in part)) {
+      continue;
+    }
+
+    switch (part.type) {
+      case "text": {
+        const text = (part as { text?: string }).text;
+        if (text) {
+          parts.push({ type: "text", text });
+        }
+        break;
+      }
+
+      case "image": {
+        const source = (part as { source?: any }).source;
+        if (!source) break;
+        if (source.type === "data") {
+          parts.push({
+            type: "image",
+            image: source.value,
+            mediaType: source.mimeType,
+          });
+        } else if (source.type === "url") {
+          try {
+            parts.push({
+              type: "image",
+              image: new URL(source.value),
+              mediaType: source.mimeType,
+            });
+          } catch {
+            console.error(
+              `[CopilotKit] convertUserMessageContent: invalid URL "${source.value}" in image part — skipping`,
+            );
+          }
+        }
+        break;
+      }
+
+      case "audio":
+      case "video":
+      case "document": {
+        const source = (part as { source?: any }).source;
+        if (!source) break;
+        if (source.type === "data") {
+          parts.push({
+            type: "file",
+            data: source.value,
+            mediaType: source.mimeType,
+          });
+        } else if (source.type === "url") {
+          try {
+            parts.push({
+              type: "file",
+              data: new URL(source.value),
+              mediaType: source.mimeType ?? "application/octet-stream",
+            });
+          } catch {
+            console.error(
+              `[CopilotKit] convertUserMessageContent: invalid URL "${source.value}" in ${part.type} part — skipping`,
+            );
+          }
+        }
+        break;
+      }
+
+      // Legacy BinaryInputContent backward compatibility
+      case "binary": {
+        const legacy = part as {
+          mimeType?: string;
+          data?: string;
+          url?: string;
+        };
+        const mimeType = legacy.mimeType ?? "application/octet-stream";
+        const isImage = mimeType.startsWith("image/");
+
+        if (legacy.data) {
+          if (isImage) {
+            parts.push({
+              type: "image",
+              image: legacy.data,
+              mediaType: mimeType,
+            });
+          } else {
+            parts.push({
+              type: "file",
+              data: legacy.data,
+              mediaType: mimeType,
+            });
+          }
+        } else if (legacy.url) {
+          try {
+            const url = new URL(legacy.url);
+            if (isImage) {
+              parts.push({ type: "image", image: url, mediaType: mimeType });
+            } else {
+              parts.push({ type: "file", data: url, mediaType: mimeType });
+            }
+          } catch {
+            console.error(
+              `[CopilotKit] convertUserMessageContent: invalid URL "${legacy.url}" in binary part — skipping`,
+            );
+          }
+        }
+        break;
+      }
+
+      default: {
+        console.error(
+          `[CopilotKit] convertUserMessageContent: unrecognized content part type "${(part as { type: string }).type}" — skipping`,
+        );
+        break;
       }
-      return "";
-    })
-    .filter((text) => text.length > 0)
-    .join("\n");
+    }
+  }
+
+  return parts.length > 0 ? parts : "";
 }
 
 /**
@@ -363,7 +474,7 @@ export function convertMessagesToVercelAISDKMessages(
     } else if (message.role === "user") {
       const userMsg: UserModelMessage = {
         role: "user",
-        content: flattenUserMessageContent(message.content),
+        content: convertUserMessageContent(message.content),
       };
       result.push(userMsg);
     } else if (message.role === "tool") {

package/src/lib/runtime/agent-integrations/langgraph/agent.ts

@@ -1,6 +1,6 @@
-import { map } from "rxjs";
+import { map, Observable } from "rxjs";
 import { LangGraphEventTypes } from "../../../../agents/langgraph/events";
-import { RawEvent } from "@ag-ui/core";
+import { BaseEvent, RawEvent } from "@ag-ui/core";
 import {
   LangGraphAgent as AGUILangGraphAgent,
   LangGraphHttpAgent,
@@ -153,7 +153,7 @@ export class LangGraphAgent extends AGUILangGraphAgent {
   }
 
   // @ts-ignore
-  run(input: RunAgentInput) {
+  run(input: RunAgentInput): Observable<BaseEvent> {
     return super.run(input).pipe(
       map((processedEvent) => {
         // Turn raw event into emit state snapshot from tool call event

package/src/lib/runtime/copilot-runtime.ts

@@ -369,6 +369,7 @@ export class CopilotRuntime<const T extends Parameter[] | [] = []> {
         : {}),
       a2ui: params?.a2ui,
       mcpApps: params?.mcpApps,
+      openGenerativeUI: params?.openGenerativeUI,
     };
     this.params = params;
     this.observability = params?.observability_c;

package/src/v2/express.ts

@@ -0,0 +1 @@
+export * from "./runtime/express";

package/src/v2/hono.ts

@@ -0,0 +1 @@
+export * from "./runtime/hono";

package/src/v2/node.ts

@@ -0,0 +1 @@
+export * from "./runtime/node";