@copilotkit/shared 1.55.0-next.9 → 1.55.1-next.0

package/package.json

@@ -1,19 +1,32 @@
 {
   "name": "@copilotkit/shared",
+  "version": "1.55.1-next.0",
   "private": false,
+  "keywords": [
+    "ai",
+    "assistant",
+    "automation",
+    "copilot",
+    "copilotkit",
+    "javascript",
+    "nextjs",
+    "nodejs",
+    "react",
+    "textarea"
+  ],
   "homepage": "https://github.com/CopilotKit/CopilotKit",
+  "license": "MIT",
   "repository": {
     "type": "git",
     "url": "https://github.com/CopilotKit/CopilotKit.git"
   },
-  "publishConfig": {
-    "access": "public"
-  },
   "type": "module",
-  "version": "1.55.0-next.9",
   "sideEffects": false,
   "main": "./dist/index.cjs",
   "module": "./dist/index.mjs",
+  "types": "./dist/index.d.cts",
+  "unpkg": "./dist/index.umd.js",
+  "jsdelivr": "./dist/index.umd.js",
   "exports": {
     ".": {
       "import": "./dist/index.mjs",
@@ -21,10 +34,21 @@
     },
     "./package.json": "./package.json"
   },
-  "unpkg": "./dist/index.umd.js",
-  "jsdelivr": "./dist/index.umd.js",
-  "types": "./dist/index.d.cts",
-  "license": "MIT",
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@ag-ui/client": "0.0.52",
+    "@copilotkit/license-verifier": "0.0.1-a1",
+    "@segment/analytics-node": "^2.1.2",
+    "@standard-schema/spec": "^1.0.0",
+    "chalk": "4.1.2",
+    "graphql": "^16.8.1",
+    "partial-json": "^0.1.7",
+    "uuid": "^11.1.0",
+    "zod": "^3.23.3",
+    "zod-to-json-schema": "^3.23.5"
+  },
   "devDependencies": {
     "@types/uuid": "^10.0.0",
     "@valibot/to-json-schema": "^1.5.0",
@@ -34,36 +58,11 @@
     "typescript": "^5.2.3",
     "valibot": "^1.2.0",
     "vitest": "^3.2.4",
-    "zod-to-json-schema": "^3.23.5",
-    "eslint-config-custom": "1.4.12",
     "tsconfig": "1.4.12"
   },
-  "dependencies": {
-    "@ag-ui/client": "0.0.47",
-    "@segment/analytics-node": "^2.1.2",
-    "@standard-schema/spec": "^1.0.0",
-    "chalk": "4.1.2",
-    "graphql": "^16.8.1",
-    "partial-json": "^0.1.7",
-    "uuid": "^11.1.0",
-    "@copilotkit/license-verifier": "0.0.1-a1",
-    "zod": "^3.23.3"
-  },
   "peerDependencies": {
-    "@ag-ui/core": "^0.0.48"
+    "@ag-ui/core": ">=0.0.48"
   },
-  "keywords": [
-    "copilotkit",
-    "copilot",
-    "react",
-    "nextjs",
-    "nodejs",
-    "ai",
-    "assistant",
-    "javascript",
-    "automation",
-    "textarea"
-  ],
   "scripts": {
     "build": "tsdown",
     "dev": "tsdown --watch",

package/src/a2ui-prompts.ts

@@ -0,0 +1,101 @@
+/**
+ * Default A2UI generation and design guideline prompts.
+ *
+ * These are the canonical prompt fragments that instruct an LLM how to call
+ * the render_a2ui tool, how to bind data, and how to style surfaces.
+ */
+
+/**
+ * Generation guidelines — protocol rules, tool arguments, path rules,
+ * data model format, and form/two-way-binding instructions.
+ */
+export const A2UI_DEFAULT_GENERATION_GUIDELINES = `\
+Generate A2UI v0.9 JSON.
+
+## A2UI Protocol Instructions
+
+A2UI (Agent to UI) is a protocol for rendering rich UI surfaces from agent responses.
+
+CRITICAL: You MUST call the render_a2ui tool with ALL of these arguments:
+- surfaceId: A unique ID for the surface (e.g. "product-comparison")
+- components: REQUIRED — the A2UI component array. NEVER omit this. Only use
+  components listed in the Available Components schema provided as context.
+- data: OPTIONAL — a JSON object written to the root of the surface data model.
+  Use for pre-filling form values or providing data for path-bound components.
+- every component must have the "component" field specifying the component type.
+  ONLY use component names from the Available Components schema — do NOT invent
+  component names or use names not in the schema.
+
+COMPONENT ID RULES:
+- Every component ID must be unique within the surface.
+- A component MUST NOT reference itself as child/children. This causes a
+  circular dependency error. For example, if a component has id="avatar",
+  its child must be a DIFFERENT id (e.g. "avatar-img"), never "avatar".
+- The child/children tree must be a DAG — no cycles allowed.
+
+REPEATING CONTENT (TEMPLATES):
+To repeat a component for each item in an array, use the structural children format:
+  children: { componentId: "card-id", path: "/items" }
+This tells the renderer to create one instance of "card-id" per item in the "/items" array.
+
+PATH RULES FOR TEMPLATES:
+Components inside a repeating template use RELATIVE paths (no leading slash).
+The path is resolved relative to each array item automatically.
+If a container has children: { componentId: "card", path: "/items" } and each item
+has a key "name", use { "path": "name" } (NO leading slash — relative to item).
+CRITICAL: Do NOT use "/name" (absolute) inside templates — use "name" (relative).
+The container's path ("/items") uses a leading slash (absolute), but all
+components INSIDE the template use paths WITHOUT leading slash.
+
+DATA MODEL:
+The "data" key in the tool args is a plain JSON object that initializes the surface
+data model. Components bound to paths (e.g. "value": { "path": "/form/name" })
+read from and write to this data model. Examples:
+  For forms:  "data": { "form": { "name": "Alice", "email": "" } }
+  For lists:  "data": { "items": [{"name": "Product A"}, {"name": "Product B"}] }
+  For mixed:  "data": { "form": { "query": "" }, "results": [...] }
+
+FORMS AND TWO-WAY DATA BINDING:
+To create editable forms, bind input components to data model paths using { "path": "..." }.
+The client automatically writes user input back to the data model at the bound path.
+CRITICAL: Using a literal value (e.g. "value": "") makes the field READ-ONLY.
+You MUST use { "path": "..." } to make inputs editable.
+
+Input components use "value" as the binding property:
+  "value": { "path": "/form/fieldName" }
+
+To retrieve form values when a button is clicked, include "context" with path references
+in the button's action. Paths are resolved to their current values at click time:
+  "action": { "event": { "name": "submit", "context": { "userName": { "path": "/form/name" } } } }
+
+To pre-fill form values, pass initial data via the "data" tool argument:
+  "data": { "form": { "name": "Markus" } }`;
+
+/**
+ * Design guidelines — visual design rules, component hierarchy tips,
+ * and action handler patterns.
+ */
+export const A2UI_DEFAULT_DESIGN_GUIDELINES = `\
+Create polished, visually appealing interfaces. ONLY use components listed in the
+Available Components schema — do NOT use component names that are not in the schema.
+
+Design principles:
+- Create clear visual hierarchy within cards and layouts.
+- Keep cards clean — avoid clutter. Whitespace is good.
+- Use consistent surfaceIds (lowercase, hyphenated).
+- NEVER use the same ID for a component and its child — this creates a
+  circular dependency. E.g. if id="avatar", child must NOT be "avatar".
+- For side-by-side comparisons, use a container with structural children
+  (children: { componentId, path }) to repeat a card template per data item.
+- Include images when relevant (logos, icons, product photos):
+  - Prefer company logos via Google favicons: https://www.google.com/s2/favicons?domain=example.com&sz=128
+  - Do NOT invent Unsplash photo-IDs — they will 404. Only use real, known URLs.
+- For buttons: action MUST use this exact nested format:
+    "action": { "event": { "name": "myAction", "context": { "key": "value" } } }
+  The "event" key holds an OBJECT with "name" (required) and "context" (optional).
+  Do NOT use a flat format like {"event": "name"} — "event" must be an object.
+- For forms: every input MUST use path binding on the "value" property
+  (e.g. "value": { "path": "/form/name" }) to be editable. The submit button's
+  action context MUST reference the same paths to capture the user's input.
+
+Use the SAME surfaceId as the main surface. Match action names to button action event names.`;

package/src/attachments/__tests__/utils.test.ts

@@ -0,0 +1,198 @@
+import {
+  getModalityFromMimeType,
+  formatFileSize,
+  exceedsMaxSize,
+  matchesAcceptFilter,
+} from "../utils";
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function mockFile(size: number): File {
+  return { size } as File;
+}
+
+function mockFileWithType(type: string, name = "test"): File {
+  return { type, name } as File;
+}
+
+// ---------------------------------------------------------------------------
+// getModalityFromMimeType
+// ---------------------------------------------------------------------------
+
+describe("getModalityFromMimeType", () => {
+  it('returns "image" for image/png', () => {
+    expect(getModalityFromMimeType("image/png")).toBe("image");
+  });
+
+  it('returns "image" for image/jpeg', () => {
+    expect(getModalityFromMimeType("image/jpeg")).toBe("image");
+  });
+
+  it('returns "audio" for audio/mp3', () => {
+    expect(getModalityFromMimeType("audio/mp3")).toBe("audio");
+  });
+
+  it('returns "audio" for audio/wav', () => {
+    expect(getModalityFromMimeType("audio/wav")).toBe("audio");
+  });
+
+  it('returns "video" for video/mp4', () => {
+    expect(getModalityFromMimeType("video/mp4")).toBe("video");
+  });
+
+  it('returns "video" for video/webm', () => {
+    expect(getModalityFromMimeType("video/webm")).toBe("video");
+  });
+
+  it('returns "document" for application/pdf', () => {
+    expect(getModalityFromMimeType("application/pdf")).toBe("document");
+  });
+
+  it('returns "document" for text/plain', () => {
+    expect(getModalityFromMimeType("text/plain")).toBe("document");
+  });
+
+  it('returns "document" for empty string (fallback)', () => {
+    expect(getModalityFromMimeType("")).toBe("document");
+  });
+});
+
+// ---------------------------------------------------------------------------
+// formatFileSize
+// ---------------------------------------------------------------------------
+
+describe("formatFileSize", () => {
+  it('formats 0 bytes as "0 B"', () => {
+    expect(formatFileSize(0)).toBe("0 B");
+  });
+
+  it('formats 512 bytes as "512 B"', () => {
+    expect(formatFileSize(512)).toBe("512 B");
+  });
+
+  it('formats 1023 bytes as "1023 B"', () => {
+    expect(formatFileSize(1023)).toBe("1023 B");
+  });
+
+  it('formats 1024 bytes as "1.0 KB"', () => {
+    expect(formatFileSize(1024)).toBe("1.0 KB");
+  });
+
+  it('formats 1536 bytes as "1.5 KB"', () => {
+    expect(formatFileSize(1536)).toBe("1.5 KB");
+  });
+
+  it('formats 1048575 bytes as "1024.0 KB" (one byte under 1 MB)', () => {
+    expect(formatFileSize(1048575)).toBe("1024.0 KB");
+  });
+
+  it('formats 1048576 bytes as "1.0 MB"', () => {
+    expect(formatFileSize(1048576)).toBe("1.0 MB");
+  });
+
+  it('formats 10485760 bytes as "10.0 MB"', () => {
+    expect(formatFileSize(10485760)).toBe("10.0 MB");
+  });
+});
+
+// ---------------------------------------------------------------------------
+// exceedsMaxSize
+// ---------------------------------------------------------------------------
+
+const MB_20 = 20 * 1024 * 1024;
+
+describe("exceedsMaxSize", () => {
+  it("returns false for a file exactly at the 20 MB default limit", () => {
+    expect(exceedsMaxSize(mockFile(MB_20))).toBe(false);
+  });
+
+  it("returns true for a file one byte over the 20 MB default limit", () => {
+    expect(exceedsMaxSize(mockFile(MB_20 + 1))).toBe(true);
+  });
+
+  it("returns false for a file well under the default limit", () => {
+    expect(exceedsMaxSize(mockFile(1024))).toBe(false);
+  });
+
+  it("returns true when file exceeds a custom maxSize", () => {
+    const MB_5 = 5 * 1024 * 1024;
+    const MB_10 = 10 * 1024 * 1024;
+    expect(exceedsMaxSize(mockFile(MB_10), MB_5)).toBe(true);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// matchesAcceptFilter
+// ---------------------------------------------------------------------------
+
+describe("matchesAcceptFilter", () => {
+  it('returns true for accept "*/*" regardless of file type', () => {
+    expect(matchesAcceptFilter(mockFileWithType("image/png"), "*/*")).toBe(
+      true,
+    );
+  });
+
+  it("returns true for empty accept string (accept all)", () => {
+    expect(matchesAcceptFilter(mockFileWithType("image/png"), "")).toBe(true);
+  });
+
+  it('returns true when accept "image/*" matches "image/png"', () => {
+    expect(matchesAcceptFilter(mockFileWithType("image/png"), "image/*")).toBe(
+      true,
+    );
+  });
+
+  it('returns false when accept "image/*" rejects "audio/mp3"', () => {
+    expect(matchesAcceptFilter(mockFileWithType("audio/mp3"), "image/*")).toBe(
+      false,
+    );
+  });
+
+  it('returns true for exact match accept "application/pdf"', () => {
+    expect(
+      matchesAcceptFilter(
+        mockFileWithType("application/pdf"),
+        "application/pdf",
+      ),
+    ).toBe(true);
+  });
+
+  it('returns false when exact accept "application/pdf" rejects "image/png"', () => {
+    expect(
+      matchesAcceptFilter(mockFileWithType("image/png"), "application/pdf"),
+    ).toBe(false);
+  });
+
+  it('returns true for comma-separated accept "image/*,application/pdf" with "image/jpeg"', () => {
+    expect(
+      matchesAcceptFilter(
+        mockFileWithType("image/jpeg"),
+        "image/*,application/pdf",
+      ),
+    ).toBe(true);
+  });
+
+  it('returns true for comma-separated accept "image/*,application/pdf" with "application/pdf"', () => {
+    expect(
+      matchesAcceptFilter(
+        mockFileWithType("application/pdf"),
+        "image/*,application/pdf",
+      ),
+    ).toBe(true);
+  });
+
+  it("handles whitespace in comma-separated accept values", () => {
+    expect(
+      matchesAcceptFilter(
+        mockFileWithType("application/pdf"),
+        "image/* , application/pdf",
+      ),
+    ).toBe(true);
+  });
+
+  it("returns false for empty file type against a specific filter", () => {
+    expect(matchesAcceptFilter(mockFileWithType(""), "image/*")).toBe(false);
+  });
+});

package/src/attachments/index.ts

@@ -0,0 +1,19 @@
+export type {
+  AttachmentsConfig,
+  AttachmentUploadResult,
+  AttachmentUploadError,
+  AttachmentUploadErrorReason,
+  Attachment,
+  AttachmentModality,
+} from "./types";
+
+export {
+  getModalityFromMimeType,
+  formatFileSize,
+  exceedsMaxSize,
+  readFileAsBase64,
+  generateVideoThumbnail,
+  matchesAcceptFilter,
+  getSourceUrl,
+  getDocumentIcon,
+} from "./utils";

package/src/attachments/types.ts

@@ -0,0 +1,67 @@
+import type {
+  InputContentDataSource,
+  InputContentUrlSource,
+} from "@ag-ui/core";
+
+export interface AttachmentUploadDataResult {
+  type: "data";
+  value: string;
+  mimeType: string;
+  /** Custom metadata to include in the InputContent part (merged with auto-generated metadata like filename). */
+  metadata?: Record<string, unknown>;
+}
+
+export interface AttachmentUploadUrlResult {
+  type: "url";
+  value: string;
+  mimeType?: string;
+  /** Custom metadata to include in the InputContent part (merged with auto-generated metadata like filename). */
+  metadata?: Record<string, unknown>;
+}
+
+export type AttachmentUploadResult =
+  | AttachmentUploadDataResult
+  | AttachmentUploadUrlResult;
+
+export type AttachmentUploadErrorReason =
+  | "file-too-large"
+  | "invalid-type"
+  | "upload-failed";
+
+export interface AttachmentUploadError {
+  /** Why the upload failed. */
+  reason: AttachmentUploadErrorReason;
+  /** The file that failed to upload. */
+  file: File;
+  /** Human-readable error message. */
+  message: string;
+}
+
+export interface AttachmentsConfig {
+  /** Enable file attachments in the chat input */
+  enabled: boolean;
+  /** MIME type filter for the file input, default all files */
+  accept?: string;
+  /** Maximum file size in bytes, default 20MB (20 * 1024 * 1024) */
+  maxSize?: number;
+  /** Custom upload handler. Return an InputContentSource with optional metadata. */
+  onUpload?: (
+    file: File,
+  ) => AttachmentUploadResult | Promise<AttachmentUploadResult>;
+  /** Called when an attachment fails validation or upload. Use this to show a toast or inline error. */
+  onUploadFailed?: (error: AttachmentUploadError) => void;
+}
+
+export type AttachmentModality = "image" | "audio" | "video" | "document";
+
+export interface Attachment {
+  id: string;
+  type: AttachmentModality;
+  source: InputContentDataSource | InputContentUrlSource;
+  filename?: string;
+  size?: number;
+  status: "uploading" | "ready";
+  thumbnail?: string;
+  /** Custom metadata from onUpload, included in the InputContent part. */
+  metadata?: Record<string, unknown>;
+}

package/src/attachments/utils.ts

@@ -0,0 +1,164 @@
+import type { AttachmentModality } from "./types";
+import type { InputContentSource } from "../types/message";
+
+const DEFAULT_MAX_SIZE = 20 * 1024 * 1024; // 20MB
+
+/**
+ * Derive the attachment modality from a MIME type string.
+ */
+export function getModalityFromMimeType(mimeType: string): AttachmentModality {
+  if (mimeType.startsWith("image/")) return "image";
+  if (mimeType.startsWith("audio/")) return "audio";
+  if (mimeType.startsWith("video/")) return "video";
+  return "document";
+}
+
+/**
+ * Format a byte count as a human-readable file size string.
+ */
+export function formatFileSize(bytes: number): string {
+  if (bytes < 1024) return `${bytes} B`;
+  if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
+  return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+}
+
+/**
+ * Check if a file exceeds the maximum allowed size.
+ */
+export function exceedsMaxSize(
+  file: File,
+  maxSize: number = DEFAULT_MAX_SIZE,
+): boolean {
+  return file.size > maxSize;
+}
+
+/**
+ * Read a File as a base64 string (without the data URL prefix).
+ */
+export function readFileAsBase64(file: File): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const reader = new FileReader();
+    reader.onload = (e) => {
+      const result = e.target?.result as string;
+      const base64 = result?.split(",")[1];
+      if (base64) {
+        resolve(base64);
+      } else {
+        reject(new Error("Failed to read file as base64"));
+      }
+    };
+    reader.onerror = reject;
+    reader.readAsDataURL(file);
+  });
+}
+
+/**
+ * Generate a thumbnail data URL from a video file by capturing a frame near the start (at 0.1s).
+ * Returns undefined if thumbnail generation fails or if called outside a browser environment.
+ */
+export function generateVideoThumbnail(
+  file: File,
+): Promise<string | undefined> {
+  if (typeof document === "undefined") {
+    return Promise.resolve(undefined);
+  }
+  return new Promise((resolve) => {
+    let resolved = false;
+    const video = document.createElement("video");
+    const canvas = document.createElement("canvas");
+    const url = URL.createObjectURL(file);
+
+    const cleanup = (result: string | undefined) => {
+      if (resolved) return;
+      resolved = true;
+      URL.revokeObjectURL(url);
+      resolve(result);
+    };
+
+    const timeout = setTimeout(() => {
+      console.warn(
+        `[CopilotKit] generateVideoThumbnail: timed out for file "${file.name}"`,
+      );
+      cleanup(undefined);
+    }, 10000);
+
+    video.preload = "metadata";
+    video.muted = true;
+    video.playsInline = true;
+
+    video.onloadeddata = () => {
+      video.currentTime = 0.1;
+    };
+
+    video.onseeked = () => {
+      clearTimeout(timeout);
+      canvas.width = video.videoWidth;
+      canvas.height = video.videoHeight;
+      const ctx = canvas.getContext("2d");
+      if (ctx) {
+        ctx.drawImage(video, 0, 0);
+        const thumbnail = canvas.toDataURL("image/jpeg", 0.7);
+        cleanup(thumbnail);
+      } else {
+        console.warn(
+          "[CopilotKit] generateVideoThumbnail: could not get 2d canvas context",
+        );
+        cleanup(undefined);
+      }
+    };
+
+    video.onerror = () => {
+      clearTimeout(timeout);
+      console.warn(
+        `[CopilotKit] generateVideoThumbnail: video element error for file "${file.name}"`,
+      );
+      cleanup(undefined);
+    };
+
+    video.src = url;
+  });
+}
+
+/**
+ * Check if a file's MIME type matches an accept filter string.
+ * Handles file extensions (e.g. ".pdf"), MIME wildcards ("image/*"), and comma-separated lists.
+ */
+export function matchesAcceptFilter(file: File, accept: string): boolean {
+  if (!accept || accept === "*/*") return true;
+
+  const filters = accept.split(",").map((f) => f.trim());
+  return filters.some((filter) => {
+    if (filter.startsWith(".")) {
+      return (file.name ?? "").toLowerCase().endsWith(filter.toLowerCase());
+    }
+    if (filter.endsWith("/*")) {
+      const prefix = filter.slice(0, -2);
+      return file.type.startsWith(prefix + "/");
+    }
+    return file.type === filter;
+  });
+}
+
+/**
+ * Convert an InputContentSource to a usable URL string.
+ * For data sources, returns a base64 data URL; for URL sources, returns the URL directly.
+ */
+export function getSourceUrl(source: InputContentSource): string {
+  if (source.type === "url") {
+    return source.value;
+  }
+  return `data:${source.mimeType};base64,${source.value}`;
+}
+
+/**
+ * Return a short human-readable label for a document MIME type (e.g. "PDF", "DOC").
+ */
+export function getDocumentIcon(mimeType: string): string {
+  if (mimeType.includes("pdf")) return "PDF";
+  if (mimeType.includes("sheet") || mimeType.includes("excel")) return "XLS";
+  if (mimeType.includes("presentation") || mimeType.includes("powerpoint"))
+    return "PPT";
+  if (mimeType.includes("word") || mimeType.includes("document")) return "DOC";
+  if (mimeType.includes("text/")) return "TXT";
+  return "FILE";
+}