@copilotkit/runtime 1.55.1 → 1.55.2-next.1

package/src/agent/__tests__/multimodal-tanstack.test.ts

@@ -0,0 +1,284 @@
+import { describe, it, expect } from "vitest";
+import { convertInputToTanStackAI } from "../converters/tanstack";
+import { createDefaultInput } from "./agent-test-helpers";
+import type { Message, InputContent } from "@ag-ui/client";
+
+/**
+ * Helper: build a user message with the given content parts, run through
+ * convertInputToTanStackAI, and return the first converted message for assertion.
+ */
+function convertUserContent(content: string | InputContent[]) {
+  const input = createDefaultInput({
+    messages: [{ role: "user", content } as Message],
+  });
+  const { messages } = convertInputToTanStackAI(input);
+  return messages[0];
+}
+
+/** Shorthand factories for AG-UI input parts */
+function dataSource(value: string, mimeType: string) {
+  return { type: "data" as const, value, mimeType };
+}
+
+function urlSource(value: string, mimeType?: string) {
+  return { type: "url" as const, value, ...(mimeType ? { mimeType } : {}) };
+}
+
+describe("convertInputToTanStackAI — multimodal", () => {
+  it("passes through plain string user content", () => {
+    const result = convertUserContent("Hello");
+    expect(result.content).toBe("Hello");
+  });
+
+  it("converts text-only InputContent[] to TanStack TextPart array", () => {
+    const result = convertUserContent([{ type: "text", text: "Hello world" }]);
+    expect(result.content).toEqual([{ type: "text", content: "Hello world" }]);
+  });
+
+  it("converts ImageInputPart with data source", () => {
+    const result = convertUserContent([
+      { type: "text", text: "What is this?" },
+      { type: "image", source: dataSource("iVBORw0KGgo=", "image/png") },
+    ]);
+    expect(result.content).toEqual([
+      { type: "text", content: "What is this?" },
+      {
+        type: "image",
+        source: { type: "data", value: "iVBORw0KGgo=", mimeType: "image/png" },
+      },
+    ]);
+  });
+
+  it("converts ImageInputPart with url source", () => {
+    const result = convertUserContent([
+      {
+        type: "image",
+        source: urlSource("https://example.com/photo.jpg", "image/jpeg"),
+      },
+    ]);
+    expect(result.content).toEqual([
+      {
+        type: "image",
+        source: {
+          type: "url",
+          value: "https://example.com/photo.jpg",
+          mimeType: "image/jpeg",
+        },
+      },
+    ]);
+  });
+
+  it("converts AudioInputPart", () => {
+    const result = convertUserContent([
+      { type: "audio", source: dataSource("base64audiodata", "audio/mp3") },
+    ]);
+    expect(result.content).toEqual([
+      {
+        type: "audio",
+        source: {
+          type: "data",
+          value: "base64audiodata",
+          mimeType: "audio/mp3",
+        },
+      },
+    ]);
+  });
+
+  it("converts VideoInputPart with url source", () => {
+    const result = convertUserContent([
+      {
+        type: "video",
+        source: urlSource("https://example.com/video.mp4", "video/mp4"),
+      },
+    ]);
+    expect(result.content).toEqual([
+      {
+        type: "video",
+        source: {
+          type: "url",
+          value: "https://example.com/video.mp4",
+          mimeType: "video/mp4",
+        },
+      },
+    ]);
+  });
+
+  it("converts DocumentInputPart", () => {
+    const result = convertUserContent([
+      {
+        type: "document",
+        source: dataSource("base64pdfdata", "application/pdf"),
+      },
+    ]);
+    expect(result.content).toEqual([
+      {
+        type: "document",
+        source: {
+          type: "data",
+          value: "base64pdfdata",
+          mimeType: "application/pdf",
+        },
+      },
+    ]);
+  });
+
+  it("handles mixed text and multimodal parts", () => {
+    const result = convertUserContent([
+      { type: "text", text: "Analyze these:" },
+      { type: "image", source: dataSource("imgdata", "image/png") },
+      {
+        type: "document",
+        source: dataSource("docdata", "application/pdf"),
+      },
+    ]);
+    expect(result.content).toEqual([
+      { type: "text", content: "Analyze these:" },
+      {
+        type: "image",
+        source: { type: "data", value: "imgdata", mimeType: "image/png" },
+      },
+      {
+        type: "document",
+        source: {
+          type: "data",
+          value: "docdata",
+          mimeType: "application/pdf",
+        },
+      },
+    ]);
+  });
+
+  it("returns empty string for empty content array", () => {
+    const result = convertUserContent([]);
+    expect(result.content).toBe("");
+  });
+
+  it("preserves empty string text parts", () => {
+    const result = convertUserContent([{ type: "text", text: "" }]);
+    expect(result.content).toEqual([{ type: "text", content: "" }]);
+  });
+
+  it("skips parts with missing source without crashing", () => {
+    const result = convertUserContent([
+      { type: "text", text: "check this" },
+      { type: "image" } as any,
+    ]);
+    expect(result.content).toEqual([{ type: "text", content: "check this" }]);
+  });
+
+  it("silently skips unknown part types", () => {
+    const result = convertUserContent([
+      { type: "text", text: "hello" },
+      { type: "spreadsheet", source: dataSource("data", "text/csv") } as any,
+    ]);
+    expect(result.content).toEqual([{ type: "text", content: "hello" }]);
+  });
+
+  it("returns null for null or undefined content", () => {
+    const nullInput = createDefaultInput({
+      messages: [{ role: "user", content: null } as unknown as Message],
+    });
+    const { messages: nullMessages } = convertInputToTanStackAI(nullInput);
+    expect(nullMessages[0].content).toBeNull();
+
+    const undefinedInput = createDefaultInput({
+      messages: [{ role: "user", content: undefined } as unknown as Message],
+    });
+    const { messages: undefinedMessages } =
+      convertInputToTanStackAI(undefinedInput);
+    expect(undefinedMessages[0].content).toBeNull();
+  });
+
+  describe("legacy BinaryInputContent backward compat", () => {
+    it("converts binary with image mimeType and data", () => {
+      const legacyPart = {
+        type: "binary",
+        mimeType: "image/jpeg",
+        data: "legacybase64",
+      };
+      const input = createDefaultInput({
+        messages: [
+          {
+            role: "user",
+            content: [legacyPart] as unknown as InputContent[],
+          } as Message,
+        ],
+      });
+      const { messages } = convertInputToTanStackAI(input);
+      expect(messages[0].content).toEqual([
+        {
+          type: "image",
+          source: {
+            type: "data",
+            value: "legacybase64",
+            mimeType: "image/jpeg",
+          },
+        },
+      ]);
+    });
+
+    it("skips binary with neither data nor url", () => {
+      const legacyPart = {
+        type: "binary",
+        mimeType: "image/png",
+      };
+      const input = createDefaultInput({
+        messages: [
+          {
+            role: "user",
+            content: [legacyPart] as unknown as InputContent[],
+          } as Message,
+        ],
+      });
+      const { messages } = convertInputToTanStackAI(input);
+      expect(messages[0].content).toBe("");
+    });
+
+    it("converts binary with non-image mimeType and url", () => {
+      const legacyPart = {
+        type: "binary",
+        mimeType: "application/pdf",
+        url: "https://example.com/doc.pdf",
+      };
+      const input = createDefaultInput({
+        messages: [
+          {
+            role: "user",
+            content: [legacyPart] as unknown as InputContent[],
+          } as Message,
+        ],
+      });
+      const { messages } = convertInputToTanStackAI(input);
+      expect(messages[0].content).toEqual([
+        {
+          type: "document",
+          source: {
+            type: "url",
+            value: "https://example.com/doc.pdf",
+            mimeType: "application/pdf",
+          },
+        },
+      ]);
+    });
+  });
+
+  it("only converts user message content, not assistant messages", () => {
+    const input = createDefaultInput({
+      messages: [
+        {
+          role: "user",
+          content: [
+            { type: "text", text: "Look at this" },
+            { type: "image", source: dataSource("imgdata", "image/png") },
+          ],
+        } as Message,
+        { role: "assistant", content: "I see an image" } as Message,
+      ],
+    });
+    const { messages } = convertInputToTanStackAI(input);
+    // User message should have array content
+    expect(Array.isArray(messages[0].content)).toBe(true);
+    // Assistant message should keep string content
+    expect(messages[1].content).toBe("I see an image");
+  });
+});

package/src/agent/__tests__/test-helpers.ts

@@ -178,16 +178,20 @@ export async function collectEvents(
 ): Promise<BaseEvent[]> {
   return new Promise((resolve, reject) => {
     const events: BaseEvent[] = [];
-    const subscription = observable.subscribe({
-      next: (event) => events.push(event),
-      error: (err: unknown) => reject(err),
-      complete: () => resolve(events),
-    });
-
-    // Set a timeout to prevent hanging tests
-    setTimeout(() => {
+    const timeoutId = setTimeout(() => {
       subscription.unsubscribe();
       reject(new Error("Observable did not complete within timeout"));
     }, 5000);
+    const subscription = observable.subscribe({
+      next: (event) => events.push(event),
+      error: (err: unknown) => {
+        clearTimeout(timeoutId);
+        reject(err);
+      },
+      complete: () => {
+        clearTimeout(timeoutId);
+        resolve(events);
+      },
+    });
   });
 }

package/src/agent/converters/aisdk.ts

@@ -0,0 +1,326 @@
+import {
+  BaseEvent,
+  EventType,
+  ReasoningEndEvent,
+  ReasoningMessageContentEvent,
+  ReasoningMessageEndEvent,
+  ReasoningMessageStartEvent,
+  ReasoningStartEvent,
+  TextMessageChunkEvent,
+  ToolCallArgsEvent,
+  ToolCallEndEvent,
+  ToolCallStartEvent,
+  ToolCallResultEvent,
+  StateSnapshotEvent,
+  StateDeltaEvent,
+} from "@ag-ui/client";
+import { randomUUID } from "@copilotkit/shared";
+
+/**
+ * Converts an AI SDK `fullStream` into AG-UI `BaseEvent` objects.
+ *
+ * This is a pure converter — it does NOT emit lifecycle events
+ * (RUN_STARTED / RUN_FINISHED / RUN_ERROR). The caller (Agent class)
+ * is responsible for those.
+ *
+ * Terminal stream events (finish, error, abort) cause the generator to
+ * return so the caller can handle lifecycle appropriately.
+ */
+export async function* convertAISDKStream(
+  fullStream: AsyncIterable<unknown>,
+  abortSignal: AbortSignal,
+): AsyncGenerator<BaseEvent> {
+  let messageId = randomUUID();
+  let reasoningMessageId = randomUUID();
+  let isInReasoning = false;
+
+  const toolCallStates = new Map<
+    string,
+    {
+      started: boolean;
+      hasArgsDelta: boolean;
+      ended: boolean;
+      toolName?: string;
+    }
+  >();
+
+  const ensureToolCallState = (toolCallId: string) => {
+    let state = toolCallStates.get(toolCallId);
+    if (!state) {
+      state = { started: false, hasArgsDelta: false, ended: false };
+      toolCallStates.set(toolCallId, state);
+    }
+    return state;
+  };
+
+  /**
+   * Auto-close an open reasoning lifecycle.
+   * Some AI SDK providers (notably @ai-sdk/anthropic) never emit "reasoning-end",
+   * which leaves downstream state machines stuck. This helper emits the
+   * missing REASONING_MESSAGE_END + REASONING_END events so the stream
+   * can transition to text, tool-call, or finish phases.
+   */
+  function* closeReasoningIfOpen(): Generator<BaseEvent> {
+    if (!isInReasoning) return;
+    isInReasoning = false;
+    const reasoningMsgEnd: ReasoningMessageEndEvent = {
+      type: EventType.REASONING_MESSAGE_END,
+      messageId: reasoningMessageId,
+    };
+    yield reasoningMsgEnd;
+    const reasoningEnd: ReasoningEndEvent = {
+      type: EventType.REASONING_END,
+      messageId: reasoningMessageId,
+    };
+    yield reasoningEnd;
+  }
+
+  try {
+    for await (const part of fullStream) {
+      const p = part as Record<string, unknown>;
+
+      // Close any open reasoning lifecycle on every event except
+      // reasoning-delta, which arrives mid-block and must not interrupt it.
+      if (p.type !== "reasoning-delta") {
+        yield* closeReasoningIfOpen();
+      }
+
+      switch (p.type) {
+        case "abort": {
+          // Terminal — let the caller handle lifecycle
+          return;
+        }
+
+        case "reasoning-start": {
+          // Use SDK-provided id, or generate a fresh UUID if id is falsy/"0"
+          // to prevent consecutive reasoning blocks from sharing a messageId
+          const providedId = "id" in p ? p.id : undefined;
+          reasoningMessageId =
+            providedId && providedId !== "0"
+              ? (providedId as string)
+              : randomUUID();
+          const reasoningStartEvent: ReasoningStartEvent = {
+            type: EventType.REASONING_START,
+            messageId: reasoningMessageId,
+          };
+          yield reasoningStartEvent;
+          const reasoningMessageStart: ReasoningMessageStartEvent = {
+            type: EventType.REASONING_MESSAGE_START,
+            messageId: reasoningMessageId,
+            role: "reasoning",
+          };
+          yield reasoningMessageStart;
+          isInReasoning = true;
+          break;
+        }
+
+        case "reasoning-delta": {
+          const delta = (p.text as string) ?? "";
+          if (!delta) break; // skip — @ag-ui/core schema requires delta to be non-empty
+          const reasoningDeltaEvent: ReasoningMessageContentEvent = {
+            type: EventType.REASONING_MESSAGE_CONTENT,
+            messageId: reasoningMessageId,
+            delta,
+          };
+          yield reasoningDeltaEvent;
+          break;
+        }
+
+        case "reasoning-end": {
+          // closeReasoningIfOpen() already called before the switch — no-op here
+          // if the SDK never emits this event (e.g. @ai-sdk/anthropic).
+          break;
+        }
+
+        case "tool-input-start": {
+          const toolCallId = p.id as string;
+          const state = ensureToolCallState(toolCallId);
+          state.toolName = p.toolName as string;
+          if (!state.started) {
+            state.started = true;
+            const startEvent: ToolCallStartEvent = {
+              type: EventType.TOOL_CALL_START,
+              parentMessageId: messageId,
+              toolCallId,
+              toolCallName: p.toolName as string,
+            };
+            yield startEvent;
+          }
+          break;
+        }
+
+        case "tool-input-delta": {
+          const toolCallId = p.id as string;
+          const state = ensureToolCallState(toolCallId);
+          state.hasArgsDelta = true;
+          const argsEvent: ToolCallArgsEvent = {
+            type: EventType.TOOL_CALL_ARGS,
+            toolCallId,
+            delta: p.delta as string,
+          };
+          yield argsEvent;
+          break;
+        }
+
+        case "tool-input-end": {
+          // No direct event – the subsequent "tool-call" part marks completion.
+          break;
+        }
+
+        case "text-start": {
+          // New text message starting - use the SDK-provided id
+          // Use randomUUID() if part.id is falsy or "0" to prevent message merging issues
+          const providedId = "id" in p ? p.id : undefined;
+          messageId =
+            providedId && providedId !== "0"
+              ? (providedId as string)
+              : randomUUID();
+          break;
+        }
+
+        case "text-delta": {
+          // AI SDK text-delta events use 'text' (not 'delta')
+          const textDelta = "text" in p ? (p.text as string) : "";
+          const textEvent: TextMessageChunkEvent = {
+            type: EventType.TEXT_MESSAGE_CHUNK,
+            role: "assistant",
+            messageId,
+            delta: textDelta,
+          };
+          yield textEvent;
+          break;
+        }
+
+        case "tool-call": {
+          const toolCallId = p.toolCallId as string;
+          const state = ensureToolCallState(toolCallId);
+          state.toolName = (p.toolName as string) ?? state.toolName;
+
+          if (!state.started) {
+            state.started = true;
+            const startEvent: ToolCallStartEvent = {
+              type: EventType.TOOL_CALL_START,
+              parentMessageId: messageId,
+              toolCallId,
+              toolCallName: p.toolName as string,
+            };
+            yield startEvent;
+          }
+
+          if (!state.hasArgsDelta && "input" in p && p.input !== undefined) {
+            let serializedInput = "";
+            if (typeof p.input === "string") {
+              serializedInput = p.input;
+            } else {
+              try {
+                serializedInput = JSON.stringify(p.input);
+              } catch {
+                serializedInput = String(p.input);
+              }
+            }
+
+            if (serializedInput.length > 0) {
+              const argsEvent: ToolCallArgsEvent = {
+                type: EventType.TOOL_CALL_ARGS,
+                toolCallId,
+                delta: serializedInput,
+              };
+              yield argsEvent;
+              state.hasArgsDelta = true;
+            }
+          }
+
+          if (!state.ended) {
+            state.ended = true;
+            const endEvent: ToolCallEndEvent = {
+              type: EventType.TOOL_CALL_END,
+              toolCallId,
+            };
+            yield endEvent;
+          }
+          break;
+        }
+
+        case "tool-result": {
+          // AI SDK tool-result uses "output"; older versions used "result" — check both
+          const toolResult =
+            "output" in p ? p.output : "result" in p ? p.result : null;
+          const toolName = "toolName" in p ? (p.toolName as string) : "";
+          toolCallStates.delete(p.toolCallId as string);
+
+          // Check if this is a state update tool
+          if (
+            toolName === "AGUISendStateSnapshot" &&
+            toolResult &&
+            typeof toolResult === "object"
+          ) {
+            const snapshot = (toolResult as Record<string, unknown>).snapshot;
+            if (snapshot !== undefined) {
+              const stateSnapshotEvent: StateSnapshotEvent = {
+                type: EventType.STATE_SNAPSHOT,
+                snapshot,
+              };
+              yield stateSnapshotEvent;
+            }
+          } else if (
+            toolName === "AGUISendStateDelta" &&
+            toolResult &&
+            typeof toolResult === "object"
+          ) {
+            const delta = (toolResult as Record<string, unknown>).delta;
+            if (delta !== undefined) {
+              const stateDeltaEvent: StateDeltaEvent = {
+                type: EventType.STATE_DELTA,
+                delta,
+              };
+              yield stateDeltaEvent;
+            }
+          }
+
+          // Always emit the tool result event for the LLM
+          let serializedResult: string;
+          try {
+            serializedResult = JSON.stringify(toolResult);
+          } catch {
+            serializedResult = `[Unserializable tool result from ${toolName || "unknown tool"}]`;
+          }
+          const resultEvent: ToolCallResultEvent = {
+            type: EventType.TOOL_CALL_RESULT,
+            role: "tool",
+            messageId: randomUUID(),
+            toolCallId: p.toolCallId as string,
+            content: serializedResult,
+          };
+          yield resultEvent;
+          break;
+        }
+
+        case "finish": {
+          // Terminal — let the caller handle lifecycle
+          return;
+        }
+
+        case "error": {
+          if (abortSignal.aborted) {
+            return;
+          }
+          // Re-throw so the caller can emit RUN_ERROR
+          const err = p.error ?? p.message ?? p.cause;
+          if (err instanceof Error) throw err;
+          throw new Error(
+            typeof err === "string"
+              ? err
+              : `AI SDK stream error: ${JSON.stringify(p)}`,
+          );
+        }
+
+        default:
+          // Unknown event types are silently ignored
+          break;
+      }
+    }
+  } finally {
+    // Always close reasoning on exit (normal or exceptional)
+    yield* closeReasoningIfOpen();
+  }
+}

package/src/agent/converters/index.ts

@@ -0,0 +1,7 @@
+export { convertAISDKStream } from "./aisdk";
+export {
+  convertTanStackStream,
+  convertInputToTanStackAI,
+  type TanStackChatMessage,
+  type TanStackInputResult,
+} from "./tanstack";