@assistant-ui/react-langchain 0.0.34 → 0.0.36

package/src/converter.test.ts

@@ -0,0 +1,322 @@
+import { describe, expect, it } from "vitest";
+import { convertLangChainContentBlock } from "./converter";
+
+describe("convertLangChainContentBlock standard content blocks", () => {
+  it("converts a base64 image block to an image part", () => {
+    expect(
+      convertLangChainContentBlock({
+        type: "image",
+        mimeType: "image/png",
+        data: "ZmFrZQ==",
+      }),
+    ).toEqual({ type: "image", image: "data:image/png;base64,ZmFrZQ==" });
+  });
+
+  it("converts a url image block to an image part", () => {
+    expect(
+      convertLangChainContentBlock({
+        type: "image",
+        mimeType: "image/png",
+        url: "https://cdn.example/a.png",
+      }),
+    ).toEqual({ type: "image", image: "https://cdn.example/a.png" });
+  });
+
+  it("keeps a base64 image block that already carries a data url", () => {
+    expect(
+      convertLangChainContentBlock({
+        type: "image",
+        mimeType: "image/png",
+        data: "data:image/png;base64,ZmFrZQ==",
+      }),
+    ).toEqual({ type: "image", image: "data:image/png;base64,ZmFrZQ==" });
+  });
+
+  it("keeps a percent-encoded data url out of a base64 envelope", () => {
+    const image = "data:image/svg+xml,%3Csvg%3E%3C/svg%3E";
+    expect(
+      convertLangChainContentBlock({
+        type: "image",
+        mimeType: "image/svg+xml",
+        data: image,
+      }),
+    ).toEqual({ type: "image", image });
+  });
+
+  it("keeps an uppercase data url scheme out of a base64 envelope", () => {
+    const image = "DATA:image/png;base64,ZmFrZQ==";
+    expect(
+      convertLangChainContentBlock({
+        type: "image",
+        mimeType: "image/png",
+        data: image,
+      }),
+    ).toEqual({ type: "image", image });
+  });
+
+  it("reads the camelCase mime type of a base64 file block", () => {
+    expect(
+      convertLangChainContentBlock({
+        type: "file",
+        mimeType: "application/pdf",
+        data: "JVBERi0=",
+      }),
+    ).toEqual({
+      type: "file",
+      filename: "file",
+      data: "JVBERi0=",
+      mimeType: "application/pdf",
+    });
+  });
+
+  it("resolves the url of a file block that carries no source_type", () => {
+    expect(
+      convertLangChainContentBlock({
+        type: "file",
+        mimeType: "application/pdf",
+        url: "https://cdn.example/a.pdf",
+        metadata: { filename: "a.pdf" },
+      }),
+    ).toEqual({
+      type: "file",
+      filename: "a.pdf",
+      data: "https://cdn.example/a.pdf",
+      mimeType: "application/pdf",
+      sourceType: "url",
+    });
+  });
+
+  it("resolves the fileId of a file block as an id source", () => {
+    expect(
+      convertLangChainContentBlock({
+        type: "file",
+        mimeType: "application/pdf",
+        fileId: "file-abc123",
+      }),
+    ).toEqual({
+      type: "file",
+      filename: "file",
+      data: "file-abc123",
+      mimeType: "application/pdf",
+      sourceType: "id",
+    });
+  });
+
+  it("reads the Python base64 field of an image block", () => {
+    expect(
+      convertLangChainContentBlock({
+        type: "image",
+        mime_type: "image/png",
+        base64: "ZmFrZQ==",
+      }),
+    ).toEqual({ type: "image", image: "data:image/png;base64,ZmFrZQ==" });
+  });
+
+  it("resolves the Python file_id field as an id source", () => {
+    expect(
+      convertLangChainContentBlock({
+        type: "file",
+        mime_type: "application/pdf",
+        file_id: "file-python-123",
+      }),
+    ).toEqual({
+      type: "file",
+      filename: "file",
+      data: "file-python-123",
+      mimeType: "application/pdf",
+      sourceType: "id",
+    });
+  });
+
+  it("converts a video block to a file part", () => {
+    expect(
+      convertLangChainContentBlock({
+        type: "video",
+        mimeType: "video/mp4",
+        data: "ZmFrZQ==",
+      }),
+    ).toEqual({
+      type: "file",
+      filename: "video.mp4",
+      data: "ZmFrZQ==",
+      mimeType: "video/mp4",
+    });
+  });
+
+  it("converts an audio block that names its mime type in camelCase", () => {
+    expect(
+      convertLangChainContentBlock({
+        type: "audio",
+        mimeType: "audio/wav",
+        data: "ZmFrZQ==",
+      }),
+    ).toEqual({
+      type: "file",
+      filename: "audio.wav",
+      data: "ZmFrZQ==",
+      mimeType: "audio/wav",
+    });
+  });
+
+  it("converts a text-plain block to a file part", () => {
+    expect(
+      convertLangChainContentBlock({
+        type: "text-plain",
+        data: "ZmFrZQ==",
+        metadata: { filename: "notes.txt" },
+      }),
+    ).toEqual({
+      type: "file",
+      filename: "notes.txt",
+      data: "ZmFrZQ==",
+      mimeType: "text/plain",
+    });
+  });
+
+  it("falls back to a file part for an image referenced by id", () => {
+    expect(
+      convertLangChainContentBlock({
+        type: "image",
+        mimeType: "image/png",
+        fileId: "file-abc123",
+      }),
+    ).toEqual({
+      type: "file",
+      filename: "file",
+      data: "file-abc123",
+      mimeType: "image/png",
+      sourceType: "id",
+    });
+  });
+
+  it("falls back to a file part for a base64 image with no mime type", () => {
+    expect(
+      convertLangChainContentBlock({ type: "image", data: "ZmFrZQ==" }),
+    ).toEqual({
+      type: "file",
+      filename: "file",
+      data: "ZmFrZQ==",
+      mimeType: "application/octet-stream",
+    });
+  });
+
+  it("reports a block whose payload is not a string as unknown", () => {
+    expect(
+      convertLangChainContentBlock({
+        type: "image",
+        mimeType: "image/png",
+        data: new Uint8Array([1, 2, 3]),
+      }),
+    ).toBeUndefined();
+  });
+
+  it("converts a text-plain block carrying inline text to a base64 file part", () => {
+    expect(
+      convertLangChainContentBlock({
+        type: "text-plain",
+        text: "hello",
+      }),
+    ).toEqual({
+      type: "file",
+      filename: "file",
+      data: "aGVsbG8=",
+      mimeType: "text/plain",
+    });
+  });
+
+  it("round-trips non-ASCII inline text through base64", () => {
+    const text = "héllo ✓ https://example.com";
+    const part = convertLangChainContentBlock({ type: "text-plain", text });
+    expect(part).toMatchObject({ type: "file", mimeType: "text/plain" });
+    const data = (part as { data: string }).data;
+    const bytes = Uint8Array.from(atob(data), (char) => char.charCodeAt(0));
+    expect(new TextDecoder().decode(bytes)).toBe(text);
+  });
+});
+
+describe("convertLangChainContentBlock legacy data content blocks", () => {
+  it("converts a base64 file block that names its mime type in snake_case", () => {
+    expect(
+      convertLangChainContentBlock({
+        type: "file",
+        data: "JVBERi0=",
+        mime_type: "application/pdf",
+        source_type: "base64",
+        metadata: { filename: "a.pdf" },
+      }),
+    ).toEqual({
+      type: "file",
+      filename: "a.pdf",
+      data: "JVBERi0=",
+      mimeType: "application/pdf",
+    });
+  });
+
+  it("converts a url source file block", () => {
+    expect(
+      convertLangChainContentBlock({
+        type: "file",
+        url: "https://cdn.example/a.pdf",
+        mime_type: "application/pdf",
+        source_type: "url",
+      }),
+    ).toEqual({
+      type: "file",
+      filename: "file",
+      data: "https://cdn.example/a.pdf",
+      mimeType: "application/pdf",
+      sourceType: "url",
+    });
+  });
+
+  it("converts an id source file block", () => {
+    expect(
+      convertLangChainContentBlock({
+        type: "file",
+        id: "file-abc123",
+        source_type: "id",
+      }),
+    ).toEqual({
+      type: "file",
+      filename: "file",
+      data: "file-abc123",
+      mimeType: "application/octet-stream",
+      sourceType: "id",
+    });
+  });
+
+  it("converts a base64 audio block", () => {
+    expect(
+      convertLangChainContentBlock({
+        type: "audio",
+        data: "ZmFrZQ==",
+        mime_type: "audio/mp3",
+        source_type: "base64",
+      }),
+    ).toEqual({
+      type: "file",
+      filename: "audio.mp3",
+      data: "ZmFrZQ==",
+      mimeType: "audio/mp3",
+    });
+  });
+
+  it("converts a base64 image data block", () => {
+    expect(
+      convertLangChainContentBlock({
+        type: "image",
+        data: "ZmFrZQ==",
+        mime_type: "image/png",
+        source_type: "base64",
+      }),
+    ).toEqual({ type: "image", image: "data:image/png;base64,ZmFrZQ==" });
+  });
+
+  it("leaves an unknown block type unconverted", () => {
+    expect(
+      convertLangChainContentBlock({
+        type: "non_standard",
+      } as never),
+    ).toBeUndefined();
+  });
+});

package/src/converter.ts

@@ -10,6 +10,7 @@ import {
   resolveFilePartSource,
 } from "@assistant-ui/core/internal";
 import type { StreamingTimingAccessors } from "@assistant-ui/core/react";
+import type { ReadonlyJSONObject } from "assistant-stream/utils";
 
 /** Known content block types from @langchain/core messages. */
 export type LangChainContentBlock =
@@ -51,12 +52,143 @@ export type LangChainContentBlock =
     }
   | { type: "tool_use" | "input_json_delta" };
 
+/**
+ * One block covers both multimodal vocabularies: the standard content blocks
+ * key their payload off a camelCase JavaScript field or snake_case Python
+ * field, while the legacy data content blocks use a `source_type` discriminator.
+ */
+type LangChainMediaBlock = {
+  type: "image" | "video" | "audio" | "file" | "text-plain";
+  mimeType?: string;
+  mime_type?: string;
+  data?: string | Uint8Array;
+  base64?: string | Uint8Array;
+  url?: string;
+  fileId?: string;
+  file_id?: string;
+  id?: string;
+  text?: string;
+  source_type?: "base64" | "url" | "id" | "text";
+  metadata?: { filename?: string };
+};
+
 type ConvertedContentPart =
   | ThreadUserMessage["content"][number]
   | ThreadAssistantMessage["content"][number];
 
+type MediaSource = { data: string; sourceType?: "url" | "id" };
+
+/**
+ * `id` names the block, not the payload, in the standard content blocks, so it
+ * only identifies a file under the legacy `source_type: "id"` discriminator.
+ */
+const resolveMediaSource = (
+  part: LangChainMediaBlock,
+): MediaSource | undefined => {
+  switch (part.source_type) {
+    case "url":
+      return typeof part.url === "string"
+        ? { data: part.url, sourceType: "url" }
+        : undefined;
+    case "id":
+      return typeof part.id === "string"
+        ? { data: part.id, sourceType: "id" }
+        : undefined;
+    case "base64": {
+      const data = part.data ?? part.base64;
+      return typeof data === "string" ? { data } : undefined;
+    }
+  }
+  if (typeof part.url === "string")
+    return { data: part.url, sourceType: "url" };
+  const fileId = part.fileId ?? part.file_id;
+  if (typeof fileId === "string") return { data: fileId, sourceType: "id" };
+  const data = part.data ?? part.base64;
+  if (typeof data === "string") return { data };
+  if (part.type === "text-plain" && typeof part.text === "string")
+    return { data: textToBase64(part.text) };
+  return undefined;
+};
+
+const textToBase64 = (text: string): string => {
+  const bytes = new TextEncoder().encode(text);
+  let binary = "";
+  for (let i = 0; i < bytes.length; i += 0x8000) {
+    binary += String.fromCharCode(...bytes.subarray(i, i + 0x8000));
+  }
+  return btoa(binary);
+};
+
+const resolveMediaMimeType = (part: LangChainMediaBlock): string | undefined =>
+  typeof part.mimeType === "string"
+    ? part.mimeType
+    : typeof part.mime_type === "string"
+      ? part.mime_type
+      : undefined;
+
+const defaultMediaFilename = (type: string, mimeType: string): string => {
+  if (type !== "audio" && type !== "video") return "file";
+  const subtype = mimeType.startsWith(`${type}/`)
+    ? mimeType.slice(type.length + 1)
+    : undefined;
+  return subtype ? `${type}.${subtype}` : type;
+};
+
+const resolveImage = (
+  source: MediaSource,
+  mimeType: string | undefined,
+): string | undefined => {
+  if (source.sourceType === "id") return undefined;
+  // `parseDataUrl` only matches the base64 form, so a percent-encoded data URL
+  // would otherwise be wrapped in a second base64 envelope.
+  if (source.sourceType === "url" || /^data:/i.test(source.data))
+    return source.data;
+  return mimeType ? `data:${mimeType};base64,${source.data}` : undefined;
+};
+
+const convertMediaBlock = (
+  part: LangChainMediaBlock,
+): ConvertedContentPart | undefined => {
+  const source = resolveMediaSource(part);
+  if (!source) return undefined;
+
+  const mimeType = resolveMediaMimeType(part);
+  if (part.type === "image") {
+    const image = resolveImage(source, mimeType);
+    if (image) return { type: "image" as const, image };
+  }
+
+  const resolvedMimeType =
+    mimeType ??
+    (part.type === "text-plain" ? "text/plain" : "application/octet-stream");
+  return {
+    type: "file" as const,
+    filename:
+      part.metadata?.filename ??
+      defaultMediaFilename(part.type, resolvedMimeType),
+    data: source.data,
+    mimeType: resolvedMimeType,
+    ...(source.sourceType && { sourceType: source.sourceType }),
+  };
+};
+
+export const normalizeToolCallArgs = (args: unknown): ReadonlyJSONObject => {
+  if (typeof args !== "object" || args === null || Array.isArray(args)) {
+    return {};
+  }
+
+  try {
+    const prototype = Object.getPrototypeOf(args);
+    return prototype === Object.prototype || prototype === null
+      ? (args as ReadonlyJSONObject)
+      : {};
+  } catch {
+    return {};
+  }
+};
+
 export const convertLangChainContentBlock = (
-  part: LangChainContentBlock,
+  part: LangChainContentBlock | LangChainMediaBlock,
 ): ConvertedContentPart | null | undefined => {
   const type = part.type;
   switch (type) {
@@ -74,33 +206,12 @@ export const convertLangChainContentBlock = (
       if (!image) return null;
       return { type: "image" as const, image };
     }
+    case "image":
+    case "video":
+    case "audio":
     case "file":
-      return {
-        type: "file" as const,
-        filename: part.metadata?.filename ?? "file",
-        data:
-          part.source_type === "url"
-            ? part.url
-            : part.source_type === "id"
-              ? part.id
-              : part.data,
-        mimeType: part.mime_type ?? "application/octet-stream",
-        ...((part.source_type === "url" || part.source_type === "id") && {
-          sourceType: part.source_type,
-        }),
-      };
-    case "audio": {
-      const mimeType = part.mime_type ?? "application/octet-stream";
-      const subtype = mimeType.startsWith("audio/")
-        ? mimeType.slice("audio/".length)
-        : undefined;
-      return {
-        type: "file" as const,
-        filename: subtype ? `audio.${subtype}` : "audio",
-        data: part.data,
-        mimeType,
-      };
-    }
+    case "text-plain":
+      return convertMediaBlock(part);
     case "thinking":
       return hasVisibleText(part.thinking)
         ? { type: "reasoning" as const, text: part.thinking }

package/src/getPendingToolCalls.test.ts

@@ -0,0 +1,35 @@
+import { describe, expect, it } from "vitest";
+import type { LangChainBaseMessage } from "./types";
+import { getPendingToolCalls } from "./useStreamRuntime";
+
+const aiWithNullToolCall = {
+  id: "ai-1",
+  _getType: () => "ai",
+  content: "",
+  tool_calls: [null, { id: "call-1", name: "lookup", args: {} }],
+} as unknown as LangChainBaseMessage;
+
+describe("getPendingToolCalls", () => {
+  it("skips a null tool_calls entry and returns the remaining pending call", () => {
+    expect(
+      getPendingToolCalls([
+        { id: "human-1", _getType: () => "human", content: "look it up" },
+        aiWithNullToolCall,
+      ]),
+    ).toEqual([{ id: "call-1", name: "lookup", args: {} }]);
+  });
+
+  it("treats a call answered by a tool message as settled next to a null entry", () => {
+    expect(
+      getPendingToolCalls([
+        aiWithNullToolCall,
+        {
+          id: "tool-1",
+          _getType: () => "tool",
+          content: "done",
+          tool_call_id: "call-1",
+        } as LangChainBaseMessage,
+      ]),
+    ).toEqual([]);
+  });
+});