@assistant-ui/core 0.3.3 → 0.3.4

package/src/utils/image-media-type.ts

@@ -0,0 +1,118 @@
+/**
+ * Leading-byte signatures for the image formats a browser or attachment
+ * adapter realistically produces. `null` matches any byte, which WebP needs
+ * because its four size bytes sit between the two markers, and AVIF and HEIC
+ * need because their leading `ftyp` box size varies by encoder.
+ *
+ * These are file format constants rather than an evolving API, so the table
+ * mirrors the one the AI SDK applies on the far side of several adapters
+ * without taking a dependency to reach it.
+ */
+const IMAGE_SIGNATURES: readonly {
+  readonly mediaType: string;
+  readonly prefix: readonly (number | null)[];
+}[] = [
+  { mediaType: "image/gif", prefix: [0x47, 0x49, 0x46] },
+  { mediaType: "image/png", prefix: [0x89, 0x50, 0x4e, 0x47] },
+  { mediaType: "image/jpeg", prefix: [0xff, 0xd8] },
+  {
+    mediaType: "image/webp",
+    prefix: [
+      0x52,
+      0x49,
+      0x46,
+      0x46,
+      null,
+      null,
+      null,
+      null,
+      0x57,
+      0x45,
+      0x42,
+      0x50,
+    ],
+  },
+  { mediaType: "image/bmp", prefix: [0x42, 0x4d] },
+  { mediaType: "image/tiff", prefix: [0x49, 0x49, 0x2a, 0x00] },
+  { mediaType: "image/tiff", prefix: [0x4d, 0x4d, 0x00, 0x2a] },
+  {
+    mediaType: "image/avif",
+    prefix: [
+      null,
+      null,
+      null,
+      null,
+      0x66,
+      0x74,
+      0x79,
+      0x70,
+      0x61,
+      0x76,
+      0x69,
+      0x66,
+    ],
+  },
+  {
+    mediaType: "image/heic",
+    prefix: [
+      null,
+      null,
+      null,
+      null,
+      0x66,
+      0x74,
+      0x79,
+      0x70,
+      0x68,
+      0x65,
+      0x69,
+      0x63,
+    ],
+  },
+];
+
+/** 16 base64 characters decode to the 12 bytes the longest signature needs. */
+const PREFIX_CHARS = 16;
+
+const decodePrefix = (base64: string): Uint8Array | undefined => {
+  const prefix = base64.slice(0, PREFIX_CHARS);
+  const aligned = prefix.slice(0, prefix.length - (prefix.length % 4));
+  if (aligned.length === 0) return undefined;
+
+  const nodeBuffer = (
+    globalThis as {
+      Buffer?: {
+        from(data: string, encoding: string): Uint8Array;
+      };
+    }
+  ).Buffer;
+  if (nodeBuffer) return new Uint8Array(nodeBuffer.from(aligned, "base64"));
+
+  try {
+    const binary = atob(aligned);
+    const bytes = new Uint8Array(binary.length);
+    for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
+    return bytes;
+  } catch {
+    return undefined;
+  }
+};
+
+/**
+ * The image media type a base64 payload's leading bytes identify, or
+ * `undefined` when they match no known signature or the input is not base64.
+ *
+ * An `ImageMessagePart` carries no media type, so an adapter that must declare
+ * one on the wire reads it from the bytes rather than assuming a format. Never
+ * throws: the input is whatever a caller put on the part.
+ */
+export function detectImageMediaType(base64: string): string | undefined {
+  const bytes = decodePrefix(base64);
+  if (!bytes) return undefined;
+
+  return IMAGE_SIGNATURES.find(
+    ({ prefix }) =>
+      bytes.length >= prefix.length &&
+      prefix.every((byte, index) => byte === null || bytes[index] === byte),
+  )?.mediaType;
+}

package/src/utils/wire-media.test.ts

@@ -0,0 +1,96 @@
+import { describe, expect, it } from "vitest";
+import {
+  resolveFileMediaType,
+  resolveImageMediaType,
+  toMediaWireUrl,
+} from "./wire-media";
+
+const JPEG = "/9j/4AAQSkZJRg==";
+
+describe("resolveImageMediaType", () => {
+  it("prefers an image content type over everything else", () => {
+    expect(
+      resolveImageMediaType(`data:image/png;base64,${JPEG}`, "image/webp"),
+    ).toBe("image/webp");
+  });
+
+  it("ignores a content type that is not an image", () => {
+    expect(resolveImageMediaType(JPEG, "application/octet-stream")).toBe(
+      "image/jpeg",
+    );
+  });
+
+  it("takes the data url declaration when it is an image type", () => {
+    expect(
+      resolveImageMediaType("data:image/svg+xml,%3Csvg%3E%3C/svg%3E"),
+    ).toBe("image/svg+xml");
+  });
+
+  it("sniffs through a declaration that is not an image type", () => {
+    expect(
+      resolveImageMediaType(`data:application/octet-stream;base64,${JPEG}`),
+    ).toBe("image/jpeg");
+  });
+
+  it("sniffs a bare base64 payload", () => {
+    expect(resolveImageMediaType(JPEG)).toBe("image/jpeg");
+  });
+
+  it("floors to image/png when there are no bytes to read", () => {
+    expect(resolveImageMediaType("https://cdn.example.com/photo")).toBe(
+      "image/png",
+    );
+  });
+
+  it("floors to image/png when the bytes match no signature", () => {
+    expect(resolveImageMediaType("QUJD")).toBe("image/png");
+  });
+});
+
+describe("resolveFileMediaType", () => {
+  it("prefers the declared type", () => {
+    expect(
+      resolveFileMediaType(
+        "data:application/octet-stream;base64,QUJD",
+        "application/pdf",
+      ),
+    ).toBe("application/pdf");
+  });
+
+  it("falls back to the data url declaration when none is declared", () => {
+    expect(resolveFileMediaType("data:text/plain,hello", "")).toBe(
+      "text/plain",
+    );
+  });
+
+  it("floors to application/octet-stream", () => {
+    expect(resolveFileMediaType("QUJD", "")).toBe("application/octet-stream");
+  });
+});
+
+describe("toMediaWireUrl", () => {
+  it("forwards an envelope that agrees byte for byte", () => {
+    const payload = "data:image/jpeg;base64,QUJD";
+    expect(toMediaWireUrl(payload, "image/jpeg")).toBe(payload);
+  });
+
+  it("rebuilds an envelope that disagrees", () => {
+    expect(
+      toMediaWireUrl("data:application/octet-stream;base64,QUJD", "image/jpeg"),
+    ).toBe("data:image/jpeg;base64,QUJD");
+  });
+
+  it("wraps a bare base64 payload", () => {
+    expect(toMediaWireUrl("QUJD", "application/pdf")).toBe(
+      "data:application/pdf;base64,QUJD",
+    );
+  });
+
+  it.each([
+    "https://cdn.example.com/a.pdf",
+    "blob:https://app.example/1-2-3",
+    "data:text/plain,hello",
+  ])("forwards %s untouched", (payload) => {
+    expect(toMediaWireUrl(payload, "application/pdf")).toBe(payload);
+  });
+});

package/src/utils/wire-media.ts

@@ -0,0 +1,70 @@
+import { dataUrlMediaType, isParsableUrl, parseDataUrl } from "./data-url";
+import { detectImageMediaType } from "./image-media-type";
+
+/**
+ * The media type to declare for an `ImageMessagePart`, which carries none of
+ * its own: an attachment's content type, then the payload's data URL
+ * declaration, then its leading bytes, then `image/png`.
+ *
+ * The floor cannot be a wildcard. `image/*` is rejected outright by the AI SDK
+ * for a url source and whenever inline bytes fail to sniff, so it fails exactly
+ * where a fallback is needed.
+ */
+export function resolveImageMediaType(
+  image: string,
+  contentType?: string | undefined,
+): string {
+  if (contentType?.startsWith("image/")) return contentType;
+
+  const declared = dataUrlMediaType(image);
+  if (declared?.startsWith("image/")) return declared;
+
+  // Read through a data URL envelope too, so a generic one such as
+  // `application/octet-stream` does not mask the format. A url of any other
+  // scheme has no local bytes to read.
+  const parsed = parseDataUrl(image);
+  const payload = parsed?.data ?? (isParsableUrl(image) ? undefined : image);
+  if (payload !== undefined) {
+    const sniffed = detectImageMediaType(payload);
+    if (sniffed) return sniffed;
+  }
+
+  return "image/png";
+}
+
+/**
+ * The media type to declare for a `FileMessagePart`: its own `mimeType`, then
+ * the payload's data URL declaration, then `application/octet-stream`.
+ *
+ * `mimeType` is a plain string, and an adapter reading `file.type` on a file
+ * the OS cannot type yields `""`, so the declared value is not always present.
+ */
+export function resolveFileMediaType(
+  data: string,
+  mimeType?: string | undefined,
+): string {
+  return mimeType || dataUrlMediaType(data) || "application/octet-stream";
+}
+
+/**
+ * A payload placed into a wire field that is contractually a url.
+ *
+ * Two hazards drive this. A consumer may hand the value to an unguarded
+ * `new URL()`, so a payload that is not a url has to be wrapped; and a data
+ * URL's own media type wins over a separately declared one downstream, so a
+ * base64 envelope that disagrees with `mediaType` is rebuilt around the same
+ * bytes. Everything else passes through byte for byte: an envelope that agrees,
+ * a url of any other scheme, and a percent-encoded data URL, whose declaration
+ * cannot be corrected without transcoding the payload and is authoritative for
+ * the bytes it carries anyway.
+ */
+export function toMediaWireUrl(payload: string, mediaType: string): string {
+  const parsed = parseDataUrl(payload);
+  if (parsed) {
+    return parsed.mimeType === mediaType
+      ? payload
+      : `data:${mediaType};base64,${parsed.data}`;
+  }
+  if (isParsableUrl(payload)) return payload;
+  return `data:${mediaType};base64,${payload}`;
+}