@runtypelabs/persona 3.20.0 → 3.21.1

package/src/client.ts

@@ -19,7 +19,8 @@ import {
   ClientChatRequest,
   ClientFeedbackRequest,
   ClientFeedbackType,
-  PersonaArtifactKind
+  PersonaArtifactKind,
+  ContentPart
 } from "./types";
 import {
   extractTextFromJson,
@@ -44,6 +45,36 @@ type SSEHandler = (event: AgentWidgetEvent) => void;
 const DEFAULT_ENDPOINT = "https://api.runtype.com/v1/dispatch";
 const DEFAULT_CLIENT_API_BASE = "https://api.runtype.com";
 
+/**
+ * Derive a download filename for `agent_media` parts that are delivered
+ * without one. Maps a few well-known MIME types to friendly extensions and
+ * falls back to `attachment.<subtype>` (or just `attachment` for opaque
+ * types like `application/octet-stream`).
+ */
+function filenameFromMediaType(mediaType: string): string {
+  // MIME types are case-insensitive (RFC 7231); compare against a lowercased
+  // copy so callers that pass mixed casing still hit the friendly extensions.
+  const lower = mediaType.toLowerCase();
+  const knownExtensions: Record<string, string> = {
+    "application/pdf": "pdf",
+    "application/json": "json",
+    "application/zip": "zip",
+    "text/plain": "txt",
+    "text/csv": "csv",
+    "text/markdown": "md"
+  };
+  const ext = knownExtensions[lower];
+  if (ext) return `attachment.${ext}`;
+  const slash = lower.indexOf("/");
+  if (slash > 0) {
+    const subtype = lower.slice(slash + 1).split(";")[0]?.trim() ?? "";
+    if (subtype && subtype !== "octet-stream" && /^[a-z0-9.+-]+$/i.test(subtype)) {
+      return `attachment.${subtype}`;
+    }
+  }
+  return "attachment";
+}
+
 /**
  * Check if a message has valid (non-empty) content for sending to the API.
  * Filters out messages with empty content that would cause validation errors.
@@ -2563,6 +2594,124 @@ export class AgentWidgetClient {
               }
             }
           }
+        } else if (payloadType === "agent_media") {
+          // A tool produced media (image / audio / video / file). Render it
+          // as a synthetic assistant message inserted at the point the tool
+          // completed — between the tool bubble and the next text turn.
+          //
+          // Wire format is the AI SDK–aligned `MediaContentPart` from
+          // @runtypelabs/shared:
+          //   { type: 'media', data, mediaType }                // AI SDK v6: base64
+          //   { type: 'image-url', url, mediaType? }            // AI SDK v3/v4
+          //   { type: 'file-url', url, mediaType }              // AI SDK v3/v4
+          const rawMedia = Array.isArray(payload.media) ? payload.media : [];
+          const mediaContentParts: ContentPart[] = [];
+          for (const part of rawMedia) {
+            if (!part || typeof part !== "object") continue;
+            const rec = part as Record<string, unknown>;
+            const partType = typeof rec.type === "string" ? rec.type : undefined;
+
+            // Resolve `(src, mediaType)` for the part.
+            // RFC 7231 says MIME types are case-insensitive, so we canonicalize
+            // to lowercase once here. That makes the `startsWith("image/")` /
+            // `"audio/"` / `"video/"` bucket checks robust to upstream tools
+            // that emit non-canonical casing like `Image/PNG`.
+            const rawMediaType =
+              typeof rec.mediaType === "string" ? rec.mediaType.toLowerCase() : "";
+            let src: string | null = null;
+            let mediaType = "";
+            if (partType === "media") {
+              const data = typeof rec.data === "string" ? rec.data : undefined;
+              if (!data) continue;
+              // Empty/missing mediaType yields `data:;base64,...` which RFC 2397
+              // resolves to `text/plain` — stamp a default so the data URI is
+              // well-formed and the part lands in the file bucket.
+              mediaType = rawMediaType.length > 0 ? rawMediaType : "application/octet-stream";
+              src = `data:${mediaType};base64,${data}`;
+            } else if (partType === "image-url") {
+              const url = typeof rec.url === "string" ? rec.url : undefined;
+              if (!url) continue;
+              mediaType = rawMediaType;
+              src = url;
+            } else if (partType === "file-url") {
+              const url = typeof rec.url === "string" ? rec.url : undefined;
+              if (!url) continue;
+              mediaType = rawMediaType;
+              src = url;
+            } else {
+              continue;
+            }
+            if (!src) continue;
+
+            // Pick the right rendering bucket based on mediaType.
+            if (partType === "image-url" || mediaType.startsWith("image/")) {
+              mediaContentParts.push({
+                type: "image",
+                image: src,
+                ...(mediaType ? { mimeType: mediaType } : {}),
+              });
+            } else if (mediaType.startsWith("audio/")) {
+              mediaContentParts.push({
+                type: "audio",
+                audio: src,
+                mimeType: mediaType,
+              });
+            } else if (mediaType.startsWith("video/")) {
+              mediaContentParts.push({
+                type: "video",
+                video: src,
+                mimeType: mediaType,
+              });
+            } else {
+              const resolvedMediaType = mediaType || "application/octet-stream";
+              mediaContentParts.push({
+                type: "file",
+                data: src,
+                mimeType: resolvedMediaType,
+                filename: filenameFromMediaType(resolvedMediaType),
+              });
+            }
+          }
+
+          if (mediaContentParts.length > 0) {
+            // Uniquify per emission. A tool may emit multiple `agent_media`
+            // events for the same `toolCallId` (e.g. streamed/batched media);
+            // sharing an id would let `emitMessage` merge them by id and
+            // overwrite the prior `contentParts`.
+            const seq = nextSequence();
+            const toolCallIdRaw = payload.toolCallId;
+            const mediaIdSuffix =
+              typeof toolCallIdRaw === "string" && toolCallIdRaw.length > 0
+                ? `${toolCallIdRaw}-${seq}`
+                : String(seq);
+            const mediaMessage: AgentWidgetMessage = {
+              id: `agent-media-${mediaIdSuffix}`,
+              role: "assistant",
+              content: "",
+              contentParts: mediaContentParts,
+              createdAt: new Date().toISOString(),
+              streaming: false,
+              sequence: seq,
+              agentMetadata: {
+                executionId: payload.executionId,
+                iteration: payload.iteration,
+              },
+            };
+            emitMessage(mediaMessage);
+
+            // Seal any in-flight assistant text bubble before splitting the
+            // stream. Without this, an orphan bubble retains `streaming: true`
+            // forever — `agent_complete` only finalizes the latest
+            // `assistantMessage`, so the typing/caret indicator would stay on
+            // the prior bubble even though no more deltas will arrive.
+            const prevAssistant = assistantMessage as AgentWidgetMessage | null;
+            if (prevAssistant) {
+              prevAssistant.streaming = false;
+              emitMessage(prevAssistant);
+            }
+            assistantMessage = null;
+            assistantMessageRef.current = null;
+          }
         } else if (payloadType === "agent_iteration_complete") {
           // Iteration complete - no special handling needed
           // In 'separate' mode, message finalization happens at next iteration_start

package/src/components/message-bubble.test.ts

@@ -3,6 +3,7 @@ import { describe, it, expect } from "vitest";
 import {
   createStandardBubble,
   isSafeImageSrc,
+  isSafeMediaSrc,
   resolveStopReasonNoticeText,
   getDefaultStopReasonNoticeCopy,
 } from "./message-bubble";
@@ -274,3 +275,194 @@ describe("createStandardBubble — stopReason notice", () => {
     expect(bubble.querySelector(".persona-message-stop-reason")).toBeNull();
   });
 });
+
+describe("isSafeMediaSrc", () => {
+  it("allows https URLs", () => {
+    expect(isSafeMediaSrc("https://example.com/audio.mp3")).toBe(true);
+  });
+
+  it("allows http URLs", () => {
+    expect(isSafeMediaSrc("http://example.com/audio.mp3")).toBe(true);
+  });
+
+  it("allows blob URLs", () => {
+    expect(isSafeMediaSrc("blob:http://example.com/abc-123")).toBe(true);
+  });
+
+  it("allows audio data URIs", () => {
+    expect(isSafeMediaSrc("data:audio/mpeg;base64,AAAA")).toBe(true);
+  });
+
+  it("allows video data URIs", () => {
+    expect(isSafeMediaSrc("data:video/mp4;base64,AAAA")).toBe(true);
+  });
+
+  it("allows binary file data URIs", () => {
+    expect(isSafeMediaSrc("data:application/pdf;base64,AAAA")).toBe(true);
+  });
+
+  it("blocks javascript: URIs", () => {
+    expect(isSafeMediaSrc("javascript:alert(1)")).toBe(false);
+  });
+
+  it("blocks data:text/html URIs", () => {
+    expect(isSafeMediaSrc("data:text/html,<script>alert(1)</script>")).toBe(false);
+    expect(isSafeMediaSrc("data:text/html;base64,PHNjcmlwdD5hbGVydCgxKTwvc2NyaXB0Pg==")).toBe(false);
+  });
+
+  it("blocks other executable data: types", () => {
+    expect(isSafeMediaSrc("data:text/javascript,alert(1)")).toBe(false);
+    expect(isSafeMediaSrc("data:text/xml,<svg onload=alert(1)/>")).toBe(false);
+    expect(isSafeMediaSrc("data:application/xhtml+xml,<html>")).toBe(false);
+  });
+
+  it("blocks data:image/svg+xml URIs (XSS via right-click open in new tab)", () => {
+    expect(isSafeMediaSrc("data:image/svg+xml,<svg onload=alert(1)>")).toBe(false);
+    expect(isSafeMediaSrc("data:image/svg+xml;base64,PHN2Zz4=")).toBe(false);
+    expect(isSafeMediaSrc("data:image/SVG+XML;base64,PHN2Zz4=")).toBe(false);
+  });
+
+  it("allows inert data:text/* payloads (plain, csv, markdown)", () => {
+    expect(isSafeMediaSrc("data:text/plain;base64,SGVsbG8=")).toBe(true);
+    expect(isSafeMediaSrc("data:text/csv;base64,YSxiCjEsMg==")).toBe(true);
+    expect(isSafeMediaSrc("data:text/markdown;base64,IyBIaQ==")).toBe(true);
+  });
+
+  it("allows relative paths", () => {
+    expect(isSafeMediaSrc("relative/file.mp3")).toBe(true);
+  });
+});
+
+describe("createStandardBubble — audio/video/file content parts", () => {
+  it("renders an <audio> element with controls for an audio content part", () => {
+    const bubble = createStandardBubble(
+      makeMessage({
+        contentParts: [
+          {
+            type: "audio",
+            audio: "data:audio/mpeg;base64,AAAA",
+            mimeType: "audio/mpeg",
+          },
+        ],
+      }),
+      ({ text }) => text
+    );
+
+    const container = bubble.querySelector('[data-message-attachments="audio"]');
+    expect(container).not.toBeNull();
+    const audio = container?.querySelector("audio") as HTMLAudioElement | null;
+    expect(audio).not.toBeNull();
+    expect(audio?.controls).toBe(true);
+    expect(audio?.getAttribute("src")).toBe("data:audio/mpeg;base64,AAAA");
+  });
+
+  it("renders an <audio> element for a URL-based audio part", () => {
+    const bubble = createStandardBubble(
+      makeMessage({
+        contentParts: [
+          {
+            type: "audio",
+            audio: "https://example.com/clip.mp3",
+            mimeType: "audio/mpeg",
+          },
+        ],
+      }),
+      ({ text }) => text
+    );
+
+    const audio = bubble.querySelector(
+      '[data-message-attachments="audio"] audio'
+    ) as HTMLAudioElement | null;
+    expect(audio?.getAttribute("src")).toBe("https://example.com/clip.mp3");
+  });
+
+  it("skips audio parts with unsafe schemes", () => {
+    const bubble = createStandardBubble(
+      makeMessage({
+        contentParts: [
+          {
+            type: "audio",
+            audio: "javascript:alert(1)",
+            mimeType: "audio/mpeg",
+          },
+        ],
+      }),
+      ({ text }) => text
+    );
+
+    expect(
+      bubble.querySelector('[data-message-attachments="audio"]')
+    ).toBeNull();
+  });
+
+  it("renders a <video> element with controls for a video content part", () => {
+    const bubble = createStandardBubble(
+      makeMessage({
+        contentParts: [
+          {
+            type: "video",
+            video: "https://example.com/clip.mp4",
+            mimeType: "video/mp4",
+          },
+        ],
+      }),
+      ({ text }) => text
+    );
+
+    const video = bubble.querySelector(
+      '[data-message-attachments="video"] video'
+    ) as HTMLVideoElement | null;
+    expect(video).not.toBeNull();
+    expect(video?.controls).toBe(true);
+    expect(video?.getAttribute("src")).toBe("https://example.com/clip.mp4");
+  });
+
+  it("renders a download link for a file content part", () => {
+    const bubble = createStandardBubble(
+      makeMessage({
+        contentParts: [
+          {
+            type: "file",
+            data: "data:application/pdf;base64,AAAA",
+            mimeType: "application/pdf",
+            filename: "report.pdf",
+          },
+        ],
+      }),
+      ({ text }) => text
+    );
+
+    const link = bubble.querySelector(
+      '[data-message-attachments="files"] a'
+    ) as HTMLAnchorElement | null;
+    expect(link).not.toBeNull();
+    expect(link?.getAttribute("href")).toBe("data:application/pdf;base64,AAAA");
+    expect(link?.getAttribute("download")).toBe("report.pdf");
+    expect(link?.textContent).toBe("report.pdf");
+    // Cross-origin URLs ignore `download`, so we open in a new tab to avoid
+    // navigating the chat page away from the conversation.
+    expect(link?.getAttribute("target")).toBe("_blank");
+    expect(link?.getAttribute("rel")).toBe("noopener noreferrer");
+  });
+
+  it("renders a download link for a URL-hosted file", () => {
+    const bubble = createStandardBubble(
+      makeMessage({
+        contentParts: [
+          {
+            type: "file",
+            data: "https://example.com/report.pdf",
+            mimeType: "application/pdf",
+            filename: "report.pdf",
+          },
+        ],
+      }),
+      ({ text }) => text
+    );
+
+    const link = bubble.querySelector(
+      '[data-message-attachments="files"] a'
+    ) as HTMLAnchorElement | null;
+    expect(link?.getAttribute("href")).toBe("https://example.com/report.pdf");
+  });
+});

package/src/components/message-bubble.ts

@@ -8,6 +8,9 @@ import {
   AgentWidgetMessageFeedback,
   LoadingIndicatorRenderContext,
   ImageContentPart,
+  AudioContentPart,
+  VideoContentPart,
+  FileContentPart,
   StopReasonKind
 } from "../types";
 import { createIconButton } from "../utils/buttons";
@@ -98,6 +101,31 @@ export const isSafeImageSrc = (src: string): boolean => {
   return false;
 };
 
+/**
+ * Validate that a media src URL (audio/video/file) uses a safe scheme.
+ * Allows http(s), blob:, and inert data: URIs. Blocks `javascript:` and the
+ * executable data: types whose payloads a browser would render or run when a
+ * user right-clicks the link → "Open in new tab" (bypassing `download`):
+ * `data:text/html`, `data:text/javascript`, `data:text/xml`,
+ * `data:application/xhtml`, and `data:image/svg+xml`. Inert text payloads
+ * (`text/plain`, `text/csv`, `text/markdown`) remain allowed so attachments
+ * with those mime types render as download links instead of vanishing.
+ */
+export const isSafeMediaSrc = (src: string): boolean => {
+  const lower = src.toLowerCase();
+  if (lower.startsWith("javascript:")) return false;
+  if (lower.startsWith("data:text/html")) return false;
+  if (lower.startsWith("data:text/javascript")) return false;
+  if (lower.startsWith("data:text/xml")) return false;
+  if (lower.startsWith("data:application/xhtml")) return false;
+  if (lower.startsWith("data:image/svg+xml")) return false;
+  if (/^(?:https?|blob):/i.test(src)) return true;
+  if (lower.startsWith("data:")) return true;
+  // Relative URLs are safe
+  if (!src.includes(":")) return true;
+  return false;
+};
+
 export type LoadingIndicatorRenderer = (context: LoadingIndicatorRenderContext) => HTMLElement | null;
 
 export type MessageTransform = (context: {
@@ -128,6 +156,36 @@ const getMessageImageParts = (message: AgentWidgetMessage): ImageContentPart[] =
   );
 };
 
+const getMessageAudioParts = (message: AgentWidgetMessage): AudioContentPart[] => {
+  if (!message.contentParts || message.contentParts.length === 0) return [];
+  return message.contentParts.filter(
+    (part): part is AudioContentPart =>
+      part.type === "audio" &&
+      typeof part.audio === "string" &&
+      part.audio.trim().length > 0
+  );
+};
+
+const getMessageVideoParts = (message: AgentWidgetMessage): VideoContentPart[] => {
+  if (!message.contentParts || message.contentParts.length === 0) return [];
+  return message.contentParts.filter(
+    (part): part is VideoContentPart =>
+      part.type === "video" &&
+      typeof part.video === "string" &&
+      part.video.trim().length > 0
+  );
+};
+
+const getMessageFileParts = (message: AgentWidgetMessage): FileContentPart[] => {
+  if (!message.contentParts || message.contentParts.length === 0) return [];
+  return message.contentParts.filter(
+    (part): part is FileContentPart =>
+      part.type === "file" &&
+      typeof part.data === "string" &&
+      part.data.trim().length > 0
+  );
+};
+
 const createMessageImagePreviews = (
   imageParts: ImageContentPart[],
   hasVisibleText: boolean,
@@ -209,6 +267,124 @@ const createMessageImagePreviews = (
   }
 };
 
+const createMessageAudioPreviews = (
+  audioParts: AudioContentPart[]
+): HTMLElement | null => {
+  if (audioParts.length === 0) return null;
+  try {
+    const container = createElement(
+      "div",
+      "persona-flex persona-flex-col persona-gap-2"
+    );
+    container.setAttribute("data-message-attachments", "audio");
+    let visible = 0;
+    audioParts.forEach((part) => {
+      if (!isSafeMediaSrc(part.audio)) return;
+      const audioElement = createElement("audio") as HTMLAudioElement;
+      audioElement.controls = true;
+      audioElement.preload = "metadata";
+      audioElement.src = part.audio;
+      audioElement.style.display = "block";
+      audioElement.style.width = "100%";
+      audioElement.style.maxWidth = `${MESSAGE_IMAGE_PREVIEW_MAX_WIDTH_PX}px`;
+      container.appendChild(audioElement);
+      visible += 1;
+    });
+    if (visible === 0) {
+      container.remove();
+      return null;
+    }
+    return container;
+  } catch {
+    return null;
+  }
+};
+
+const createMessageVideoPreviews = (
+  videoParts: VideoContentPart[]
+): HTMLElement | null => {
+  if (videoParts.length === 0) return null;
+  try {
+    const container = createElement(
+      "div",
+      "persona-flex persona-flex-col persona-gap-2"
+    );
+    container.setAttribute("data-message-attachments", "video");
+    let visible = 0;
+    videoParts.forEach((part) => {
+      if (!isSafeMediaSrc(part.video)) return;
+      const videoElement = createElement("video") as HTMLVideoElement;
+      videoElement.controls = true;
+      videoElement.preload = "metadata";
+      videoElement.src = part.video;
+      videoElement.style.display = "block";
+      videoElement.style.width = "100%";
+      videoElement.style.maxWidth = `${MESSAGE_IMAGE_PREVIEW_MAX_WIDTH_PX}px`;
+      videoElement.style.maxHeight = `${MESSAGE_IMAGE_PREVIEW_MAX_HEIGHT_PX}px`;
+      videoElement.style.borderRadius = "10px";
+      videoElement.style.backgroundColor =
+        "var(--persona-attachment-image-bg, var(--persona-container, #f3f4f6))";
+      container.appendChild(videoElement);
+      visible += 1;
+    });
+    if (visible === 0) {
+      container.remove();
+      return null;
+    }
+    return container;
+  } catch {
+    return null;
+  }
+};
+
+const createMessageFilePreviews = (
+  fileParts: FileContentPart[]
+): HTMLElement | null => {
+  if (fileParts.length === 0) return null;
+  try {
+    const container = createElement(
+      "div",
+      "persona-flex persona-flex-col persona-gap-2"
+    );
+    container.setAttribute("data-message-attachments", "files");
+    let visible = 0;
+    fileParts.forEach((part) => {
+      if (!isSafeMediaSrc(part.data)) return;
+      const link = createElement("a") as HTMLAnchorElement;
+      link.href = part.data;
+      link.download = part.filename;
+      // Cross-origin URLs ignore the `download` attribute, so without
+      // `target=_blank` the link would navigate the chat page to the file.
+      // Pair with `rel="noopener noreferrer"` to prevent reverse tabnabbing.
+      link.target = "_blank";
+      link.rel = "noopener noreferrer";
+      link.textContent = part.filename;
+      link.className = "persona-message-file-attachment";
+      link.style.display = "inline-flex";
+      link.style.alignItems = "center";
+      link.style.gap = "6px";
+      link.style.padding = "6px 10px";
+      link.style.borderRadius = "8px";
+      link.style.fontSize = "0.875rem";
+      link.style.textDecoration = "underline";
+      link.style.backgroundColor =
+        "var(--persona-attachment-file-bg, var(--persona-container, #f3f4f6))";
+      link.style.border =
+        "1px solid var(--persona-attachment-file-border, var(--persona-border, #e5e7eb))";
+      link.style.color = "inherit";
+      container.appendChild(link);
+      visible += 1;
+    });
+    if (visible === 0) {
+      container.remove();
+      return null;
+    }
+    return container;
+  } catch {
+    return null;
+  }
+};
+
 // Create typing indicator element
 export const createTypingIndicator = (): HTMLElement => {
   const container = document.createElement("div");
@@ -710,6 +886,30 @@ export const createStandardBubble = (
     }
   }
 
+  const audioParts = getMessageAudioParts(message);
+  if (audioParts.length > 0) {
+    const audioPreviews = createMessageAudioPreviews(audioParts);
+    if (audioPreviews) {
+      bubble.appendChild(audioPreviews);
+    }
+  }
+
+  const videoParts = getMessageVideoParts(message);
+  if (videoParts.length > 0) {
+    const videoPreviews = createMessageVideoPreviews(videoParts);
+    if (videoPreviews) {
+      bubble.appendChild(videoPreviews);
+    }
+  }
+
+  const fileParts = getMessageFileParts(message);
+  if (fileParts.length > 0) {
+    const filePreviews = createMessageFilePreviews(fileParts);
+    if (filePreviews) {
+      bubble.appendChild(filePreviews);
+    }
+  }
+
   bubble.appendChild(contentDiv);
 
   // Add timestamp below if configured