@ai-sdk/devtools 1.0.7 → 1.0.9

package/src/viewer/client/components/output-components.tsx

@@ -8,16 +8,27 @@ import type {
   TextContentPart,
   ReasoningContentPart,
 } from '../types';
-import { safeParseJson, formatToolParams } from '../utils';
-import { JsonBlock, ReasoningBlock, TextBlock } from './shared-components';
+import {
+  safeParseJson,
+  formatToolParams,
+  getOutputToolResults,
+} from '../utils';
+import {
+  MediaAwareValue,
+  ReasoningBlock,
+  TextBlock,
+} from './shared-components';
+import { MediaPreviewList } from './media-components';
 
 export function OutputDisplay({
   output,
-  toolResults = [],
+  fallbackToolResults = [],
 }: {
   output: ParsedOutput;
-  toolResults?: ContentPart[];
+  fallbackToolResults?: ContentPart[];
 }) {
+  const toolResults = getOutputToolResults(output, fallbackToolResults);
+
   const getToolResult = (toolCallId: string) => {
     return toolResults.find(
       (r): r is ToolResultContentPart =>
@@ -62,6 +73,15 @@ export function OutputDisplay({
         <TextBlock content={textContent} defaultExpanded={!!isTextOnly} />
       )}
 
+      <MediaPreviewList
+        data={output?.content?.filter(
+          part =>
+            part.type === 'file' ||
+            part.type === 'reasoning-file' ||
+            part.type === 'image',
+        )}
+      />
+
       {toolCalls.map((call, i) => {
         const result = call.toolCallId
           ? getToolResult(call.toolCallId)
@@ -127,7 +147,7 @@ function ToolCallCard({
             <div className="text-[10px] font-medium uppercase tracking-wider text-muted-foreground mb-2">
               Input
             </div>
-            <JsonBlock data={parsedArgs} />
+            <MediaAwareValue data={parsedArgs} />
           </div>
 
           {parsedResult != null && (
@@ -135,7 +155,7 @@ function ToolCallCard({
               <div className="text-[10px] font-medium uppercase tracking-wider text-success mb-2">
                 Output
               </div>
-              <JsonBlock data={parsedResult} />
+              <MediaAwareValue data={parsedResult} />
             </div>
           )}
         </>

package/src/viewer/client/components/shared-components.tsx

@@ -29,21 +29,33 @@ import {
   getInputTokenBreakdown,
   getOutputTokenBreakdown,
 } from '../utils';
+import { MediaPreviewList } from './media-components';
 
 export function JsonBlock({
   data,
   compact = false,
   size = 'sm',
+  maxStringLength,
 }: {
   data: unknown;
   compact?: boolean;
   size?: 'sm' | 'base' | 'lg';
+  maxStringLength?: number;
 }) {
   const [copied, setCopied] = useState(false);
 
-  const jsonString = JSON.stringify(data, null, 2);
+  const replacer =
+    maxStringLength == null
+      ? undefined
+      : (_key: string, value: unknown) =>
+          typeof value === 'string' && value.length > maxStringLength
+            ? `${value.slice(0, maxStringLength)}… [${value.length - maxStringLength} characters omitted from the viewer]`
+            : value;
+  const jsonString = JSON.stringify(data, replacer, 2);
   const displayString =
-    compact && jsonString.length > 200 ? JSON.stringify(data) : jsonString;
+    compact && jsonString.length > 200
+      ? JSON.stringify(data, replacer)
+      : jsonString;
 
   const sizeClasses = {
     sm: 'text-xs',
@@ -81,6 +93,15 @@ export function JsonBlock({
   );
 }
 
+export function MediaAwareValue({ data }: { data: unknown }) {
+  return (
+    <div className="space-y-3">
+      <MediaPreviewList data={data} />
+      <JsonBlock data={data} maxStringLength={16 * 1024} />
+    </div>
+  );
+}
+
 export function RawDataSection({
   rawRequest,
   rawResponse,
@@ -516,7 +537,7 @@ export function CollapsibleToolCall({
       </button>
       {expanded && (
         <div className="p-3 border-t bg-card/50 border-tool/30">
-          <JsonBlock data={parsedData} />
+          <MediaAwareValue data={parsedData} />
         </div>
       )}
     </div>
@@ -559,7 +580,7 @@ export function CollapsibleToolResult({
       </button>
       {expanded && (
         <div className="p-3 border-t bg-card/50 border-success/30">
-          <JsonBlock data={data} />
+          <MediaAwareValue data={data} />
         </div>
       )}
     </div>

package/src/viewer/client/components/step-card.tsx

@@ -256,7 +256,7 @@ export function StepDetailContent({
   const nextInput = nextStep
     ? (parseJson(nextStep.input) as ParsedInput | null)
     : null;
-  const toolResults: ContentPart[] =
+  const fallbackToolResults: ContentPart[] =
     nextInput?.prompt
       ?.filter((msg: PromptMessage) => msg.role === 'tool')
       ?.flatMap((msg: PromptMessage) =>
@@ -290,7 +290,10 @@ export function StepDetailContent({
               {step.error}
             </div>
           ) : output ? (
-            <OutputDisplay output={output} toolResults={toolResults} />
+            <OutputDisplay
+              output={output}
+              fallbackToolResults={fallbackToolResults}
+            />
           ) : isActiveStep ? (
             <div className="flex items-center gap-2 text-sm text-info">
               <Loader2 className="size-4 animate-spin" />

package/src/viewer/client/components/trace-timeline.tsx

@@ -14,6 +14,7 @@ import type {
   SpanKind,
   TraceSpan,
   ParseJson,
+  ParsedInput,
   ParsedOutput,
   ContentPart,
   ToolCallContentPart,
@@ -21,11 +22,12 @@ import type {
 } from '../types';
 import {
   buildTraceSpans,
+  getOutputToolResults,
   safeParseJson,
   SPAN_COLORS,
   SPAN_COLORS_MUTED,
 } from '../utils';
-import { JsonBlock } from './shared-components';
+import { MediaAwareValue } from './shared-components';
 import { StepDetailContent } from './step-card';
 
 export function TraceTimeline({
@@ -424,7 +426,17 @@ function SpanDetailPanel({
         'toolCallId' in p &&
         p.toolCallId === span.toolCallId,
     );
-    const toolResult = contentParts.find(
+    const nextStep = steps[stepIndex + 1];
+    const nextInput = nextStep
+      ? (parseJson(nextStep.input) as ParsedInput | null)
+      : null;
+    const fallbackToolResults =
+      nextInput?.prompt
+        ?.filter(message => message.role === 'tool')
+        .flatMap(message =>
+          Array.isArray(message.content) ? message.content : [],
+        ) ?? [];
+    const toolResult = getOutputToolResults(output, fallbackToolResults).find(
       (p): p is ToolResultContentPart =>
         p.type === 'tool-result' &&
         'toolCallId' in p &&
@@ -450,7 +462,7 @@ function SpanDetailPanel({
             <h4 className="text-[11px] font-semibold uppercase tracking-wider text-muted-foreground mb-2">
               Input
             </h4>
-            <JsonBlock data={parsedArgs} />
+            <MediaAwareValue data={parsedArgs} />
           </div>
         )}
         {parsedResult != null && (
@@ -458,7 +470,7 @@ function SpanDetailPanel({
             <h4 className="text-[11px] font-semibold uppercase tracking-wider text-success mb-2">
               Output
             </h4>
-            <JsonBlock data={parsedResult} />
+            <MediaAwareValue data={parsedResult} />
           </div>
         )}
         {!parsedArgs && !parsedResult && (

package/src/viewer/client/media.ts

@@ -0,0 +1,440 @@
+export type MediaKind = 'image' | 'audio' | 'video' | 'file';
+
+export interface MediaPreviewData {
+  filename?: string;
+  kind: MediaKind;
+  mediaType: string;
+  source?: string;
+  sourceType?: 'inline' | 'remote';
+  unavailableReason?: string;
+}
+
+export interface MediaPreviewLimits {
+  maxCount: number;
+  maxDepth: number;
+  maxInlineBytes: number;
+  maxNodes: number;
+}
+
+export const DEFAULT_MEDIA_PREVIEW_LIMITS: MediaPreviewLimits = {
+  maxCount: 8,
+  maxDepth: 12,
+  maxInlineBytes: 5 * 1024 * 1024,
+  maxNodes: 1000,
+};
+
+const safeInlineMediaTypes = new Set([
+  'audio/m4a',
+  'audio/mp3',
+  'audio/mp4',
+  'audio/mpeg',
+  'audio/ogg',
+  'audio/wav',
+  'audio/webm',
+  'image/avif',
+  'image/bmp',
+  'image/gif',
+  'image/jpeg',
+  'image/png',
+  'image/webp',
+  'image/x-icon',
+  'video/mp4',
+  'video/ogg',
+  'video/webm',
+]);
+
+const mediaTypesByExtension: Record<string, string> = {
+  avif: 'image/avif',
+  bmp: 'image/bmp',
+  gif: 'image/gif',
+  ico: 'image/x-icon',
+  jpeg: 'image/jpeg',
+  jpg: 'image/jpeg',
+  m4a: 'audio/mp4',
+  mp3: 'audio/mpeg',
+  mp4: 'video/mp4',
+  oga: 'audio/ogg',
+  ogg: 'audio/ogg',
+  ogv: 'video/ogg',
+  png: 'image/png',
+  wav: 'audio/wav',
+  webm: 'video/webm',
+  webp: 'image/webp',
+};
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return value != null && typeof value === 'object' && !Array.isArray(value);
+}
+
+function getKind(mediaType: string): MediaKind {
+  const topLevelType = mediaType.toLowerCase().split('/')[0];
+  return topLevelType === 'image' ||
+    topLevelType === 'audio' ||
+    topLevelType === 'video'
+    ? topLevelType
+    : 'file';
+}
+
+function getBase64Prefix(data: string, maxBytes = 16): number[] {
+  const alphabet =
+    'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/';
+  const bytes: number[] = [];
+  let buffer = 0;
+  let bitCount = 0;
+
+  for (const character of data) {
+    if (character === '=') break;
+
+    const value = alphabet.indexOf(character);
+    if (value === -1) return [];
+
+    buffer = (buffer << 6) | value;
+    bitCount += 6;
+
+    if (bitCount >= 8) {
+      bitCount -= 8;
+      bytes.push((buffer >> bitCount) & 0xff);
+      if (bytes.length >= maxBytes) break;
+    }
+  }
+
+  return bytes;
+}
+
+function hasBytes(bytes: number[], expected: number[], offset = 0): boolean {
+  return expected.every((value, index) => bytes[offset + index] === value);
+}
+
+function detectInlineMediaType({
+  data,
+  mediaType,
+}: {
+  data: string;
+  mediaType: string;
+}): string | undefined {
+  const normalizedMediaType = mediaType.toLowerCase();
+  if (safeInlineMediaTypes.has(normalizedMediaType)) {
+    return normalizedMediaType;
+  }
+
+  const bytes = getBase64Prefix(data);
+
+  if (normalizedMediaType === 'image') {
+    if (hasBytes(bytes, [0x89, 0x50, 0x4e, 0x47])) return 'image/png';
+    if (hasBytes(bytes, [0xff, 0xd8, 0xff])) return 'image/jpeg';
+    if (hasBytes(bytes, [0x47, 0x49, 0x46, 0x38])) return 'image/gif';
+    if (
+      hasBytes(bytes, [0x52, 0x49, 0x46, 0x46]) &&
+      hasBytes(bytes, [0x57, 0x45, 0x42, 0x50], 8)
+    ) {
+      return 'image/webp';
+    }
+    if (hasBytes(bytes, [0x42, 0x4d])) return 'image/bmp';
+  }
+
+  if (normalizedMediaType === 'audio') {
+    if (hasBytes(bytes, [0x49, 0x44, 0x33])) return 'audio/mpeg';
+    if (bytes[0] === 0xff && bytes[1] != null && (bytes[1] & 0xe0) === 0xe0) {
+      return 'audio/mpeg';
+    }
+    if (
+      hasBytes(bytes, [0x52, 0x49, 0x46, 0x46]) &&
+      hasBytes(bytes, [0x57, 0x41, 0x56, 0x45], 8)
+    ) {
+      return 'audio/wav';
+    }
+    if (hasBytes(bytes, [0x4f, 0x67, 0x67, 0x53])) return 'audio/ogg';
+    if (hasBytes(bytes, [0x1a, 0x45, 0xdf, 0xa3])) return 'audio/webm';
+    if (hasBytes(bytes, [0x66, 0x74, 0x79, 0x70], 4)) return 'audio/mp4';
+  }
+
+  if (normalizedMediaType === 'video') {
+    if (hasBytes(bytes, [0x4f, 0x67, 0x67, 0x53])) return 'video/ogg';
+    if (hasBytes(bytes, [0x1a, 0x45, 0xdf, 0xa3])) return 'video/webm';
+    if (hasBytes(bytes, [0x66, 0x74, 0x79, 0x70], 4)) return 'video/mp4';
+  }
+
+  return undefined;
+}
+
+function inferMediaTypeFromUrl(source: unknown): string | undefined {
+  if (typeof source !== 'string') return undefined;
+
+  try {
+    const extension = new URL(source).pathname.split('.').pop()?.toLowerCase();
+    return extension == null ? undefined : mediaTypesByExtension[extension];
+  } catch {
+    return undefined;
+  }
+}
+
+function getBase64ByteLength(value: string): number | undefined {
+  if (
+    value.length === 0 ||
+    value.length % 4 !== 0 ||
+    !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(
+      value,
+    )
+  ) {
+    return undefined;
+  }
+
+  const padding = value.endsWith('==') ? 2 : value.endsWith('=') ? 1 : 0;
+  return (value.length / 4) * 3 - padding;
+}
+
+function getSafeSource({
+  source,
+  mediaType,
+  maxInlineBytes,
+}: {
+  source: unknown;
+  mediaType: string;
+  maxInlineBytes: number;
+}): Pick<MediaPreviewData, 'source' | 'sourceType' | 'unavailableReason'> {
+  if (typeof source !== 'string') {
+    return {};
+  }
+
+  if (source.startsWith('data:')) {
+    const match = /^data:([^;,]+);base64,(.*)$/i.exec(source);
+    const sourceMediaType = match?.[1]?.toLowerCase();
+    const byteLength =
+      match?.[2] == null ? undefined : getBase64ByteLength(match[2]);
+    const declaredTopLevelType = mediaType.toLowerCase().split('/')[0];
+    if (
+      sourceMediaType == null ||
+      byteLength == null ||
+      (sourceMediaType !== mediaType.toLowerCase() &&
+        sourceMediaType.split('/')[0] !== declaredTopLevelType) ||
+      !safeInlineMediaTypes.has(sourceMediaType)
+    ) {
+      return {};
+    }
+
+    if (byteLength > maxInlineBytes) {
+      return {
+        unavailableReason: `Inline preview exceeds the ${maxInlineBytes}-byte limit.`,
+      };
+    }
+
+    return { source, sourceType: 'inline' };
+  }
+
+  try {
+    const url = new URL(source);
+    return (url.protocol === 'http:' || url.protocol === 'https:') &&
+      url.username === '' &&
+      url.password === ''
+      ? { source: url.toString(), sourceType: 'remote' }
+      : {};
+  } catch {
+    const inlineMediaType = detectInlineMediaType({
+      data: source,
+      mediaType,
+    });
+    const byteLength = getBase64ByteLength(source);
+    if (inlineMediaType == null || byteLength == null) {
+      return {};
+    }
+
+    if (byteLength > maxInlineBytes) {
+      return {
+        unavailableReason: `Inline preview exceeds the ${maxInlineBytes}-byte limit.`,
+      };
+    }
+
+    return {
+      source: `data:${inlineMediaType};base64,${source}`,
+      sourceType: 'inline',
+    };
+  }
+}
+
+function getFileData(
+  value: Record<string, unknown>,
+  maxInlineBytes: number,
+): Pick<MediaPreviewData, 'source' | 'sourceType' | 'unavailableReason'> {
+  const generatedFile = isRecord(value.file) ? value.file : undefined;
+  const mediaType =
+    typeof value.mediaType === 'string'
+      ? value.mediaType
+      : typeof generatedFile?.mediaType === 'string'
+        ? generatedFile.mediaType
+        : undefined;
+  if (mediaType == null) {
+    return {};
+  }
+
+  const data = value.data;
+  if (isRecord(data)) {
+    if (data.type === 'data') {
+      return getSafeSource({ source: data.data, mediaType, maxInlineBytes });
+    }
+    if (data.type === 'url') {
+      return getSafeSource({ source: data.url, mediaType, maxInlineBytes });
+    }
+    return {};
+  }
+
+  const generatedFileData =
+    generatedFile?.base64 ??
+    generatedFile?.base64Data ??
+    generatedFile?.uint8Array ??
+    generatedFile?.uint8ArrayData;
+
+  return getSafeSource({
+    source: data ?? generatedFileData,
+    mediaType,
+    maxInlineBytes,
+  });
+}
+
+function parseMediaPart(
+  value: unknown,
+  maxInlineBytes: number,
+): MediaPreviewData | undefined {
+  if (!isRecord(value) || typeof value.type !== 'string') {
+    return undefined;
+  }
+
+  if (value.type === 'file' || value.type === 'reasoning-file') {
+    const generatedFile = isRecord(value.file) ? value.file : undefined;
+    const mediaType =
+      typeof value.mediaType === 'string'
+        ? value.mediaType
+        : typeof generatedFile?.mediaType === 'string'
+          ? generatedFile.mediaType
+          : undefined;
+    if (mediaType == null) {
+      return undefined;
+    }
+
+    return {
+      filename:
+        typeof value.filename === 'string'
+          ? value.filename
+          : typeof generatedFile?.filename === 'string'
+            ? generatedFile.filename
+            : undefined,
+      kind: getKind(mediaType),
+      mediaType,
+      ...getFileData(value, maxInlineBytes),
+    };
+  }
+
+  if (value.type === 'image') {
+    const mediaType =
+      typeof value.mediaType === 'string' ? value.mediaType : 'image/png';
+    const image = value.image;
+    const source =
+      isRecord(image) && image.type === 'url'
+        ? image.url
+        : isRecord(image) && image.type === 'data'
+          ? image.data
+          : image;
+
+    return {
+      kind: 'image',
+      mediaType,
+      ...getSafeSource({ source, mediaType, maxInlineBytes }),
+    };
+  }
+
+  if (
+    value.type === 'media' ||
+    value.type === 'file-data' ||
+    value.type === 'image-data'
+  ) {
+    const mediaType =
+      typeof value.mediaType === 'string'
+        ? value.mediaType
+        : value.type === 'image-data'
+          ? 'image/png'
+          : undefined;
+    if (mediaType == null) {
+      return undefined;
+    }
+
+    return {
+      filename: typeof value.filename === 'string' ? value.filename : undefined,
+      kind: getKind(mediaType),
+      mediaType,
+      ...getSafeSource({
+        source: value.data,
+        mediaType,
+        maxInlineBytes,
+      }),
+    };
+  }
+
+  if (value.type === 'image-url' || value.type === 'file-url') {
+    const mediaType =
+      typeof value.mediaType === 'string'
+        ? value.mediaType
+        : value.type === 'image-url'
+          ? 'image'
+          : (inferMediaTypeFromUrl(value.url) ?? 'application/octet-stream');
+
+    return {
+      filename: typeof value.filename === 'string' ? value.filename : undefined,
+      kind: getKind(mediaType),
+      mediaType,
+      ...getSafeSource({ source: value.url, mediaType, maxInlineBytes }),
+    };
+  }
+
+  return undefined;
+}
+
+export function findMediaPreviews(
+  value: unknown,
+  limitOverrides: Partial<MediaPreviewLimits> = {},
+): MediaPreviewData[] {
+  const limits = { ...DEFAULT_MEDIA_PREVIEW_LIMITS, ...limitOverrides };
+  const previews: MediaPreviewData[] = [];
+  const visited = new WeakSet<object>();
+  const pending: Array<{ value: unknown; depth: number }> = [
+    { value, depth: 0 },
+  ];
+  let visitedNodes = 0;
+
+  while (
+    pending.length > 0 &&
+    previews.length < limits.maxCount &&
+    visitedNodes < limits.maxNodes
+  ) {
+    const { value: current, depth } = pending.pop()!;
+    visitedNodes++;
+
+    const preview = parseMediaPart(current, limits.maxInlineBytes);
+    if (preview != null) {
+      previews.push(preview);
+      continue;
+    }
+
+    if (
+      depth >= limits.maxDepth ||
+      current == null ||
+      typeof current !== 'object'
+    ) {
+      continue;
+    }
+
+    if (visited.has(current)) {
+      continue;
+    }
+    visited.add(current);
+
+    const children = Array.isArray(current)
+      ? current
+      : isRecord(current)
+        ? Object.values(current)
+        : [];
+    for (let index = children.length - 1; index >= 0; index--) {
+      pending.push({ value: children[index], depth: depth + 1 });
+    }
+  }
+
+  return previews;
+}

package/src/viewer/client/types.ts

@@ -137,11 +137,25 @@ export interface ReasoningContentPart {
   toolCallId?: string;
 }
 
+export interface MediaContentPart {
+  type:
+    | 'file'
+    | 'reasoning-file'
+    | 'image'
+    | 'media'
+    | 'file-data'
+    | 'file-url'
+    | 'image-data'
+    | 'image-url';
+  [key: string]: unknown;
+}
+
 export type ContentPart =
   | TextContentPart
   | ToolCallContentPart
   | ToolResultContentPart
-  | ReasoningContentPart;
+  | ReasoningContentPart
+  | MediaContentPart;
 
 export type MessageRole = 'user' | 'assistant' | 'system' | 'tool';
 
@@ -169,9 +183,13 @@ export interface ToolDefinition {
 export interface ParsedOutput {
   finishReason?: string | { unified?: string; raw?: string };
   toolCalls?: ToolCallContentPart[];
+  toolResults?: ToolResultContentPart[];
   textParts?: TextContentPart[];
   reasoningParts?: ReasoningContentPart[];
   content?: ContentPart[];
+  response?: {
+    messages?: PromptMessage[];
+  };
 }
 
 export interface ParsedUsage {