ai 7.0.162 → 7.0.164

package/src/generate-text/to-response-messages.ts

@@ -39,8 +39,12 @@ export async function toResponseMessages<TOOLS extends ToolSet>({
       continue;
     }
 
-    // Skip empty text
-    if (part.type === 'text' && part.text.length === 0) {
+    // Skip empty text without provider metadata.
+    if (
+      part.type === 'text' &&
+      part.text.length === 0 &&
+      part.providerMetadata == null
+    ) {
       continue;
     }
 

package/src/generate-video/generate-video-result.ts

@@ -9,15 +9,24 @@ import type { Warning } from '../types/warning';
  */
 export interface GenerateVideoResult {
   /**
-   * The first video that was generated.
+   * The first video available to the SDK.
+   *
+   * @throws {NoVideoGeneratedError} When all videos were delivered directly
+   * to caller-managed storage. Use `uploadedVideos` to handle upload-only results.
    */
   readonly video: GeneratedFile;
 
   /**
-   * All videos that were generated.
+   * All videos available to the SDK, excluding videos uploaded by the provider.
    */
   readonly videos: Array<GeneratedFile>;
 
+  /**
+   * Number of videos delivered directly to caller-managed storage.
+   * These videos are not downloaded and are not included in `videos`.
+   */
+  readonly uploadedVideos: number;
+
   /**
    * Warnings for the call, e.g. unsupported settings.
    */

package/src/generate-video/generate-video.ts

@@ -377,12 +377,18 @@ export async function experimental_generateVideo({
 
   // collect result videos, warnings, and response metadata
   const videos: Array<GeneratedFile> = [];
+  let uploadedVideos = 0;
   const responses: Array<VideoModelResponseMetadata> = [];
   const providerMetadata: SharedV4ProviderMetadata = {};
 
   for (const result of results) {
     for (const videoData of result.videos) {
       switch (videoData.type) {
+        case 'uploaded': {
+          uploadedVideos++;
+          break;
+        }
+
         case 'url': {
           const { data, mediaType: downloadedMediaType } = await downloadFn({
             url: new URL(videoData.url),
@@ -455,7 +461,7 @@ export async function experimental_generateVideo({
     }
   }
 
-  if (videos.length === 0) {
+  if (videos.length === 0 && uploadedVideos === 0) {
     throw new NoVideoGeneratedError({ responses });
   }
 
@@ -467,13 +473,31 @@ export async function experimental_generateVideo({
     });
   }
 
-  return {
+  const result = {
     video: videos[0],
     videos,
+    uploadedVideos,
     warnings,
     responses,
     providerMetadata,
   };
+
+  if (videos.length === 0) {
+    // Keep upload-only results safe to serialize or spread. Normal results
+    // retain their existing enumerable video data property.
+    Object.defineProperty(result, 'video', {
+      enumerable: false,
+      get() {
+        throw new NoVideoGeneratedError({
+          message:
+            'Videos were delivered to the upload destination. Use uploadedVideos instead of video.',
+          responses,
+        });
+      },
+    });
+  }
+
+  return result;
 }
 
 async function executeStartStatusFlow({

package/src/model/as-transcription-model-v4.ts

@@ -19,6 +19,31 @@ export function asTranscriptionModelV4(
   return new Proxy(v3Model, {
     get(target, prop: keyof TranscriptionModelV3) {
       if (prop === 'specificationVersion') return 'v4';
+      if (prop === 'doGenerate') {
+        return async ({
+          timestampGranularities,
+          ...options
+        }: Parameters<TranscriptionModelV4['doGenerate']>[0]) => {
+          // Bind to the original model, including when target is a v2 proxy.
+          const result = await target.doGenerate.call(model, options);
+
+          if (!timestampGranularities?.length) {
+            return result;
+          }
+
+          return {
+            ...result,
+            warnings: [
+              ...result.warnings,
+              {
+                type: 'unsupported' as const,
+                feature: 'timestampGranularities',
+                details: `Shared timestamp granularities are not supported by ${model.specificationVersion} transcription models.`,
+              },
+            ],
+          };
+        };
+      }
       return target[prop];
     },
   }) as unknown as TranscriptionModelV4;

package/src/prompt/convert-to-language-model-prompt.ts

@@ -445,8 +445,10 @@ export async function downloadAssets(
         }
 
         for (const contentPart of part.output.value) {
-          if (contentPart.type === 'file') {
-            downloadableFiles.push(contentPart);
+          const filePart = getDownloadableFilePart(contentPart);
+
+          if (filePart != null) {
+            downloadableFiles.push(filePart);
           }
         }
       }
@@ -461,8 +463,10 @@ export async function downloadAssets(
           continue;
         }
         for (const contentPart of part.output.value) {
-          if (contentPart.type === 'file') {
-            downloadableFiles.push(contentPart);
+          const filePart = getDownloadableFilePart(contentPart);
+
+          if (filePart != null) {
+            downloadableFiles.push(filePart);
           }
         }
       }
@@ -503,6 +507,74 @@ export async function downloadAssets(
   );
 }
 
+/**
+ * Resolves the tool result content parts that point at a remote file to the
+ * tagged file shape, including the deprecated `file-url` and `image-url`
+ * types, so that they are downloaded and inlined exactly like their `file`
+ * replacement. Returns `undefined` for content that is already inline or
+ * provider-referenced.
+ */
+function getDownloadableFilePart(
+  contentPart: Extract<ToolResultOutput, { type: 'content' }>['value'][number],
+): FilePart | undefined {
+  switch (contentPart.type) {
+    case 'file':
+      return contentPart;
+    case 'file-url':
+      return {
+        type: 'file',
+        data: { type: 'url', url: new URL(contentPart.url) },
+        mediaType:
+          contentPart.mediaType ?? getMediaTypeFromUrl(contentPart.url),
+        providerOptions: contentPart.providerOptions,
+      };
+    case 'image-url':
+      return {
+        type: 'file',
+        data: { type: 'url', url: new URL(contentPart.url) },
+        mediaType: 'image',
+        providerOptions: contentPart.providerOptions,
+      };
+    default:
+      return undefined;
+  }
+}
+
+/**
+ * Resolves file data and media type for a remote tool result content part: the downloaded
+ * bytes when the asset was fetched, otherwise the URL reference. `originalUrl`
+ * is preserved when the URL string does not round-trip through `URL`.
+ */
+function resolveDownloadedFile({
+  url,
+  originalUrl,
+  downloadedAssets,
+  mediaType,
+}: {
+  url: URL;
+  originalUrl: string;
+  downloadedAssets: Record<
+    string,
+    { mediaType: string | undefined; data: Uint8Array }
+  >;
+  mediaType: string;
+}): Pick<LanguageModelV4FilePart, 'data' | 'mediaType'> {
+  const convertedPart = convertPartToLanguageModelPart(
+    {
+      type: 'file',
+      data: {
+        type: 'url',
+        url,
+        ...(url.toString() !== originalUrl ? { originalUrl } : {}),
+      },
+      mediaType,
+    },
+    downloadedAssets,
+  ) as LanguageModelV4FilePart;
+
+  return { data: convertedPart.data, mediaType: convertedPart.mediaType };
+}
+
 /**
  * Convert part of a user message to a LanguageModelV4Part.
  *
@@ -641,12 +713,12 @@ export function mapToolResultOutput({
           });
           return {
             type: 'file' as const,
-            data: {
-              type: 'url' as const,
+            ...resolveDownloadedFile({
               url,
-              ...(url.toString() !== item.url ? { originalUrl: item.url } : {}),
-            },
-            mediaType,
+              originalUrl: item.url,
+              downloadedAssets,
+              mediaType,
+            }),
             providerOptions: item.providerOptions,
           };
         }
@@ -709,12 +781,12 @@ export function mapToolResultOutput({
           });
           return {
             type: 'file' as const,
-            data: {
-              type: 'url' as const,
+            ...resolveDownloadedFile({
               url,
-              ...(url.toString() !== item.url ? { originalUrl: item.url } : {}),
-            },
-            mediaType: 'image',
+              originalUrl: item.url,
+              downloadedAssets,
+              mediaType: 'image',
+            }),
             providerOptions: item.providerOptions,
           };
         }

package/src/realtime/index.ts

@@ -18,7 +18,10 @@ export type {
   RealtimeFactoryGetTokenOptions as Experimental_RealtimeFactoryGetTokenOptions,
   RealtimeFactoryGetTokenResult as Experimental_RealtimeFactoryGetTokenResult,
   RealtimeModel as Experimental_RealtimeModel,
+  RealtimeServerConnection as Experimental_RealtimeServerConnection,
   RealtimeServerEvent as Experimental_RealtimeServerEvent,
   RealtimeSessionConfig as Experimental_RealtimeSessionConfig,
+  RealtimeSessionIntent as Experimental_RealtimeSessionIntent,
   RealtimeToolDefinition as Experimental_RealtimeToolDefinition,
+  RealtimeUsage as Experimental_RealtimeUsage,
 } from '../types/realtime-model';

package/src/realtime/realtime-attempt.ts

@@ -1,6 +1,7 @@
 /** Cancellation and settlement belong to one connection attempt. */
 export class RealtimeAttempt {
   readonly abort = new AbortController();
+  readonly reportedTransportErrors = new WeakSet<Error>();
   private timers = new Map<string, ReturnType<typeof setTimeout>>();
   private settleClose?: () => void;
   closePromise?: Promise<void>;

package/src/realtime/realtime-session.ts

@@ -1,4 +1,7 @@
-import { UnsupportedFunctionalityError } from '@ai-sdk/provider';
+import {
+  UnsupportedFunctionalityError,
+  type Experimental_RealtimeModelV4Transport as RealtimeModelTransport,
+} from '@ai-sdk/provider';
 import type {
   RealtimeClientEvent,
   RealtimeModel,
@@ -28,15 +31,35 @@ export type { RealtimeState, RealtimeStatus } from './realtime-event-reducer';
 
 export type RealtimeSessionOptions = {
   model: RealtimeModel;
-  /** websocket uses a raw-protocol relay; session exchanges SDP; token fetches client-secret setup. */
+  /** Application endpoints for token, SDP, provider-managed, or relay connections. */
   api:
-    | { token: string; session?: never; websocket?: never; protocols?: never }
-    | { session: string; token?: never; websocket?: never; protocols?: never }
+    | {
+        token: string;
+        session?: never;
+        websocket?: never;
+        protocols?: never;
+        provider?: never;
+      }
+    | {
+        session: string;
+        token?: never;
+        websocket?: never;
+        protocols?: never;
+        provider?: never;
+      }
+    | {
+        provider: string;
+        token?: never;
+        session?: never;
+        websocket?: never;
+        protocols?: never;
+      }
     | {
         websocket: string;
         protocols?: string[];
         token?: never;
         session?: never;
+        provider?: never;
       };
   sessionConfig?: Partial<RealtimeSessionConfig>;
   sampleRate?: number;
@@ -72,6 +95,7 @@ export abstract class AbstractRealtimeSession {
   private audio?: BrowserRealtimeAudio;
   private pcm?: BrowserRealtimeLiveWebSocket;
   private rtc?: BrowserRealtimeWebRTC;
+  private providerTransport?: RealtimeModelTransport;
   private suppliedStream?: MediaStream;
   private captureGeneration = 0;
   private captureRequested = false;
@@ -116,28 +140,36 @@ export abstract class AbstractRealtimeSession {
   private validateConnection(): void {
     const { model, api } = this.options;
     if (
-      (!this.continuous || api.session != null) &&
+      (!this.continuous || api.session != null || api.provider != null) &&
       this.options.maxPlaybackBufferSeconds != null
     )
       throw new Error(
         'maxPlaybackBufferSeconds is supported only for continuous PCM sessions',
       );
     const connection =
-      api.session != null
-        ? 'webrtc'
-        : api.token != null
-          ? 'client-secret-websocket'
-          : 'server-websocket';
+      api.provider != null
+        ? 'provider'
+        : api.session != null
+          ? 'webrtc'
+          : api.token != null
+            ? 'client-secret-websocket'
+            : 'server-websocket';
     if (
       !(
         model.capabilities?.connections ?? ['client-secret-websocket']
       ).includes(connection) ||
-      (model.capabilities?.transports != null &&
+      (api.provider == null &&
+        model.capabilities?.transports != null &&
         !model.capabilities.transports.includes(
           api.session != null ? 'webrtc' : 'websocket',
         ))
     )
       throw new Error(`Realtime model does not support ${connection}`);
+    if (api.provider != null) {
+      if (model.createTransport == null)
+        throw new Error('Realtime model does not support a provider transport');
+      return;
+    }
     if (api.session != null) {
       if (model.getWebRTCConfig == null)
         throw new Error('Realtime model does not support WebRTC configuration');
@@ -179,7 +211,8 @@ export abstract class AbstractRealtimeSession {
           this.captureRequested &&
           !this.sessionLifecycle &&
           !this.continuous &&
-          api.session == null
+          api.session == null &&
+          api.provider == null
         )
           this.startAudioCapture(connectOptions.stream);
         else this.suppliedStream = connectOptions.stream;
@@ -206,7 +239,7 @@ export abstract class AbstractRealtimeSession {
         if (current())
           this.fail(new Error('Realtime session startup timed out'));
       });
-      const reportedTransportErrors = new WeakSet<Error>();
+      const reportedTransportErrors = attempt.reportedTransportErrors;
       const callbacks = {
         model,
         onEvent: async (event: RealtimeServerEvent) => {
@@ -217,7 +250,12 @@ export abstract class AbstractRealtimeSession {
             if (current())
               this.fail(
                 error,
-                (this.rtc ?? this.pcm ?? this.transport)?.finish(),
+                (
+                  this.providerTransport ??
+                  this.rtc ??
+                  this.pcm ??
+                  this.transport
+                )?.finish(),
               );
           }
         },
@@ -248,6 +286,19 @@ export abstract class AbstractRealtimeSession {
           const finalizationConfirmed =
             model.capabilities?.finalization === 'session-close' &&
             this.state.session?.finalization === 'confirmed';
+          // Local media can close before remote cleanup settles. The provider's
+          // close promise and its deadline own completion during shutdown.
+          if (api.provider != null && attempt.closing) {
+            if (
+              error != null &&
+              !finalizationConfirmed &&
+              !reportedTransportErrors.has(error)
+            ) {
+              reportedTransportErrors.add(error);
+              void this.reportError(error, attempt);
+            }
+            return;
+          }
           if (error != null && !finalizationConfirmed)
             this.fail(error, undefined, !reportedTransportErrors.has(error));
           else if (!attempt.ready && !finalizationConfirmed)
@@ -263,7 +314,21 @@ export abstract class AbstractRealtimeSession {
           if (current()) this.applyState({ ...this.state, isPlaying });
         },
       };
-      if (api.session != null) {
+      if (api.provider != null) {
+        const transport = model.createTransport!(callbacks);
+        if (!current()) {
+          transport.dispose();
+          return;
+        }
+        this.providerTransport = transport;
+        await transport.connect({
+          api: api.provider,
+          sessionConfig,
+          stream: connectOptions?.stream,
+          capture: connectOptions?.capture,
+          abortSignal: attempt.abort.signal,
+        });
+      } else if (api.session != null) {
         this.rtc = new BrowserRealtimeWebRTC({
           ...callbacks,
           disconnectTimeoutMs: this.options.rtcDisconnectTimeoutMs,
@@ -394,8 +459,13 @@ export abstract class AbstractRealtimeSession {
     if (this.attempt !== attempt || !attempt.active || attempt.transportClosing)
       return;
     attempt.clearTimer('close');
-    const transport = this.rtc ?? this.pcm ?? this.transport;
+    const transport =
+      this.providerTransport ?? this.rtc ?? this.pcm ?? this.transport;
     void this.drainAttempt(attempt, transport?.finish());
+    if (error instanceof Error) {
+      if (attempt.reportedTransportErrors.has(error)) return;
+      attempt.reportedTransportErrors.add(error);
+    }
     void this.reportError(error, attempt);
   }
 
@@ -408,16 +478,19 @@ export abstract class AbstractRealtimeSession {
     const audio = this.audio;
     const pcm = this.pcm;
     const rtc = this.rtc;
+    const providerTransport = this.providerTransport;
     this.transport = undefined;
     this.audio = undefined;
     this.pcm = undefined;
     this.rtc = undefined;
+    this.providerTransport = undefined;
     this.commands = undefined;
     attempt?.retire();
     transport?.dispose();
     audio?.dispose();
     pcm?.dispose();
     rtc?.dispose();
+    providerTransport?.dispose();
     if (this.attempt !== attempt) return;
     const session = this.state.session;
     this.applyState({
@@ -436,10 +509,28 @@ export abstract class AbstractRealtimeSession {
     });
   }
 
-  /** Wait for final usage when the model supports a session-close acknowledgement. */
+  /** Wait for provider cleanup or supported session-close finalization. */
   close(options?: { eventId?: string }): Promise<void> {
     const attempt = this.attempt;
     if (attempt?.closePromise != null) return attempt.closePromise;
+    const providerTransport = this.providerTransport;
+    if (providerTransport != null && attempt?.active) {
+      const promise = attempt.beginClose();
+      attempt.clearTimer('startup');
+      this.applyState({ ...this.state, status: 'closing' });
+      if (this.attempt !== attempt || !attempt.active) return promise;
+      this.stopAudioCapture();
+      if (this.attempt !== attempt || !attempt.active) return promise;
+      attempt.timer('close', this.options.closeTimeoutMs ?? 15_000, () => {
+        if (this.attempt === attempt && attempt.active)
+          this.fail(new Error('Realtime provider session cleanup timed out'));
+      });
+      void providerTransport.close().then(
+        () => this.finishAttempt(attempt),
+        error => this.closeFailed(attempt, error),
+      );
+      return promise;
+    }
     if (
       this.state.status !== 'connected' ||
       this.options.model.capabilities?.finalization !== 'session-close' ||
@@ -475,8 +566,18 @@ export abstract class AbstractRealtimeSession {
     guard?: () => boolean,
     automaticAudio = false,
   ): Promise<void> {
-    const transport = this.rtc ?? this.pcm ?? this.transport;
+    const transport =
+      this.providerTransport ?? this.rtc ?? this.pcm ?? this.transport;
     if (transport == null) throw new Error('Realtime connection is not open');
+    if (transport === this.providerTransport) {
+      const attempt = this.attempt;
+      const sent = transport.sendEvent(event, guard);
+      void sent.catch(error => {
+        if (attempt?.active && this.attempt === attempt)
+          void this.reportError(error, attempt);
+      });
+      return sent;
+    }
     if (transport === this.transport)
       return this.transport.sendEvent(event, guard, automaticAudio);
     return transport.sendEvent(event, guard);
@@ -606,6 +707,7 @@ export abstract class AbstractRealtimeSession {
 
   private maybeRequestToolResponse(): void {
     if (
+      this.providerTransport != null ||
       this.state.status !== 'connected' ||
       this.attempt?.closing ||
       !this.responseToolCallsClosed ||
@@ -625,7 +727,8 @@ export abstract class AbstractRealtimeSession {
     const legacy =
       !this.sessionLifecycle &&
       !this.continuous &&
-      this.options.api.session == null;
+      this.options.api.session == null &&
+      this.options.api.provider == null;
     if (!legacy && this.state.status !== 'connected')
       throw new Error('Realtime session is not accepting capture');
     if (
@@ -655,6 +758,8 @@ export abstract class AbstractRealtimeSession {
     if (!accepting())
       throw new Error('Realtime session is not accepting capture');
     this.captureRequested = true;
+    if (this.providerTransport != null)
+      return this.providerTransport.startCapture(this.suppliedStream);
     if (this.rtc != null) return this.rtc.startCapture(this.suppliedStream);
     if (this.pcm != null) return this.pcm.resumeCapture(this.suppliedStream);
     const audio = this.audio;
@@ -694,7 +799,13 @@ export abstract class AbstractRealtimeSession {
     const pcm = this.pcm;
     const audio = this.audio;
     const rtc = this.rtc;
+    const providerTransport = this.providerTransport;
     const attempt = this.attempt;
+    if (providerTransport != null)
+      void providerTransport.stopCapture().catch(error => {
+        if (attempt?.active && this.attempt === attempt)
+          void this.reportError(error, attempt);
+      });
     if (rtc != null)
       void rtc.stopCapture().catch(error => {
         if (attempt?.active && this.attempt === attempt)
@@ -704,10 +815,20 @@ export abstract class AbstractRealtimeSession {
     audio?.stopCapture();
   }
   stopPlayback(): void {
-    (this.rtc ?? this.pcm ?? this.audio)?.stopPlayback();
+    (
+      this.providerTransport ??
+      this.rtc ??
+      this.pcm ??
+      this.audio
+    )?.stopPlayback();
   }
   async resumePlayback(): Promise<void> {
-    await (this.rtc ?? this.pcm ?? this.audio)?.resumePlayback();
+    await (
+      this.providerTransport ??
+      this.rtc ??
+      this.pcm ??
+      this.audio
+    )?.resumePlayback();
   }
   dispose(): void {
     this.disconnect();

package/src/telemetry/create-telemetry-dispatcher.ts

@@ -18,6 +18,7 @@ import {
 } from './tracing-channel-publisher';
 import { getGlobalTelemetryIntegrations } from './telemetry-registry';
 import type { TelemetryOptions } from './telemetry-options';
+import { filterTelemetryEvent } from './filter-telemetry-event';
 
 /**
  * The subset of `TelemetryDispatcher` keys whose values are Callback callbacks.
@@ -100,7 +101,7 @@ function augmentEvent<EVENT>(
     }
   }
 
-  return augmentedEvent;
+  return filterTelemetryEvent(augmentedEvent, telemetry);
 }
 
 /**
@@ -141,6 +142,9 @@ export function createTelemetryDispatcher({
     includeRuntimeContext: telemetry?.includeRuntimeContext,
     includeToolsContext: telemetry?.includeToolsContext,
   };
+  // Raw operation results may contain both inputs and outputs.
+  const recordResult =
+    telemetry?.recordInputs !== false && telemetry?.recordOutputs !== false;
 
   const mergeTelemetryCallback = <KEY extends TelemetryCallbackKey>(
     key: KEY,
@@ -199,6 +203,7 @@ export function createTelemetryDispatcher({
           event: augmentEvent(event, telemetryMetadata, true),
         },
         execute,
+        recordResult,
       ),
 
     startTracingChannelContext: ({ type, event, completion }) =>
@@ -208,6 +213,7 @@ export function createTelemetryDispatcher({
           event: augmentEvent(event, telemetryMetadata, true),
         },
         completion,
+        recordResult,
       }),
 
     onStart: mergeTelemetryCallback('onStart'),
@@ -276,6 +282,7 @@ export function createTelemetryDispatcher({
       return await runWithTracingChannelSpan(
         { type: 'languageModelCall', event: augmentedEvent },
         wrappedExecute,
+        recordResult,
       );
     },