ai 7.0.162 → 7.0.164

package/src/telemetry/filter-telemetry-event.ts

@@ -0,0 +1,165 @@
+import type { TelemetryOptions } from './telemetry-options';
+
+/**
+ * Filters SDK-owned content fields on a copy. Only descend into SDK event
+ * containers; user context has its own inclusion policy.
+ */
+export function filterTelemetryEvent<EVENT>(
+  event: EVENT,
+  telemetry: Pick<TelemetryOptions, 'recordInputs' | 'recordOutputs'>,
+): EVENT {
+  const hideInputs = telemetry.recordInputs === false;
+  const hideOutputs = telemetry.recordOutputs === false;
+  if (
+    (!hideInputs && !hideOutputs) ||
+    event == null ||
+    typeof event !== 'object'
+  ) {
+    return event;
+  }
+
+  const filtered = Object.assign(
+    Object.create(Object.getPrototypeOf(event)),
+    event,
+  ) as Record<string, unknown>;
+  const replace = (fields: Record<string, unknown>) => {
+    for (const [key, value] of Object.entries(fields)) {
+      if (Object.hasOwn(event, key)) {
+        filtered[key] = value;
+      }
+    }
+  };
+
+  const isSpeech = filtered.operationId === 'ai.generateSpeech';
+  if (hideInputs) {
+    replace({
+      instructions: undefined,
+      system: undefined,
+      prompt: undefined,
+      messages: [],
+      promptMessages: [],
+      experimental_steeringMessages: [],
+      tools: undefined,
+      stepTools: undefined,
+      schema: undefined,
+      schemaName: undefined,
+      schemaDescription: undefined,
+      ...(typeof filtered.output === 'object' ? { output: undefined } : {}),
+      providerOptions: {},
+      value: '',
+      values: [],
+      query: '',
+      documents: [],
+      state: Array.isArray(filtered.state)
+        ? []
+        : typeof filtered.state === 'string'
+          ? ''
+          : {},
+      questions: {},
+      ...(isSpeech ? { text: '' } : {}),
+    });
+    if (filtered.toolCall != null) {
+      filtered.toolCall = { ...filtered.toolCall, input: undefined };
+    }
+    if (filtered.toolOutput != null) {
+      filtered.toolOutput = { ...filtered.toolOutput, input: undefined };
+    }
+    if (Array.isArray(filtered.ranking)) {
+      filtered.ranking = filtered.ranking.map(entry =>
+        'document' in entry
+          ? { ...entry, document: typeof entry.document === 'string' ? '' : {} }
+          : entry,
+      );
+    }
+  }
+
+  if (hideOutputs) {
+    replace({
+      content: [],
+      ...(!isSpeech ? { text: '' } : {}),
+      reasoning: Array.isArray(filtered.reasoning) ? [] : undefined,
+      reasoningText: undefined,
+      files: [],
+      sources: [],
+      toolCalls: [],
+      staticToolCalls: [],
+      dynamicToolCalls: [],
+      toolResults: [],
+      staticToolResults: [],
+      dynamicToolResults: [],
+      responseMessages: [],
+      object: undefined,
+      objectText: '',
+      embedding: [],
+      embeddings: [],
+      ranking: [],
+      answers: {},
+      segments: [],
+    });
+    if (filtered.toolOutput != null) {
+      filtered.toolOutput = {
+        ...filtered.toolOutput,
+        output: undefined,
+        error: undefined,
+      };
+    }
+  }
+
+  // Raw provider payloads and errors can contain either inputs or outputs.
+  replace({
+    providerMetadata: undefined,
+    error:
+      filtered.error == null
+        ? filtered.error
+        : new Error('AI SDK operation failed'),
+  });
+  for (const key of ['toolCall', 'toolOutput'] as const) {
+    const toolEvent = filtered[key];
+    if (toolEvent != null && typeof toolEvent === 'object') {
+      filtered[key] = {
+        ...toolEvent,
+        providerMetadata: undefined,
+        ...('error' in toolEvent
+          ? {
+              error:
+                toolEvent.error == null
+                  ? toolEvent.error
+                  : new Error('AI SDK operation failed'),
+            }
+          : {}),
+      };
+    }
+  }
+  for (const key of ['request', 'response'] as const) {
+    const filterMetadata = (metadata: unknown) => {
+      if (metadata == null || typeof metadata !== 'object') {
+        return metadata;
+      }
+      const filteredMetadata: Record<string, unknown> = { ...metadata };
+      delete filteredMetadata.body;
+      if (key === 'request' ? hideInputs : hideOutputs) {
+        if ('messages' in metadata) filteredMetadata.messages = [];
+        if ('instructions' in metadata)
+          filteredMetadata.instructions = undefined;
+      }
+      return filteredMetadata;
+    };
+    const metadata = filtered[key];
+    if (metadata != null) {
+      filtered[key] = Array.isArray(metadata)
+        ? metadata.map(filterMetadata)
+        : filterMetadata(metadata);
+    }
+  }
+
+  if (Array.isArray(filtered.steps)) {
+    filtered.steps = filtered.steps.map(step =>
+      filterTelemetryEvent(step, telemetry),
+    );
+  }
+  if (Object.hasOwn(event, 'finalStep')) {
+    filtered.finalStep = filterTelemetryEvent(filtered.finalStep, telemetry);
+  }
+
+  return filtered as EVENT;
+}

package/src/telemetry/get-request-origin.ts

@@ -0,0 +1,25 @@
+import type { LanguageModelV4GenerateResult } from '@ai-sdk/provider';
+
+/** Remove credentials, paths, queries, and fragments before emitting telemetry. */
+export function getRequestOrigin(url: string | undefined): string | undefined {
+  if (url == null) return undefined;
+
+  try {
+    const endpoint = new URL(url);
+    return endpoint.protocol === 'http:' || endpoint.protocol === 'https:'
+      ? endpoint.origin
+      : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+/** Preserve request metadata while keeping unknown or sensitive URL fields out of results. */
+export function sanitizeRequestMetadata(
+  request: LanguageModelV4GenerateResult['request'],
+): LanguageModelV4GenerateResult['request'] {
+  if (request == null || !('url' in request)) return request;
+  const { url, ...rest } = request;
+  const origin = getRequestOrigin(url);
+  return { ...rest, ...(origin != null ? { url: origin } : {}) };
+}

package/src/telemetry/tracing-channel-publisher.ts

@@ -66,6 +66,7 @@ function loadBuiltinModule<T>(id: string): T | undefined {
 export async function runWithTracingChannelSpan<T>(
   message: TelemetryTracingChannelMessage,
   execute: () => PromiseLike<T>,
+  recordResult = true,
 ): Promise<T> {
   const diagnosticsChannel = await loadDiagnosticsChannel();
   const tracingChannel = diagnosticsChannel?.tracingChannel?.(
@@ -102,11 +103,21 @@ export async function runWithTracingChannelSpan<T>(
       },
     );
 
-    return executePromise;
+    // tracePromise publishes its resolved value on the channel. Keep the
+    // application's result separately when content recording is disabled.
+    return recordResult
+      ? executePromise
+      : executePromise.then(
+          () => undefined,
+          () => {
+            throw new Error('AI SDK operation failed');
+          },
+        );
   };
 
   try {
-    return await tracingChannel.tracePromise(tracedExecute, message);
+    await tracingChannel.tracePromise(tracedExecute, message);
+    return executionResult as T;
   } catch {
     if (hasExecutionError) {
       throw executionError;
@@ -131,9 +142,11 @@ export async function runWithTracingChannelSpan<T>(
 export function openTelemetryChannelSpanContext({
   message,
   completion,
+  recordResult = true,
 }: {
   message: TelemetryTracingChannelMessage;
   completion: PromiseLike<unknown>;
+  recordResult?: boolean;
 }): TracingChannelContext | undefined {
   if (!isNodeRuntime()) {
     Promise.resolve(completion).catch(() => {});
@@ -186,11 +199,13 @@ export function openTelemetryChannelSpanContext({
     asyncEndPublished = true;
 
     if (error !== undefined) {
-      context.error = error;
+      context.error = recordResult
+        ? error
+        : new Error('AI SDK operation failed');
       safePublish(() => tracingChannel.error.publish(context));
     }
 
-    if (result !== undefined) {
+    if (recordResult && result !== undefined) {
       context.result = result;
     }
 

package/src/test/mock-transcription-model-v4.ts

@@ -8,21 +8,25 @@ export class MockTranscriptionModelV4 implements TranscriptionModelV4 {
 
   doGenerate: TranscriptionModelV4['doGenerate'];
   doStream: TranscriptionModelV4['doStream'];
+  doGenerateFromUrl: TranscriptionModelV4['doGenerateFromUrl'];
 
   constructor({
     provider = 'mock-provider',
     modelId = 'mock-model-id',
     doGenerate = notImplemented,
     doStream,
+    doGenerateFromUrl,
   }: {
     provider?: TranscriptionModelV4['provider'];
     modelId?: TranscriptionModelV4['modelId'];
     doGenerate?: TranscriptionModelV4['doGenerate'];
     doStream?: TranscriptionModelV4['doStream'];
+    doGenerateFromUrl?: TranscriptionModelV4['doGenerateFromUrl'];
   } = {}) {
     this.provider = provider;
     this.modelId = modelId;
     this.doGenerate = doGenerate;
     this.doStream = doStream;
+    this.doGenerateFromUrl = doGenerateFromUrl;
   }
 }

package/src/transcribe/transcribe-result.ts

@@ -15,6 +15,7 @@ export interface TranscriptionResult {
   /**
    * Array of transcript segments with timing information.
    * Each segment represents a portion of the transcribed text with start and end times.
+   * Segments retain the provider's behavior and may represent individual words.
    */
   readonly segments: Array<{
     /**
@@ -31,6 +32,25 @@ export interface TranscriptionResult {
     readonly endSecond: number;
   }>;
 
+  /**
+   * Word-level timings returned by the provider, whether explicitly requested
+   * or returned by default. Undefined when the provider does not return them.
+   */
+  readonly words?: Array<{
+    /**
+     * The text of this word.
+     */
+    readonly text: string;
+    /**
+     * The start time of this word in seconds.
+     */
+    readonly startSecond: number;
+    /**
+     * The end time of this word in seconds.
+     */
+    readonly endSecond: number;
+  }>;
+
   /**
    * The detected language of the audio content, as an ISO-639-1 code (e.g., 'en' for English).
    * May be undefined if the language couldn't be detected.

package/src/transcribe/transcribe.ts

@@ -1,7 +1,12 @@
-import type { JSONObject } from '@ai-sdk/provider';
+import type {
+  JSONObject,
+  TranscriptionModelV4TimestampGranularity,
+} from '@ai-sdk/provider';
 import {
   createIdGenerator,
   detectMediaType,
+  getTopLevelMediaType,
+  isFullMediaType,
   withUserAgentSuffix,
   type ProviderOptions,
 } from '@ai-sdk/provider-utils';
@@ -49,6 +54,7 @@ export async function transcribe({
   model,
   audio,
   providerOptions = {},
+  timestampGranularities,
   maxRetries: maxRetriesArg,
   abortSignal,
   headers,
@@ -82,6 +88,16 @@ export async function transcribe({
    */
   providerOptions?: ProviderOptions;
 
+  /**
+   * Word and segment timings to request.
+   *
+   * Supported adapters map this onto the provider's native control when the
+   * matching provider option is omitted. An explicit provider option wins.
+   * Supported adapters warn when they cannot return a requested granularity.
+   * An empty array requests no shared timing granularity; provider defaults may vary.
+   */
+  timestampGranularities?: Array<TranscriptionModelV4TimestampGranularity>;
+
   /**
    * Maximum number of retries per transcript model call. Set to 0 to disable retries.
    *
@@ -107,6 +123,8 @@ export async function transcribe({
 
   /**
    * Custom download function for fetching audio from URLs.
+   * Providing this option forces local downloading even when the provider
+   * supports native URL transcription.
    * Use `createDownload()` from `ai` to create a download function with custom size limits.
    *
    * @default createDownload() (2 GiB limit)
@@ -164,6 +182,7 @@ export async function transcribe({
     maxRetries,
     headers,
     providerOptions,
+    timestampGranularities,
   });
 
   const executeWithStartEvent = async <T>({
@@ -188,14 +207,22 @@ export async function transcribe({
 
   let preparationError: unknown;
   let preparationFailed = false;
-  let audioData: Uint8Array;
+  let audioData: Uint8Array | URL;
   let downloadedMediaType: string | undefined;
 
   try {
     if (audio instanceof URL) {
-      const downloadResult = await downloadFn({ url: audio, abortSignal });
-      audioData = downloadResult.data;
-      downloadedMediaType = downloadResult.mediaType;
+      if (
+        resolvedModel.doGenerateFromUrl != null &&
+        downloadFn === defaultDownload &&
+        (audio.protocol === 'https:' || audio.protocol === 'http:')
+      ) {
+        audioData = audio;
+      } else {
+        const downloadResult = await downloadFn({ url: audio, abortSignal });
+        audioData = downloadResult.data;
+        downloadedMediaType = downloadResult.mediaType;
+      }
     } else {
       audioData = convertDataContentToUint8Array(audio);
     }
@@ -224,15 +251,18 @@ export async function transcribe({
     }
   }
 
-  const mediaType =
-    downloadedMediaType ??
-    detectMediaType({
-      data: audioData,
-      topLevelType: 'audio',
-    }) ??
-    'audio/wav';
+  let mediaType =
+    audioData instanceof URL
+      ? undefined
+      : (getAudioMediaType(downloadedMediaType) ??
+        detectMediaType({
+          data: audioData,
+          topLevelType: 'audio',
+        }) ??
+        'audio/wav');
+  let byteLength = audioData instanceof URL ? undefined : audioData.byteLength;
   const startEvent = createStartEvent({
-    byteLength: audioData.byteLength,
+    byteLength,
     mediaType,
   });
 
@@ -240,15 +270,41 @@ export async function transcribe({
     return await executeWithStartEvent({
       startEvent,
       execute: async () => {
-        const result = await retry(() =>
-          resolvedModel.doGenerate({
-            audio: audioData,
+        const result = await retry(async () => {
+          const callOptions = {
             abortSignal,
             headers: headersWithUserAgent,
             providerOptions,
-            mediaType,
-          }),
-        );
+            ...(timestampGranularities != null
+              ? { timestampGranularities }
+              : {}),
+          };
+          if (audioData instanceof URL) {
+            const nativeResult = await resolvedModel.doGenerateFromUrl?.({
+              ...callOptions,
+              audio: audioData,
+            });
+            if (nativeResult != null) return nativeResult;
+
+            // A capable provider can decline an unsupported URL (for example,
+            // a redirect). Keep the existing secure download and file path.
+            const downloaded = await downloadFn({
+              url: audioData,
+              abortSignal,
+            });
+            audioData = downloaded.data;
+            mediaType =
+              getAudioMediaType(downloaded.mediaType) ??
+              detectMediaType({ data: audioData, topLevelType: 'audio' }) ??
+              'audio/wav';
+            byteLength = audioData.byteLength;
+          }
+          return resolvedModel.doGenerate({
+            ...callOptions,
+            audio: audioData,
+            mediaType: mediaType ?? 'audio/wav',
+          });
+        });
 
         logWarnings({
           warnings: result.warnings,
@@ -268,11 +324,12 @@ export async function transcribe({
           provider: resolvedModel.provider,
           modelId: resolvedModel.modelId,
           audio: {
-            byteLength: audioData.byteLength,
+            byteLength,
             mediaType,
           },
           text: result.text,
           segments: result.segments,
+          words: result.words,
           language: result.language,
           durationInSeconds: result.durationInSeconds,
           usage: result.usage,
@@ -289,6 +346,7 @@ export async function transcribe({
         return new DefaultTranscriptionResult({
           text: result.text,
           segments: result.segments,
+          words: result.words,
           language: result.language,
           durationInSeconds: result.durationInSeconds,
           warnings: result.warnings,
@@ -303,6 +361,24 @@ export async function transcribe({
   }
 }
 
+/**
+ * Normalizes a downloaded `Content-Type` header value and returns it only
+ * when it is a full `audio/<subtype>` media type. Generic or mislabeled types
+ * (e.g. `application/octet-stream`, `binary/octet-stream`, `text/plain`)
+ * return `undefined` so that the media type is detected from the audio bytes
+ * instead.
+ */
+function getAudioMediaType(
+  contentType: string | undefined,
+): string | undefined {
+  const mediaType = contentType?.split(';')[0].trim().toLowerCase();
+  return mediaType != null &&
+    getTopLevelMediaType(mediaType) === 'audio' &&
+    isFullMediaType(mediaType)
+    ? mediaType
+    : undefined;
+}
+
 class DefaultTranscriptionResult implements TranscriptionResult {
   readonly text: string;
   readonly segments: Array<{
@@ -310,6 +386,11 @@ class DefaultTranscriptionResult implements TranscriptionResult {
     startSecond: number;
     endSecond: number;
   }>;
+  readonly words?: Array<{
+    text: string;
+    startSecond: number;
+    endSecond: number;
+  }>;
   readonly language: string | undefined;
   readonly durationInSeconds: number | undefined;
   readonly warnings: Array<Warning>;
@@ -323,6 +404,11 @@ class DefaultTranscriptionResult implements TranscriptionResult {
       startSecond: number;
       endSecond: number;
     }>;
+    words?: Array<{
+      text: string;
+      startSecond: number;
+      endSecond: number;
+    }>;
     language: string | undefined;
     durationInSeconds: number | undefined;
     warnings: Array<Warning>;
@@ -331,6 +417,7 @@ class DefaultTranscriptionResult implements TranscriptionResult {
   }) {
     this.text = options.text;
     this.segments = options.segments;
+    this.words = options.words;
     this.language = options.language;
     this.durationInSeconds = options.durationInSeconds;
     this.warnings = options.warnings;

package/src/transcribe/transcription-events.ts

@@ -1,4 +1,8 @@
-import type { JSONObject, SharedV4AudioFormat } from '@ai-sdk/provider';
+import type {
+  JSONObject,
+  SharedV4AudioFormat,
+  TranscriptionModelV4TimestampGranularity,
+} from '@ai-sdk/provider';
 import type { ProviderOptions } from '@ai-sdk/provider-utils';
 import type { ProviderMetadata } from '../types';
 import type { TranscriptionModelResponseMetadata } from '../types/transcription-model-response-metadata';
@@ -21,8 +25,9 @@ type TranscriptionStartEventFor<OPERATION_ID extends string> = {
   readonly modelId: string;
 
   /**
-   * Metadata about the input audio. The byte length is known immediately for
-   * non-streaming transcription and is reported on the end event for streams.
+   * Metadata about the input audio. Native URL transcription does not inspect
+   * the audio, so its byte length and media type are unknown. Streaming byte
+   * length is reported on the end event after consuming the stream.
    */
   readonly audio: {
     readonly byteLength: number | undefined;
@@ -47,7 +52,10 @@ type TranscriptionStartEventFor<OPERATION_ID extends string> = {
 };
 
 export type TranscriptionStartEvent =
-  TranscriptionStartEventFor<'ai.transcribe'>;
+  TranscriptionStartEventFor<'ai.transcribe'> & {
+    /** Requested shared timing granularities, when specified. */
+    readonly timestampGranularities?: Array<TranscriptionModelV4TimestampGranularity>;
+  };
 
 export type StreamTranscriptionStartEvent =
   TranscriptionStartEventFor<'ai.streamTranscribe'>;
@@ -72,14 +80,18 @@ type TranscriptionEndEventFor<OPERATION_ID extends string> = {
 
   /** Metadata about the input audio. */
   readonly audio: {
-    readonly byteLength: number;
-    readonly mediaType: string;
+    readonly byteLength: OPERATION_ID extends 'ai.transcribe'
+      ? number | undefined
+      : number;
+    readonly mediaType: OPERATION_ID extends 'ai.transcribe'
+      ? string | undefined
+      : string;
   };
 
   /** The complete transcript. */
   readonly text: string;
 
-  /** Transcript segments with timing information. */
+  /** Transcript segments with timing information, retaining provider behavior. */
   readonly segments: Array<{
     readonly text: string;
     readonly startSecond: number;
@@ -105,7 +117,15 @@ type TranscriptionEndEventFor<OPERATION_ID extends string> = {
   readonly response: TranscriptionModelResponseMetadata;
 };
 
-export type TranscriptionEndEvent = TranscriptionEndEventFor<'ai.transcribe'>;
+export type TranscriptionEndEvent =
+  TranscriptionEndEventFor<'ai.transcribe'> & {
+    /** Word timings returned by the provider, whether requested or returned by default. */
+    readonly words?: Array<{
+      readonly text: string;
+      readonly startSecond: number;
+      readonly endSecond: number;
+    }>;
+  };
 
 export type StreamTranscriptionEndEvent =
   TranscriptionEndEventFor<'ai.streamTranscribe'>;

package/src/types/language-model-request-metadata.ts

@@ -4,6 +4,9 @@ import type { ModelMessage } from '@ai-sdk/provider-utils';
  * Metadata for a language model request.
  */
 export type LanguageModelRequestMetadata = {
+  /** HTTP origin used for this request, without URL credentials, paths, queries or fragments. */
+  readonly url?: string;
+
   /**
    * The input messages that were sent to the model for this step.
    */

package/src/types/realtime-model.ts

@@ -4,9 +4,12 @@ import type {
   Experimental_RealtimeFactoryV4GetTokenResult as RealtimeFactoryV4GetTokenResult,
   Experimental_RealtimeModelV4 as RealtimeModelV4,
   Experimental_RealtimeModelV4ClientEvent as RealtimeModelV4ClientEvent,
+  Experimental_RealtimeModelV4ServerConnection as RealtimeModelV4ServerConnection,
   Experimental_RealtimeModelV4ServerEvent as RealtimeModelV4ServerEvent,
   Experimental_RealtimeModelV4SessionConfig as RealtimeModelV4SessionConfig,
+  Experimental_RealtimeModelV4SessionIntent as RealtimeModelV4SessionIntent,
   Experimental_RealtimeModelV4ToolDefinition as RealtimeModelV4ToolDefinition,
+  Experimental_RealtimeModelV4Usage as RealtimeModelV4Usage,
 } from '@ai-sdk/provider';
 
 export type RealtimeFactory = RealtimeFactoryV4;
@@ -19,8 +22,14 @@ export type RealtimeModel = RealtimeModelV4;
 
 export type RealtimeClientEvent = RealtimeModelV4ClientEvent;
 
+export type RealtimeServerConnection = RealtimeModelV4ServerConnection;
+
 export type RealtimeServerEvent = RealtimeModelV4ServerEvent;
 
 export type RealtimeSessionConfig = RealtimeModelV4SessionConfig;
 
+export type RealtimeSessionIntent = RealtimeModelV4SessionIntent;
+
 export type RealtimeToolDefinition = RealtimeModelV4ToolDefinition;
+
+export type RealtimeUsage = RealtimeModelV4Usage;