ai 7.0.122 → 7.0.124

package/src/transcribe/transcribe.ts

@@ -1,21 +1,34 @@
 import type { JSONObject } from '@ai-sdk/provider';
 import {
+  createIdGenerator,
   detectMediaType,
   withUserAgentSuffix,
   type ProviderOptions,
 } from '@ai-sdk/provider-utils';
 import { NoTranscriptGeneratedError } from '../error/no-transcript-generated-error';
 import { logWarnings } from '../logger/log-warnings';
+import { resolveTranscriptionModel } from '../model/resolve-model';
 import type { DataContent } from '../prompt';
 import { convertDataContentToUint8Array } from '../prompt/data-content';
+import { createTelemetryDispatcher } from '../telemetry/create-telemetry-dispatcher';
+import type { TelemetryOptions } from '../telemetry/telemetry-options';
 import type { TranscriptionModel } from '../types/transcription-model';
 import type { TranscriptionModelResponseMetadata } from '../types/transcription-model-response-metadata';
 import { createDownload } from '../util/download/create-download';
+import { notify } from '../util/notify';
 import { prepareRetries } from '../util/prepare-retries';
 import type { TranscriptionResult } from './transcribe-result';
 import { VERSION } from '../version';
-import { resolveTranscriptionModel } from '../model/resolve-model';
 import type { Warning } from '../types';
+import type {
+  TranscriptionEndEvent,
+  TranscriptionStartEvent,
+} from './transcription-events';
+
+const originalGenerateCallId = createIdGenerator({
+  prefix: 'call',
+  size: 24,
+});
 /**
  * Generates transcripts using a transcription model.
  *
@@ -26,6 +39,7 @@ import type { Warning } from '../types';
  * @param maxRetries - Maximum number of retries. Set to 0 to disable retries. Default: 2.
  * @param abortSignal - An optional abort signal that can be used to cancel the call.
  * @param headers - Additional HTTP headers to be sent with the request. Only applicable for HTTP-based providers.
+ * @param telemetry - Optional telemetry configuration.
  *
  * @returns A result object that contains the generated transcript.
  */
@@ -38,7 +52,9 @@ export async function transcribe({
   maxRetries: maxRetriesArg,
   abortSignal,
   headers,
+  telemetry,
   download: downloadFn = defaultDownload,
+  _internal: { generateCallId = originalGenerateCallId } = {},
 }: {
   /**
    * The transcription model to use.
@@ -84,6 +100,11 @@ export async function transcribe({
    */
   headers?: Record<string, string>;
 
+  /**
+   * Optional telemetry configuration.
+   */
+  telemetry?: TelemetryOptions;
+
   /**
    * Custom download function for fetching audio from URLs.
    * Use `createDownload()` from `ai` to create a download function with custom size limits.
@@ -94,13 +115,20 @@ export async function transcribe({
     url: URL;
     abortSignal?: AbortSignal;
   }) => Promise<{ data: Uint8Array; mediaType: string | undefined }>;
+
+  /**
+   * Internal. For test use only. May change without notice.
+   */
+  _internal?: {
+    generateCallId?: () => string;
+  };
 }): Promise<TranscriptionResult> {
   const resolvedModel = resolveTranscriptionModel(model);
   if (!resolvedModel) {
     throw new Error('Model could not be resolved');
   }
 
-  const { retry } = prepareRetries({
+  const { maxRetries, retry } = prepareRetries({
     maxRetries: maxRetriesArg,
     abortSignal,
   });
@@ -110,44 +138,169 @@ export async function transcribe({
     `ai/${VERSION}`,
   );
 
-  const audioData =
-    audio instanceof URL
-      ? (await downloadFn({ url: audio, abortSignal })).data
-      : convertDataContentToUint8Array(audio);
-
-  const result = await retry(() =>
-    resolvedModel.doGenerate({
-      audio: audioData,
-      abortSignal,
-      headers: headersWithUserAgent,
-      providerOptions,
-      mediaType:
-        detectMediaType({
-          data: audioData,
-          topLevelType: 'audio',
-        }) ?? 'audio/wav',
-    }),
-  );
+  const callId = generateCallId();
+  const telemetryDispatcher = createTelemetryDispatcher({ telemetry });
+  const runInTracingChannelSpan =
+    telemetryDispatcher.runInTracingChannelSpan ??
+    (async <T>({ execute }: { execute: () => PromiseLike<T> }) =>
+      await execute());
 
-  logWarnings({
-    warnings: result.warnings,
+  const createStartEvent = ({
+    byteLength,
+    mediaType,
+  }: {
+    byteLength: number | undefined;
+    mediaType: string | undefined;
+  }): TranscriptionStartEvent => ({
+    callId,
+    operationId: 'ai.transcribe',
     provider: resolvedModel.provider,
-    model: resolvedModel.modelId,
+    modelId: resolvedModel.modelId,
+    audio: {
+      byteLength,
+      mediaType,
+    },
+    inputAudioFormat: undefined,
+    maxRetries,
+    headers,
+    providerOptions,
   });
 
-  if (!result.text) {
-    throw new NoTranscriptGeneratedError({ responses: [result.response] });
+  const executeWithStartEvent = async <T>({
+    startEvent,
+    execute,
+  }: {
+    startEvent: TranscriptionStartEvent;
+    execute: () => PromiseLike<T>;
+  }) =>
+    await runInTracingChannelSpan({
+      type: 'transcribe',
+      event: startEvent,
+      execute: async () => {
+        await notify({
+          event: startEvent,
+          callbacks: [telemetryDispatcher.onStart],
+        });
+
+        return await execute();
+      },
+    });
+
+  let preparationError: unknown;
+  let preparationFailed = false;
+  let audioData: Uint8Array;
+  let downloadedMediaType: string | undefined;
+
+  try {
+    if (audio instanceof URL) {
+      const downloadResult = await downloadFn({ url: audio, abortSignal });
+      audioData = downloadResult.data;
+      downloadedMediaType = downloadResult.mediaType;
+    } else {
+      audioData = convertDataContentToUint8Array(audio);
+    }
+  } catch (error) {
+    preparationFailed = true;
+    preparationError = error;
+    audioData = new Uint8Array();
+  }
+
+  if (preparationFailed) {
+    const startEvent = createStartEvent({
+      byteLength: undefined,
+      mediaType: undefined,
+    });
+
+    try {
+      return await executeWithStartEvent({
+        startEvent,
+        execute: () => {
+          throw preparationError;
+        },
+      });
+    } catch (error) {
+      await telemetryDispatcher.onError?.({ callId, error });
+      throw error;
+    }
   }
 
-  return new DefaultTranscriptionResult({
-    text: result.text,
-    segments: result.segments,
-    language: result.language,
-    durationInSeconds: result.durationInSeconds,
-    warnings: result.warnings,
-    responses: [result.response],
-    providerMetadata: result.providerMetadata,
+  const mediaType =
+    downloadedMediaType ??
+    detectMediaType({
+      data: audioData,
+      topLevelType: 'audio',
+    }) ??
+    'audio/wav';
+  const startEvent = createStartEvent({
+    byteLength: audioData.byteLength,
+    mediaType,
   });
+
+  try {
+    return await executeWithStartEvent({
+      startEvent,
+      execute: async () => {
+        const result = await retry(() =>
+          resolvedModel.doGenerate({
+            audio: audioData,
+            abortSignal,
+            headers: headersWithUserAgent,
+            providerOptions,
+            mediaType,
+          }),
+        );
+
+        logWarnings({
+          warnings: result.warnings,
+          provider: resolvedModel.provider,
+          model: resolvedModel.modelId,
+        });
+
+        if (!result.text) {
+          throw new NoTranscriptGeneratedError({
+            responses: [result.response],
+          });
+        }
+
+        const endEvent: TranscriptionEndEvent = {
+          callId,
+          operationId: 'ai.transcribe',
+          provider: resolvedModel.provider,
+          modelId: resolvedModel.modelId,
+          audio: {
+            byteLength: audioData.byteLength,
+            mediaType,
+          },
+          text: result.text,
+          segments: result.segments,
+          language: result.language,
+          durationInSeconds: result.durationInSeconds,
+          usage: result.usage,
+          warnings: result.warnings,
+          providerMetadata: result.providerMetadata,
+          response: result.response,
+        };
+
+        await notify({
+          event: endEvent,
+          callbacks: [telemetryDispatcher.onEnd],
+        });
+
+        return new DefaultTranscriptionResult({
+          text: result.text,
+          segments: result.segments,
+          language: result.language,
+          durationInSeconds: result.durationInSeconds,
+          warnings: result.warnings,
+          responses: [result.response],
+          providerMetadata: result.providerMetadata,
+        });
+      },
+    });
+  } catch (error) {
+    await telemetryDispatcher.onError?.({ callId, error });
+    throw error;
+  }
 }
 
 class DefaultTranscriptionResult implements TranscriptionResult {

package/src/transcribe/transcription-events.ts

@@ -0,0 +1,111 @@
+import type { JSONObject, SharedV4AudioFormat } from '@ai-sdk/provider';
+import type { ProviderOptions } from '@ai-sdk/provider-utils';
+import type { ProviderMetadata } from '../types';
+import type { TranscriptionModelResponseMetadata } from '../types/transcription-model-response-metadata';
+import type { Warning } from '../types/warning';
+
+/**
+ * Event passed to telemetry integrations when transcription begins.
+ */
+type TranscriptionStartEventFor<OPERATION_ID extends string> = {
+  /** Unique identifier for this transcription call. */
+  readonly callId: string;
+
+  /** Identifies the operation type (`ai.transcribe` or `ai.streamTranscribe`). */
+  readonly operationId: OPERATION_ID;
+
+  /** The provider identifier. */
+  readonly provider: string;
+
+  /** The specific model identifier. */
+  readonly modelId: string;
+
+  /**
+   * Metadata about the input audio. The byte length is known immediately for
+   * non-streaming transcription and is reported on the end event for streams.
+   */
+  readonly audio: {
+    readonly byteLength: number | undefined;
+    /**
+     * The media type, when it is known before input preparation completes.
+     * URL downloads and invalid inline inputs can begin without this metadata.
+     */
+    readonly mediaType: string | undefined;
+  };
+
+  /** The raw input format for streaming transcription. */
+  readonly inputAudioFormat: SharedV4AudioFormat | undefined;
+
+  /** Maximum number of retries for non-streaming transcription. */
+  readonly maxRetries: number | undefined;
+
+  /** Additional HTTP or WebSocket headers sent with the request. */
+  readonly headers: Record<string, string> | undefined;
+
+  /** Additional provider-specific options. */
+  readonly providerOptions: ProviderOptions;
+};
+
+export type TranscriptionStartEvent =
+  TranscriptionStartEventFor<'ai.transcribe'>;
+
+export type StreamTranscriptionStartEvent =
+  TranscriptionStartEventFor<'ai.streamTranscribe'>;
+
+/**
+ * Event passed to telemetry integrations when transcription completes.
+ *
+ * Raw audio is intentionally not included.
+ */
+type TranscriptionEndEventFor<OPERATION_ID extends string> = {
+  /** Unique identifier for this transcription call. */
+  readonly callId: string;
+
+  /** Identifies the operation type (`ai.transcribe` or `ai.streamTranscribe`). */
+  readonly operationId: OPERATION_ID;
+
+  /** The provider identifier. */
+  readonly provider: string;
+
+  /** The specific model identifier. */
+  readonly modelId: string;
+
+  /** Metadata about the input audio. */
+  readonly audio: {
+    readonly byteLength: number;
+    readonly mediaType: string;
+  };
+
+  /** The complete transcript. */
+  readonly text: string;
+
+  /** Transcript segments with timing information. */
+  readonly segments: Array<{
+    readonly text: string;
+    readonly startSecond: number;
+    readonly endSecond: number;
+  }>;
+
+  /** The detected language. */
+  readonly language: string | undefined;
+
+  /** The duration of the input audio, when available. */
+  readonly durationInSeconds: number | undefined;
+
+  /** Usage reported by the provider, when available. */
+  readonly usage: JSONObject | undefined;
+
+  /** Warnings from the transcription model. */
+  readonly warnings: Array<Warning>;
+
+  /** Optional provider-specific metadata. */
+  readonly providerMetadata: ProviderMetadata | undefined;
+
+  /** Response metadata from the provider. */
+  readonly response: TranscriptionModelResponseMetadata;
+};
+
+export type TranscriptionEndEvent = TranscriptionEndEventFor<'ai.transcribe'>;
+
+export type StreamTranscriptionEndEvent =
+  TranscriptionEndEventFor<'ai.streamTranscribe'>;

package/src/ui/convert-to-model-messages.ts

@@ -59,16 +59,28 @@ export async function convertToModelMessages<UI_MESSAGE extends UIMessage>(
 
   warnIfUIMessageHasDeprecatedRawInput(messages);
 
-  if (options?.ignoreIncompleteToolCalls) {
-    messages = messages.map(message => ({
+  // A later user message supersedes unresolved approval requests. Keeping
+  // those requests would leave unmatched tool calls at the user boundary.
+  const lastUserMessageIndex = messages.reduce(
+    (lastIndex, message, index) =>
+      message.role === 'user' ? index : lastIndex,
+    -1,
+  );
+
+  if (options?.ignoreIncompleteToolCalls || lastUserMessageIndex > 0) {
+    messages = messages.map((message, messageIndex) => ({
       ...message,
       parts: message.parts.filter(
         part =>
           !isToolUIPart(part) ||
-          part.state === 'approval-responded' ||
-          (part.state === 'output-available' && part.preliminary !== true) ||
-          part.state === 'output-error' ||
-          part.state === 'output-denied',
+          ((part.state !== 'approval-requested' ||
+            messageIndex >= lastUserMessageIndex) &&
+            (!options?.ignoreIncompleteToolCalls ||
+              part.state === 'approval-responded' ||
+              (part.state === 'output-available' &&
+                part.preliminary !== true) ||
+              part.state === 'output-error' ||
+              part.state === 'output-denied')),
       ),
     }));
   }

package/src/ui-message-stream/create-sse-stream-with-keep-alive.ts

@@ -0,0 +1,89 @@
+const STREAM_OPEN_COMMENT = ': stream-open\n\n';
+const KEEP_ALIVE_COMMENT = ': keep-alive\n\n';
+
+export function createSseStreamWithKeepAlive({
+  stream,
+  keepAliveMs,
+}: {
+  stream: ReadableStream<string>;
+  keepAliveMs: number | undefined;
+}): ReadableStream<string> {
+  if (keepAliveMs == null) {
+    return stream;
+  }
+
+  if (
+    !Number.isFinite(keepAliveMs) ||
+    keepAliveMs <= 0 ||
+    keepAliveMs > 2_147_483_647
+  ) {
+    throw new Error(
+      'keepAliveMs must be a positive finite timer duration no greater than 2147483647',
+    );
+  }
+
+  const reader = stream.getReader();
+  let keepAliveTimeout: ReturnType<typeof setTimeout> | undefined;
+  let isCancelled = false;
+
+  const clearKeepAliveTimeout = () => {
+    clearTimeout(keepAliveTimeout);
+    keepAliveTimeout = undefined;
+  };
+
+  const scheduleKeepAlive = (
+    controller: ReadableStreamDefaultController<string>,
+  ) => {
+    clearKeepAliveTimeout();
+    keepAliveTimeout = setTimeout(() => {
+      if (isCancelled) {
+        return;
+      }
+
+      if (controller.desiredSize != null && controller.desiredSize > 0) {
+        controller.enqueue(KEEP_ALIVE_COMMENT);
+      }
+
+      scheduleKeepAlive(controller);
+    }, keepAliveMs);
+  };
+
+  return new ReadableStream<string>({
+    start(controller) {
+      controller.enqueue(STREAM_OPEN_COMMENT);
+      scheduleKeepAlive(controller);
+    },
+
+    pull(controller) {
+      return reader.read().then(
+        result => {
+          clearKeepAliveTimeout();
+
+          if (isCancelled) {
+            return;
+          }
+
+          if (result.done) {
+            controller.close();
+          } else {
+            controller.enqueue(result.value);
+            scheduleKeepAlive(controller);
+          }
+        },
+        error => {
+          clearKeepAliveTimeout();
+
+          if (!isCancelled) {
+            throw error;
+          }
+        },
+      );
+    },
+
+    async cancel(reason) {
+      isCancelled = true;
+      clearKeepAliveTimeout();
+      await reader.cancel(reason);
+    },
+  });
+}

package/src/ui-message-stream/create-ui-message-stream-response.ts

@@ -1,4 +1,5 @@
 import { prepareHeaders } from '../util/prepare-headers';
+import { createSseStreamWithKeepAlive } from './create-sse-stream-with-keep-alive';
 import { JsonToSseTransformStream } from './json-to-sse-transform-stream';
 import { UI_MESSAGE_STREAM_HEADERS } from './ui-message-stream-headers';
 import type { UIMessageChunk } from './ui-message-chunks';
@@ -12,6 +13,7 @@ import type { UIMessageStreamResponseInit } from './ui-message-stream-response-i
  * @param options.statusText - The HTTP status text for the response.
  * @param options.headers - Additional HTTP headers to include in the response.
  * @param options.stream - The UI message chunk stream to send.
+ * @param options.keepAliveMs - Optional interval for sending SSE keep-alive comments.
  * @param options.consumeSseStream - Optional callback to consume a copy of the SSE stream independently.
  *
  * @returns A `Response` object with the UI message stream as the body.
@@ -21,11 +23,15 @@ export function createUIMessageStreamResponse({
   statusText,
   headers,
   stream,
+  keepAliveMs,
   consumeSseStream,
 }: UIMessageStreamResponseInit & {
   stream: ReadableStream<UIMessageChunk>;
 }): Response {
-  let sseStream = stream.pipeThrough(new JsonToSseTransformStream());
+  let sseStream = createSseStreamWithKeepAlive({
+    stream: stream.pipeThrough(new JsonToSseTransformStream()),
+    keepAliveMs,
+  });
 
   // when the consumeSseStream is provided, we need to tee the stream
   // and send the second part to the consumeSseStream function

package/src/ui-message-stream/pipe-ui-message-stream-to-response.ts

@@ -1,6 +1,7 @@
 import type { ServerResponse } from 'node:http';
 import { prepareHeaders } from '../util/prepare-headers';
 import { writeToServerResponse } from '../util/write-to-server-response';
+import { createSseStreamWithKeepAlive } from './create-sse-stream-with-keep-alive';
 import { JsonToSseTransformStream } from './json-to-sse-transform-stream';
 import { UI_MESSAGE_STREAM_HEADERS } from './ui-message-stream-headers';
 import type { UIMessageChunk } from './ui-message-chunks';
@@ -15,6 +16,7 @@ import type { UIMessageStreamResponseInit } from './ui-message-stream-response-i
  * @param options.statusText - The HTTP status text for the response.
  * @param options.headers - Additional HTTP headers to include in the response.
  * @param options.stream - The UI message chunk stream to send.
+ * @param options.keepAliveMs - Optional interval for sending SSE keep-alive comments.
  * @param options.consumeSseStream - Optional callback to consume a copy of the SSE stream independently.
  * @returns A promise that resolves when the stream has been written.
  */
@@ -24,12 +26,16 @@ export function pipeUIMessageStreamToResponse({
   statusText,
   headers,
   stream,
+  keepAliveMs,
   consumeSseStream,
 }: {
   response: ServerResponse;
   stream: ReadableStream<UIMessageChunk>;
 } & UIMessageStreamResponseInit): Promise<void> {
-  let sseStream = stream.pipeThrough(new JsonToSseTransformStream());
+  let sseStream = createSseStreamWithKeepAlive({
+    stream: stream.pipeThrough(new JsonToSseTransformStream()),
+    keepAliveMs,
+  });
 
   // when the consumeSseStream is provided, we need to tee the stream
   // and send the second part to the consumeSseStream function

package/src/ui-message-stream/ui-message-stream-response-init.ts

@@ -3,6 +3,13 @@
  * Extends the standard `ResponseInit` with additional streaming options.
  */
 export type UIMessageStreamResponseInit = ResponseInit & {
+  /**
+   * Optional interval in milliseconds for sending SSE keep-alive comments.
+   * When set, an opening comment is sent immediately and additional comments
+   * are sent after the stream has been idle for the configured interval.
+   */
+  keepAliveMs?: number;
+
   /**
    * Optional callback to consume a copy of the SSE stream independently.
    * This is useful for logging, debugging, or processing the stream in parallel.