ai 7.0.123 → 7.0.126

package/src/transcribe/stream-transcribe.ts

@@ -5,6 +5,7 @@ import {
   type SharedV4AudioFormat,
 } from '@ai-sdk/provider';
 import {
+  createIdGenerator,
   DelayedPromise,
   withUserAgentSuffix,
   type ProviderOptions,
@@ -12,16 +13,23 @@ import {
 import { NoTranscriptGeneratedError } from '../error/no-transcript-generated-error';
 import { logWarnings } from '../logger/log-warnings';
 import { resolveTranscriptionModel } from '../model/resolve-model';
+import { createTelemetryDispatcher } from '../telemetry/create-telemetry-dispatcher';
+import type { TelemetryOptions } from '../telemetry/telemetry-options';
 import type { TranscriptionModel } from '../types/transcription-model';
 import type { TranscriptionModelResponseMetadata } from '../types/transcription-model-response-metadata';
 import type { Warning } from '../types/warning';
 import { asAsyncIterableStream } from '../util/async-iterable-stream';
 import { mergeAbortSignals } from '../util/merge-abort-signals';
+import { notify } from '../util/notify';
 import { VERSION } from '../version';
 import type {
   StreamTranscriptionResult,
   TranscriptionStreamPart,
 } from './stream-transcribe-result';
+import type {
+  StreamTranscriptionEndEvent,
+  StreamTranscriptionStartEvent,
+} from './transcription-events';
 
 type TranscriptSegment = {
   text: string;
@@ -29,6 +37,11 @@ type TranscriptSegment = {
   endSecond: number;
 };
 
+const originalGenerateCallId = createIdGenerator({
+  prefix: 'call',
+  size: 24,
+});
+
 /**
  * Streams transcripts using a transcription model.
  *
@@ -38,6 +51,7 @@ type TranscriptSegment = {
  * @param providerOptions - Additional provider-specific options.
  * @param abortSignal - An optional abort signal that can be used to cancel the call.
  * @param headers - Additional HTTP/WebSocket headers to send when supported by the provider.
+ * @param telemetry - Optional telemetry configuration.
  *
  * @returns A result object that contains the streaming transcript and final transcript metadata.
  */
@@ -49,7 +63,11 @@ export function streamTranscribe({
   abortSignal,
   headers,
   includeRawChunks,
-  _internal: { currentDate = () => new Date() } = {},
+  telemetry,
+  _internal: {
+    currentDate = () => new Date(),
+    generateCallId = originalGenerateCallId,
+  } = {},
 }: {
   /**
    * The transcription model to use.
@@ -86,11 +104,17 @@ export function streamTranscribe({
    */
   includeRawChunks?: boolean;
 
+  /**
+   * Optional telemetry configuration.
+   */
+  telemetry?: TelemetryOptions;
+
   /**
    * Internal test hooks.
    */
   _internal?: {
     currentDate?: () => Date;
+    generateCallId?: () => string;
   };
 }): StreamTranscriptionResult {
   const resolvedModel = resolveTranscriptionModel(model);
@@ -118,6 +142,34 @@ export function streamTranscribe({
     `ai/${VERSION}`,
   );
 
+  const callId = generateCallId();
+  const telemetryDispatcher = createTelemetryDispatcher({ telemetry });
+  const runInTracingChannelSpan =
+    telemetryDispatcher.runInTracingChannelSpan ??
+    (async <T>({ execute }: { execute: () => PromiseLike<T> }) =>
+      await execute());
+  let audioByteLength = 0;
+  const countedAudio =
+    telemetryDispatcher.experimental_onStreamTranscriptionEnd != null
+      ? createByteCountingStream(audio, byteLength => {
+          audioByteLength += byteLength;
+        })
+      : audio;
+  const startEvent: StreamTranscriptionStartEvent = {
+    callId,
+    operationId: 'ai.streamTranscribe',
+    provider: resolvedModel.provider,
+    modelId: resolvedModel.modelId,
+    audio: {
+      byteLength: undefined,
+      mediaType: inputAudioFormat.type,
+    },
+    inputAudioFormat,
+    maxRetries: undefined,
+    headers,
+    providerOptions,
+  };
+
   const textPromise = new DelayedPromise<string>();
   const segmentsPromise = new DelayedPromise<Array<TranscriptSegment>>();
   const languagePromise = new DelayedPromise<string | undefined>();
@@ -129,6 +181,8 @@ export function streamTranscribe({
   const providerMetadataPromise = new DelayedPromise<
     Record<string, JSONObject>
   >();
+  let warnings: Array<Warning> = [];
+  let telemetrySettled = false;
 
   const rejectPendingPromises = (error: unknown) => {
     for (const promise of [
@@ -151,10 +205,11 @@ export function streamTranscribe({
   const currentResponseMetadata = () =>
     response ?? { timestamp: startedAt, modelId: resolvedModel.modelId };
 
-  const resolveWarnings = (warnings: Array<Warning>) => {
-    warningsPromise.resolve(warnings);
+  const resolveWarnings = (resolvedWarnings: Array<Warning>) => {
+    warnings = resolvedWarnings;
+    warningsPromise.resolve(resolvedWarnings);
     logWarnings({
-      warnings,
+      warnings: resolvedWarnings,
       provider: resolvedModel.provider,
       model: resolvedModel.modelId,
     });
@@ -172,7 +227,7 @@ export function streamTranscribe({
     Experimental_TranscriptionModelV4StreamPart,
     TranscriptionStreamPart
   > & { cancel?: (reason?: unknown) => void } = {
-    transform(value, controller) {
+    async transform(value, controller) {
       switch (value.type) {
         case 'stream-start': {
           resolveWarnings(value.warnings);
@@ -214,6 +269,37 @@ export function streamTranscribe({
           durationInSecondsPromise.resolve(value.durationInSeconds);
           responsesPromise.resolve([currentResponseMetadata()]);
           providerMetadataPromise.resolve(value.providerMetadata ?? {});
+
+          telemetrySettled = true;
+          if (
+            telemetryDispatcher.experimental_onStreamTranscriptionEnd != null
+          ) {
+            const endEvent: StreamTranscriptionEndEvent = {
+              callId,
+              operationId: 'ai.streamTranscribe',
+              provider: resolvedModel.provider,
+              modelId: resolvedModel.modelId,
+              audio: {
+                byteLength: audioByteLength,
+                mediaType: inputAudioFormat.type,
+              },
+              text: value.text,
+              segments: value.segments,
+              language: value.language,
+              durationInSeconds: value.durationInSeconds,
+              usage: value.usage,
+              warnings,
+              providerMetadata: value.providerMetadata,
+              response: currentResponseMetadata(),
+            };
+
+            await notify({
+              event: endEvent,
+              callbacks: [
+                telemetryDispatcher.experimental_onStreamTranscriptionEnd,
+              ],
+            });
+          }
           break;
         }
       }
@@ -241,36 +327,50 @@ export function streamTranscribe({
 
   // Piping (instead of an eager read loop) preserves consumer backpressure
   // and propagates cancellation of `fullStream` to the model stream.
-  void (async () => {
-    const result = await doStream({
-      audio,
-      inputAudioFormat,
-      providerOptions,
-      // merged so cancelling fullStream also aborts a still-pending doStream
-      abortSignal: mergeAbortSignals(abortSignal, pipeAbortController.signal),
-      headers: headersWithUserAgent,
-      includeRawChunks,
-    });
-
-    response = {
-      timestamp: result.response?.timestamp ?? startedAt,
-      modelId: result.response?.modelId ?? resolvedModel.modelId,
-      headers: result.response?.headers,
-    };
-
-    await result.stream.pipeTo(transform.writable, {
-      signal: pipeAbortController.signal,
-    });
-  })().catch(error => {
+  void runInTracingChannelSpan({
+    type: 'streamTranscribe',
+    event: startEvent,
+    execute: async () => {
+      await notify({
+        event: startEvent,
+        callbacks: [
+          telemetryDispatcher.experimental_onStreamTranscriptionStart,
+        ],
+      });
+
+      const result = await doStream({
+        audio: countedAudio,
+        inputAudioFormat,
+        providerOptions,
+        // merged so cancelling fullStream also aborts a still-pending doStream
+        abortSignal: mergeAbortSignals(abortSignal, pipeAbortController.signal),
+        headers: headersWithUserAgent,
+        includeRawChunks,
+      });
+
+      response = {
+        timestamp: result.response?.timestamp ?? startedAt,
+        modelId: result.response?.modelId ?? resolvedModel.modelId,
+        headers: result.response?.headers,
+      };
+
+      await result.stream.pipeTo(transform.writable, {
+        signal: pipeAbortController.signal,
+      });
+    },
+  }).catch(async error => {
     const reason =
       error ?? new Error('Transcription stream was cancelled or errored.');
     rejectPendingPromises(reason);
+    if (!telemetrySettled) {
+      telemetrySettled = true;
+      await telemetryDispatcher.onError?.({ callId, error: reason });
+    }
     // When `doStream` rejects before the model stream exists (e.g. auth or
-    // header resolution failure), nothing has taken ownership of `audio` yet,
-    // so cancel it directly — otherwise an upstream producer piping into it
-    // hangs forever. When the model did take a reader, `audio` is locked and
-    // the cancel rejects, which is fine: the model's cleanup owns it then.
-    audio.cancel(reason).catch(() => {});
+    // header resolution failure), nothing has taken ownership of `countedAudio`
+    // yet, so cancel it directly. When the model did take a reader, the cancel
+    // rejects and the model's cleanup owns it.
+    countedAudio.cancel(reason).catch(() => {});
     transform.writable.abort(reason).catch(() => {
       // the writable is already errored when the model stream failed mid-pipe
     });
@@ -346,3 +446,54 @@ export function streamTranscribe({
     },
   };
 }
+
+function createByteCountingStream(
+  audio: ReadableStream<Uint8Array | string>,
+  onChunk: (byteLength: number) => void,
+): ReadableStream<Uint8Array | string> {
+  let audioReader: ReadableStreamDefaultReader<Uint8Array | string> | undefined;
+
+  return new ReadableStream<Uint8Array | string>({
+    async pull(controller) {
+      audioReader ??= audio.getReader();
+      const { done, value } = await audioReader.read();
+      if (done) {
+        controller.close();
+        return;
+      }
+
+      onChunk(
+        typeof value === 'string'
+          ? getBase64ByteLength(value)
+          : value.byteLength,
+      );
+      controller.enqueue(value);
+    },
+    async cancel(reason) {
+      if (audioReader != null) {
+        await audioReader.cancel(reason);
+      } else {
+        await audio.cancel(reason);
+      }
+    },
+  });
+}
+
+function getBase64ByteLength(value: string): number {
+  let characterCount = 0;
+  let lastCharacter = '';
+  let secondLastCharacter = '';
+
+  for (const character of value) {
+    if (/\s/.test(character)) {
+      continue;
+    }
+    characterCount++;
+    secondLastCharacter = lastCharacter;
+    lastCharacter = character;
+  }
+
+  const padding =
+    lastCharacter === '=' ? (secondLastCharacter === '=' ? 2 : 1) : 0;
+  return Math.floor((characterCount * 3) / 4) - padding;
+}

package/src/transcribe/transcribe.ts

@@ -1,21 +1,34 @@
 import type { JSONObject } from '@ai-sdk/provider';
 import {
+  createIdGenerator,
   detectMediaType,
   withUserAgentSuffix,
   type ProviderOptions,
 } from '@ai-sdk/provider-utils';
 import { NoTranscriptGeneratedError } from '../error/no-transcript-generated-error';
 import { logWarnings } from '../logger/log-warnings';
+import { resolveTranscriptionModel } from '../model/resolve-model';
 import type { DataContent } from '../prompt';
 import { convertDataContentToUint8Array } from '../prompt/data-content';
+import { createTelemetryDispatcher } from '../telemetry/create-telemetry-dispatcher';
+import type { TelemetryOptions } from '../telemetry/telemetry-options';
 import type { TranscriptionModel } from '../types/transcription-model';
 import type { TranscriptionModelResponseMetadata } from '../types/transcription-model-response-metadata';
 import { createDownload } from '../util/download/create-download';
+import { notify } from '../util/notify';
 import { prepareRetries } from '../util/prepare-retries';
 import type { TranscriptionResult } from './transcribe-result';
 import { VERSION } from '../version';
-import { resolveTranscriptionModel } from '../model/resolve-model';
 import type { Warning } from '../types';
+import type {
+  TranscriptionEndEvent,
+  TranscriptionStartEvent,
+} from './transcription-events';
+
+const originalGenerateCallId = createIdGenerator({
+  prefix: 'call',
+  size: 24,
+});
 /**
  * Generates transcripts using a transcription model.
  *
@@ -26,6 +39,7 @@ import type { Warning } from '../types';
  * @param maxRetries - Maximum number of retries. Set to 0 to disable retries. Default: 2.
  * @param abortSignal - An optional abort signal that can be used to cancel the call.
  * @param headers - Additional HTTP headers to be sent with the request. Only applicable for HTTP-based providers.
+ * @param telemetry - Optional telemetry configuration.
  *
  * @returns A result object that contains the generated transcript.
  */
@@ -38,7 +52,9 @@ export async function transcribe({
   maxRetries: maxRetriesArg,
   abortSignal,
   headers,
+  telemetry,
   download: downloadFn = defaultDownload,
+  _internal: { generateCallId = originalGenerateCallId } = {},
 }: {
   /**
    * The transcription model to use.
@@ -84,6 +100,11 @@ export async function transcribe({
    */
   headers?: Record<string, string>;
 
+  /**
+   * Optional telemetry configuration.
+   */
+  telemetry?: TelemetryOptions;
+
   /**
    * Custom download function for fetching audio from URLs.
    * Use `createDownload()` from `ai` to create a download function with custom size limits.
@@ -94,13 +115,20 @@ export async function transcribe({
     url: URL;
     abortSignal?: AbortSignal;
   }) => Promise<{ data: Uint8Array; mediaType: string | undefined }>;
+
+  /**
+   * Internal. For test use only. May change without notice.
+   */
+  _internal?: {
+    generateCallId?: () => string;
+  };
 }): Promise<TranscriptionResult> {
   const resolvedModel = resolveTranscriptionModel(model);
   if (!resolvedModel) {
     throw new Error('Model could not be resolved');
   }
 
-  const { retry } = prepareRetries({
+  const { maxRetries, retry } = prepareRetries({
     maxRetries: maxRetriesArg,
     abortSignal,
   });
@@ -110,44 +138,169 @@ export async function transcribe({
     `ai/${VERSION}`,
   );
 
-  const audioData =
-    audio instanceof URL
-      ? (await downloadFn({ url: audio, abortSignal })).data
-      : convertDataContentToUint8Array(audio);
-
-  const result = await retry(() =>
-    resolvedModel.doGenerate({
-      audio: audioData,
-      abortSignal,
-      headers: headersWithUserAgent,
-      providerOptions,
-      mediaType:
-        detectMediaType({
-          data: audioData,
-          topLevelType: 'audio',
-        }) ?? 'audio/wav',
-    }),
-  );
+  const callId = generateCallId();
+  const telemetryDispatcher = createTelemetryDispatcher({ telemetry });
+  const runInTracingChannelSpan =
+    telemetryDispatcher.runInTracingChannelSpan ??
+    (async <T>({ execute }: { execute: () => PromiseLike<T> }) =>
+      await execute());
 
-  logWarnings({
-    warnings: result.warnings,
+  const createStartEvent = ({
+    byteLength,
+    mediaType,
+  }: {
+    byteLength: number | undefined;
+    mediaType: string | undefined;
+  }): TranscriptionStartEvent => ({
+    callId,
+    operationId: 'ai.transcribe',
     provider: resolvedModel.provider,
-    model: resolvedModel.modelId,
+    modelId: resolvedModel.modelId,
+    audio: {
+      byteLength,
+      mediaType,
+    },
+    inputAudioFormat: undefined,
+    maxRetries,
+    headers,
+    providerOptions,
   });
 
-  if (!result.text) {
-    throw new NoTranscriptGeneratedError({ responses: [result.response] });
+  const executeWithStartEvent = async <T>({
+    startEvent,
+    execute,
+  }: {
+    startEvent: TranscriptionStartEvent;
+    execute: () => PromiseLike<T>;
+  }) =>
+    await runInTracingChannelSpan({
+      type: 'transcribe',
+      event: startEvent,
+      execute: async () => {
+        await notify({
+          event: startEvent,
+          callbacks: [telemetryDispatcher.onStart],
+        });
+
+        return await execute();
+      },
+    });
+
+  let preparationError: unknown;
+  let preparationFailed = false;
+  let audioData: Uint8Array;
+  let downloadedMediaType: string | undefined;
+
+  try {
+    if (audio instanceof URL) {
+      const downloadResult = await downloadFn({ url: audio, abortSignal });
+      audioData = downloadResult.data;
+      downloadedMediaType = downloadResult.mediaType;
+    } else {
+      audioData = convertDataContentToUint8Array(audio);
+    }
+  } catch (error) {
+    preparationFailed = true;
+    preparationError = error;
+    audioData = new Uint8Array();
+  }
+
+  if (preparationFailed) {
+    const startEvent = createStartEvent({
+      byteLength: undefined,
+      mediaType: undefined,
+    });
+
+    try {
+      return await executeWithStartEvent({
+        startEvent,
+        execute: () => {
+          throw preparationError;
+        },
+      });
+    } catch (error) {
+      await telemetryDispatcher.onError?.({ callId, error });
+      throw error;
+    }
   }
 
-  return new DefaultTranscriptionResult({
-    text: result.text,
-    segments: result.segments,
-    language: result.language,
-    durationInSeconds: result.durationInSeconds,
-    warnings: result.warnings,
-    responses: [result.response],
-    providerMetadata: result.providerMetadata,
+  const mediaType =
+    downloadedMediaType ??
+    detectMediaType({
+      data: audioData,
+      topLevelType: 'audio',
+    }) ??
+    'audio/wav';
+  const startEvent = createStartEvent({
+    byteLength: audioData.byteLength,
+    mediaType,
   });
+
+  try {
+    return await executeWithStartEvent({
+      startEvent,
+      execute: async () => {
+        const result = await retry(() =>
+          resolvedModel.doGenerate({
+            audio: audioData,
+            abortSignal,
+            headers: headersWithUserAgent,
+            providerOptions,
+            mediaType,
+          }),
+        );
+
+        logWarnings({
+          warnings: result.warnings,
+          provider: resolvedModel.provider,
+          model: resolvedModel.modelId,
+        });
+
+        if (!result.text) {
+          throw new NoTranscriptGeneratedError({
+            responses: [result.response],
+          });
+        }
+
+        const endEvent: TranscriptionEndEvent = {
+          callId,
+          operationId: 'ai.transcribe',
+          provider: resolvedModel.provider,
+          modelId: resolvedModel.modelId,
+          audio: {
+            byteLength: audioData.byteLength,
+            mediaType,
+          },
+          text: result.text,
+          segments: result.segments,
+          language: result.language,
+          durationInSeconds: result.durationInSeconds,
+          usage: result.usage,
+          warnings: result.warnings,
+          providerMetadata: result.providerMetadata,
+          response: result.response,
+        };
+
+        await notify({
+          event: endEvent,
+          callbacks: [telemetryDispatcher.onEnd],
+        });
+
+        return new DefaultTranscriptionResult({
+          text: result.text,
+          segments: result.segments,
+          language: result.language,
+          durationInSeconds: result.durationInSeconds,
+          warnings: result.warnings,
+          responses: [result.response],
+          providerMetadata: result.providerMetadata,
+        });
+      },
+    });
+  } catch (error) {
+    await telemetryDispatcher.onError?.({ callId, error });
+    throw error;
+  }
 }
 
 class DefaultTranscriptionResult implements TranscriptionResult {