ai 7.0.123 → 7.0.124

package/src/telemetry/telemetry.ts

@@ -40,6 +40,16 @@ import type {
   RerankingModelCallEndEvent,
   RerankingModelCallStartEvent,
 } from '../rerank/rerank-events';
+import type {
+  GenerateSpeechEndEvent,
+  GenerateSpeechStartEvent,
+} from '../generate-speech/speech-events';
+import type {
+  StreamTranscriptionEndEvent,
+  StreamTranscriptionStartEvent,
+  TranscriptionEndEvent,
+  TranscriptionStartEvent,
+} from '../transcribe/transcription-events';
 import type { Callback } from '../util/callback';
 import type { TelemetryOptions } from '../telemetry/telemetry-options';
 import type { TelemetryTracingEventType } from './tracing-channel';
@@ -51,17 +61,25 @@ export type InferTelemetryEvent<EVENT> = EVENT &
     'integrations' | 'isEnabled' | 'includeRuntimeContext'
   >;
 
+type BivariantCallback<EVENT> = {
+  bivarianceHack(event: EVENT): PromiseLike<void> | void;
+}['bivarianceHack'];
+
 type OperationStartEvent =
   | GenerateTextStartEvent
   | GenerateObjectStartEvent
   | EmbedStartEvent
-  | RerankStartEvent;
+  | RerankStartEvent
+  | GenerateSpeechStartEvent
+  | TranscriptionStartEvent;
 
 type OperationEndEvent =
   | GenerateTextEndEvent<ToolSet>
   | GenerateObjectEndEvent<unknown>
   | EmbedEndEvent
-  | RerankEndEvent;
+  | RerankEndEvent
+  | GenerateSpeechEndEvent
+  | TranscriptionEndEvent;
 
 export interface TelemetryDispatcher {
   /**
@@ -102,6 +120,8 @@ export interface TelemetryDispatcher {
   experimental_onEvaluationModelCallStart?: Callback<EvaluationModelCallStartEvent>;
   experimental_onEvaluationModelCallEnd?: Callback<EvaluationModelCallEndEvent>;
   experimental_onEvaluateEnd?: Callback<EvaluateEndEvent>;
+  experimental_onStreamTranscriptionStart?: Callback<StreamTranscriptionStartEvent>;
+  experimental_onStreamTranscriptionEnd?: Callback<StreamTranscriptionEndEvent>;
   onEnd?: Callback<OperationEndEvent>;
   onAbort?: Callback<GenerateTextAbortEvent<ToolSet>>;
   onError?: Callback<unknown>;
@@ -117,11 +137,12 @@ export interface Telemetry {
   /**
    * Called when an operation begins. Fired for text generation
    * (generateText/streamText), object generation (generateObject/streamObject),
-   * embedding (embed/embedMany), and reranking operations.
+   * embedding (embed/embedMany), reranking, speech generation, and
+   * non-streaming transcription operations.
    *
    * Use the `operationId` field to distinguish between operation types.
    */
-  onStart?: Callback<InferTelemetryEvent<OperationStartEvent>>;
+  onStart?: BivariantCallback<InferTelemetryEvent<OperationStartEvent>>;
 
   /**
    * Called when an individual step (single LLM invocation) begins.
@@ -248,14 +269,25 @@ export interface Telemetry {
   /** Called when an experimental evaluation operation completes. */
   experimental_onEvaluateEnd?: Callback<InferTelemetryEvent<EvaluateEndEvent>>;
 
+  /** Called when an experimental streaming transcription operation begins. */
+  experimental_onStreamTranscriptionStart?: Callback<
+    InferTelemetryEvent<StreamTranscriptionStartEvent>
+  >;
+
+  /** Called when an experimental streaming transcription operation completes. */
+  experimental_onStreamTranscriptionEnd?: Callback<
+    InferTelemetryEvent<StreamTranscriptionEndEvent>
+  >;
+
   /**
    * Called when an operation completes. Fired for text generation
    * (generateText/streamText), object generation (generateObject/streamObject),
-   * embedding (embed/embedMany), and reranking operations.
+   * embedding (embed/embedMany), reranking, speech generation, and
+   * non-streaming transcription operations.
    *
    * Use the event shape or `operationId` to distinguish between operation types.
    */
-  onEnd?: Callback<InferTelemetryEvent<OperationEndEvent>>;
+  onEnd?: BivariantCallback<InferTelemetryEvent<OperationEndEvent>>;
 
   /**
    * Called when a streaming text generation operation is aborted before it

package/src/telemetry/tracing-channel.ts

@@ -9,6 +9,9 @@ export type TelemetryTracingEventType =
   | 'embed'
   | 'embedMany'
   | 'rerank'
+  | 'generateSpeech'
+  | 'transcribe'
+  | 'streamTranscribe'
   | 'experimental_evaluate';
 
 export type TelemetryTracingChannelMessage<EVENT = unknown> = {

package/src/transcribe/index.ts

@@ -8,6 +8,12 @@ export type {
   TranscriptionStreamPart,
 } from './stream-transcribe-result';
 export type { TranscriptionResult } from './transcribe-result';
+export type {
+  StreamTranscriptionEndEvent as Experimental_StreamTranscriptionEndEvent,
+  StreamTranscriptionStartEvent as Experimental_StreamTranscriptionStartEvent,
+  TranscriptionEndEvent,
+  TranscriptionStartEvent,
+} from './transcription-events';
 
 // deprecated exports
 

package/src/transcribe/stream-transcribe.ts

@@ -5,6 +5,7 @@ import {
   type SharedV4AudioFormat,
 } from '@ai-sdk/provider';
 import {
+  createIdGenerator,
   DelayedPromise,
   withUserAgentSuffix,
   type ProviderOptions,
@@ -12,16 +13,23 @@ import {
 import { NoTranscriptGeneratedError } from '../error/no-transcript-generated-error';
 import { logWarnings } from '../logger/log-warnings';
 import { resolveTranscriptionModel } from '../model/resolve-model';
+import { createTelemetryDispatcher } from '../telemetry/create-telemetry-dispatcher';
+import type { TelemetryOptions } from '../telemetry/telemetry-options';
 import type { TranscriptionModel } from '../types/transcription-model';
 import type { TranscriptionModelResponseMetadata } from '../types/transcription-model-response-metadata';
 import type { Warning } from '../types/warning';
 import { asAsyncIterableStream } from '../util/async-iterable-stream';
 import { mergeAbortSignals } from '../util/merge-abort-signals';
+import { notify } from '../util/notify';
 import { VERSION } from '../version';
 import type {
   StreamTranscriptionResult,
   TranscriptionStreamPart,
 } from './stream-transcribe-result';
+import type {
+  StreamTranscriptionEndEvent,
+  StreamTranscriptionStartEvent,
+} from './transcription-events';
 
 type TranscriptSegment = {
   text: string;
@@ -29,6 +37,11 @@ type TranscriptSegment = {
   endSecond: number;
 };
 
+const originalGenerateCallId = createIdGenerator({
+  prefix: 'call',
+  size: 24,
+});
+
 /**
  * Streams transcripts using a transcription model.
  *
@@ -38,6 +51,7 @@ type TranscriptSegment = {
  * @param providerOptions - Additional provider-specific options.
  * @param abortSignal - An optional abort signal that can be used to cancel the call.
  * @param headers - Additional HTTP/WebSocket headers to send when supported by the provider.
+ * @param telemetry - Optional telemetry configuration.
  *
  * @returns A result object that contains the streaming transcript and final transcript metadata.
  */
@@ -49,7 +63,11 @@ export function streamTranscribe({
   abortSignal,
   headers,
   includeRawChunks,
-  _internal: { currentDate = () => new Date() } = {},
+  telemetry,
+  _internal: {
+    currentDate = () => new Date(),
+    generateCallId = originalGenerateCallId,
+  } = {},
 }: {
   /**
    * The transcription model to use.
@@ -86,11 +104,17 @@ export function streamTranscribe({
    */
   includeRawChunks?: boolean;
 
+  /**
+   * Optional telemetry configuration.
+   */
+  telemetry?: TelemetryOptions;
+
   /**
    * Internal test hooks.
    */
   _internal?: {
     currentDate?: () => Date;
+    generateCallId?: () => string;
   };
 }): StreamTranscriptionResult {
   const resolvedModel = resolveTranscriptionModel(model);
@@ -118,6 +142,34 @@ export function streamTranscribe({
     `ai/${VERSION}`,
   );
 
+  const callId = generateCallId();
+  const telemetryDispatcher = createTelemetryDispatcher({ telemetry });
+  const runInTracingChannelSpan =
+    telemetryDispatcher.runInTracingChannelSpan ??
+    (async <T>({ execute }: { execute: () => PromiseLike<T> }) =>
+      await execute());
+  let audioByteLength = 0;
+  const countedAudio =
+    telemetryDispatcher.experimental_onStreamTranscriptionEnd != null
+      ? createByteCountingStream(audio, byteLength => {
+          audioByteLength += byteLength;
+        })
+      : audio;
+  const startEvent: StreamTranscriptionStartEvent = {
+    callId,
+    operationId: 'ai.streamTranscribe',
+    provider: resolvedModel.provider,
+    modelId: resolvedModel.modelId,
+    audio: {
+      byteLength: undefined,
+      mediaType: inputAudioFormat.type,
+    },
+    inputAudioFormat,
+    maxRetries: undefined,
+    headers,
+    providerOptions,
+  };
+
   const textPromise = new DelayedPromise<string>();
   const segmentsPromise = new DelayedPromise<Array<TranscriptSegment>>();
   const languagePromise = new DelayedPromise<string | undefined>();
@@ -129,6 +181,8 @@ export function streamTranscribe({
   const providerMetadataPromise = new DelayedPromise<
     Record<string, JSONObject>
   >();
+  let warnings: Array<Warning> = [];
+  let telemetrySettled = false;
 
   const rejectPendingPromises = (error: unknown) => {
     for (const promise of [
@@ -151,10 +205,11 @@ export function streamTranscribe({
   const currentResponseMetadata = () =>
     response ?? { timestamp: startedAt, modelId: resolvedModel.modelId };
 
-  const resolveWarnings = (warnings: Array<Warning>) => {
-    warningsPromise.resolve(warnings);
+  const resolveWarnings = (resolvedWarnings: Array<Warning>) => {
+    warnings = resolvedWarnings;
+    warningsPromise.resolve(resolvedWarnings);
     logWarnings({
-      warnings,
+      warnings: resolvedWarnings,
       provider: resolvedModel.provider,
       model: resolvedModel.modelId,
     });
@@ -172,7 +227,7 @@ export function streamTranscribe({
     Experimental_TranscriptionModelV4StreamPart,
     TranscriptionStreamPart
   > & { cancel?: (reason?: unknown) => void } = {
-    transform(value, controller) {
+    async transform(value, controller) {
       switch (value.type) {
         case 'stream-start': {
           resolveWarnings(value.warnings);
@@ -214,6 +269,37 @@ export function streamTranscribe({
           durationInSecondsPromise.resolve(value.durationInSeconds);
           responsesPromise.resolve([currentResponseMetadata()]);
           providerMetadataPromise.resolve(value.providerMetadata ?? {});
+
+          telemetrySettled = true;
+          if (
+            telemetryDispatcher.experimental_onStreamTranscriptionEnd != null
+          ) {
+            const endEvent: StreamTranscriptionEndEvent = {
+              callId,
+              operationId: 'ai.streamTranscribe',
+              provider: resolvedModel.provider,
+              modelId: resolvedModel.modelId,
+              audio: {
+                byteLength: audioByteLength,
+                mediaType: inputAudioFormat.type,
+              },
+              text: value.text,
+              segments: value.segments,
+              language: value.language,
+              durationInSeconds: value.durationInSeconds,
+              usage: value.usage,
+              warnings,
+              providerMetadata: value.providerMetadata,
+              response: currentResponseMetadata(),
+            };
+
+            await notify({
+              event: endEvent,
+              callbacks: [
+                telemetryDispatcher.experimental_onStreamTranscriptionEnd,
+              ],
+            });
+          }
           break;
         }
       }
@@ -241,36 +327,50 @@ export function streamTranscribe({
 
   // Piping (instead of an eager read loop) preserves consumer backpressure
   // and propagates cancellation of `fullStream` to the model stream.
-  void (async () => {
-    const result = await doStream({
-      audio,
-      inputAudioFormat,
-      providerOptions,
-      // merged so cancelling fullStream also aborts a still-pending doStream
-      abortSignal: mergeAbortSignals(abortSignal, pipeAbortController.signal),
-      headers: headersWithUserAgent,
-      includeRawChunks,
-    });
-
-    response = {
-      timestamp: result.response?.timestamp ?? startedAt,
-      modelId: result.response?.modelId ?? resolvedModel.modelId,
-      headers: result.response?.headers,
-    };
-
-    await result.stream.pipeTo(transform.writable, {
-      signal: pipeAbortController.signal,
-    });
-  })().catch(error => {
+  void runInTracingChannelSpan({
+    type: 'streamTranscribe',
+    event: startEvent,
+    execute: async () => {
+      await notify({
+        event: startEvent,
+        callbacks: [
+          telemetryDispatcher.experimental_onStreamTranscriptionStart,
+        ],
+      });
+
+      const result = await doStream({
+        audio: countedAudio,
+        inputAudioFormat,
+        providerOptions,
+        // merged so cancelling fullStream also aborts a still-pending doStream
+        abortSignal: mergeAbortSignals(abortSignal, pipeAbortController.signal),
+        headers: headersWithUserAgent,
+        includeRawChunks,
+      });
+
+      response = {
+        timestamp: result.response?.timestamp ?? startedAt,
+        modelId: result.response?.modelId ?? resolvedModel.modelId,
+        headers: result.response?.headers,
+      };
+
+      await result.stream.pipeTo(transform.writable, {
+        signal: pipeAbortController.signal,
+      });
+    },
+  }).catch(async error => {
     const reason =
       error ?? new Error('Transcription stream was cancelled or errored.');
     rejectPendingPromises(reason);
+    if (!telemetrySettled) {
+      telemetrySettled = true;
+      await telemetryDispatcher.onError?.({ callId, error: reason });
+    }
     // When `doStream` rejects before the model stream exists (e.g. auth or
-    // header resolution failure), nothing has taken ownership of `audio` yet,
-    // so cancel it directly — otherwise an upstream producer piping into it
-    // hangs forever. When the model did take a reader, `audio` is locked and
-    // the cancel rejects, which is fine: the model's cleanup owns it then.
-    audio.cancel(reason).catch(() => {});
+    // header resolution failure), nothing has taken ownership of `countedAudio`
+    // yet, so cancel it directly. When the model did take a reader, the cancel
+    // rejects and the model's cleanup owns it.
+    countedAudio.cancel(reason).catch(() => {});
     transform.writable.abort(reason).catch(() => {
       // the writable is already errored when the model stream failed mid-pipe
     });
@@ -346,3 +446,54 @@ export function streamTranscribe({
     },
   };
 }
+
+function createByteCountingStream(
+  audio: ReadableStream<Uint8Array | string>,
+  onChunk: (byteLength: number) => void,
+): ReadableStream<Uint8Array | string> {
+  let audioReader: ReadableStreamDefaultReader<Uint8Array | string> | undefined;
+
+  return new ReadableStream<Uint8Array | string>({
+    async pull(controller) {
+      audioReader ??= audio.getReader();
+      const { done, value } = await audioReader.read();
+      if (done) {
+        controller.close();
+        return;
+      }
+
+      onChunk(
+        typeof value === 'string'
+          ? getBase64ByteLength(value)
+          : value.byteLength,
+      );
+      controller.enqueue(value);
+    },
+    async cancel(reason) {
+      if (audioReader != null) {
+        await audioReader.cancel(reason);
+      } else {
+        await audio.cancel(reason);
+      }
+    },
+  });
+}
+
+function getBase64ByteLength(value: string): number {
+  let characterCount = 0;
+  let lastCharacter = '';
+  let secondLastCharacter = '';
+
+  for (const character of value) {
+    if (/\s/.test(character)) {
+      continue;
+    }
+    characterCount++;
+    secondLastCharacter = lastCharacter;
+    lastCharacter = character;
+  }
+
+  const padding =
+    lastCharacter === '=' ? (secondLastCharacter === '=' ? 2 : 1) : 0;
+  return Math.floor((characterCount * 3) / 4) - padding;
+}