ai 7.0.123 → 7.0.126

package/src/generate-speech/generate-speech.ts

@@ -1,5 +1,6 @@
 import type { JSONObject } from '@ai-sdk/provider';
 import {
+  createIdGenerator,
   detectMediaType,
   withUserAgentSuffix,
   type ProviderOptions,
@@ -7,16 +8,28 @@ import {
 import { NoSpeechGeneratedError } from '../error/no-speech-generated-error';
 import { logWarnings } from '../logger/log-warnings';
 import { resolveSpeechModel } from '../model/resolve-model';
+import { createTelemetryDispatcher } from '../telemetry/create-telemetry-dispatcher';
+import type { TelemetryOptions } from '../telemetry/telemetry-options';
 import type { SpeechModel } from '../types/speech-model';
 import type { SpeechModelResponseMetadata } from '../types/speech-model-response-metadata';
 import type { Warning } from '../types/warning';
 import { prepareRetries } from '../util/prepare-retries';
+import { notify } from '../util/notify';
 import { VERSION } from '../version';
 import type { SpeechResult } from './generate-speech-result';
 import {
   DefaultGeneratedAudioFile,
   type GeneratedAudioFile,
 } from './generated-audio-file';
+import type {
+  GenerateSpeechEndEvent,
+  GenerateSpeechStartEvent,
+} from './speech-events';
+
+const originalGenerateCallId = createIdGenerator({
+  prefix: 'call',
+  size: 24,
+});
 /**
  * Generates speech audio using a speech model.
  *
@@ -32,6 +45,7 @@ import {
  * @param maxRetries - Maximum number of retries. Set to 0 to disable retries. Default: 2.
  * @param abortSignal - An optional abort signal that can be used to cancel the call.
  * @param headers - Additional HTTP headers to be sent with the request. Only applicable for HTTP-based providers.
+ * @param telemetry - Optional telemetry configuration.
  *
  * @returns A result object that contains the generated audio data.
  */
@@ -47,6 +61,8 @@ export async function generateSpeech({
   maxRetries: maxRetriesArg,
   abortSignal,
   headers,
+  telemetry,
+  _internal: { generateCallId = originalGenerateCallId } = {},
 }: {
   /**
    * The speech model to use.
@@ -115,6 +131,18 @@ export async function generateSpeech({
    * Only applicable for HTTP-based providers.
    */
   headers?: Record<string, string>;
+
+  /**
+   * Optional telemetry configuration.
+   */
+  telemetry?: TelemetryOptions;
+
+  /**
+   * Internal. For test use only. May change without notice.
+   */
+  _internal?: {
+    generateCallId?: () => string;
+  };
 }): Promise<SpeechResult> {
   const resolvedModel = resolveSpeechModel(model);
   if (!resolvedModel) {
@@ -126,55 +154,152 @@ export async function generateSpeech({
     `ai/${VERSION}`,
   );
 
-  const { retry } = prepareRetries({
+  const { maxRetries, retry } = prepareRetries({
     maxRetries: maxRetriesArg,
     abortSignal,
   });
 
-  const result = await retry(() =>
-    resolvedModel.doGenerate({
-      text,
-      voice,
-      outputFormat,
-      instructions,
-      speed,
-      language,
-      abortSignal,
-      headers: headersWithUserAgent,
-      providerOptions,
-    }),
-  );
-
-  if (!result.audio || result.audio.length === 0) {
-    throw new NoSpeechGeneratedError({ responses: [result.response] });
-  }
+  const callId = generateCallId();
+  const telemetryDispatcher = createTelemetryDispatcher({ telemetry });
+  const runInTracingChannelSpan =
+    telemetryDispatcher.runInTracingChannelSpan ??
+    (async <T>({ execute }: { execute: () => PromiseLike<T> }) =>
+      await execute());
 
-  logWarnings({
-    warnings: result.warnings,
+  const startEvent: GenerateSpeechStartEvent = {
+    callId,
+    operationId: 'ai.generateSpeech',
     provider: resolvedModel.provider,
-    model: resolvedModel.modelId,
-  });
+    modelId: resolvedModel.modelId,
+    text,
+    voice,
+    outputFormat,
+    instructions,
+    speed,
+    language,
+    maxRetries,
+    headers,
+    providerOptions,
+  };
 
-  const detectedMediaType = detectMediaType({
-    data: result.audio,
-    topLevelType: 'audio',
-  });
+  return await runInTracingChannelSpan({
+    type: 'generateSpeech',
+    event: startEvent,
+    execute: async () => {
+      await notify({
+        event: startEvent,
+        callbacks: [telemetryDispatcher.onStart],
+      });
+
+      try {
+        const result = await retry(() =>
+          resolvedModel.doGenerate({
+            text,
+            voice,
+            outputFormat,
+            instructions,
+            speed,
+            language,
+            abortSignal,
+            headers: headersWithUserAgent,
+            providerOptions,
+          }),
+        );
+
+        if (!result.audio || result.audio.length === 0) {
+          throw new NoSpeechGeneratedError({ responses: [result.response] });
+        }
 
-  return new DefaultSpeechResult({
-    audio: new DefaultGeneratedAudioFile({
-      data: result.audio,
-      mediaType:
-        detectedMediaType ??
-        getResponseAudioMediaType(result.response.headers) ??
-        getOutputFormatMediaType(outputFormat) ??
-        'audio/mp3',
-    }),
-    warnings: result.warnings,
-    responses: [result.response],
-    providerMetadata: result.providerMetadata,
+        logWarnings({
+          warnings: result.warnings,
+          provider: resolvedModel.provider,
+          model: resolvedModel.modelId,
+        });
+
+        const detectedMediaType = detectMediaType({
+          data: result.audio,
+          topLevelType: 'audio',
+        });
+
+        const audio = new DefaultGeneratedAudioFile({
+          data: result.audio,
+          mediaType:
+            detectedMediaType ??
+            getResponseAudioMediaType(result.response.headers) ??
+            getOutputFormatMediaType(outputFormat) ??
+            'audio/mp3',
+        });
+
+        if (telemetryDispatcher.onEnd != null) {
+          const endEvent: GenerateSpeechEndEvent = {
+            callId,
+            operationId: 'ai.generateSpeech',
+            provider: resolvedModel.provider,
+            modelId: resolvedModel.modelId,
+            text,
+            audio: {
+              byteLength: getBase64OrBinaryByteLength(result.audio),
+              mediaType: audio.mediaType,
+              format: audio.format,
+            },
+            usage: result.usage,
+            warnings: result.warnings,
+            providerMetadata: result.providerMetadata,
+            response: getTelemetryResponseMetadata(result.response),
+          };
+
+          await notify({
+            event: endEvent,
+            callbacks: [telemetryDispatcher.onEnd],
+          });
+        }
+
+        return new DefaultSpeechResult({
+          audio,
+          warnings: result.warnings,
+          responses: [result.response],
+          providerMetadata: result.providerMetadata,
+        });
+      } catch (error) {
+        await telemetryDispatcher.onError?.({ callId, error });
+        throw error;
+      }
+    },
   });
 }
 
+function getBase64OrBinaryByteLength(audio: string | Uint8Array): number {
+  if (audio instanceof Uint8Array) {
+    return audio.byteLength;
+  }
+
+  let characterCount = 0;
+  let lastCharacter = '';
+  let secondLastCharacter = '';
+  for (const character of audio) {
+    if (/\s/.test(character)) {
+      continue;
+    }
+    characterCount++;
+    secondLastCharacter = lastCharacter;
+    lastCharacter = character;
+  }
+
+  const padding =
+    lastCharacter === '=' ? (secondLastCharacter === '=' ? 2 : 1) : 0;
+  return Math.floor((characterCount * 3) / 4) - padding;
+}
+
+function getTelemetryResponseMetadata(
+  response: SpeechModelResponseMetadata,
+): SpeechModelResponseMetadata {
+  return {
+    timestamp: response.timestamp,
+    modelId: response.modelId,
+    headers: response.headers,
+  };
+}
+
 function getResponseAudioMediaType(
   headers: Record<string, string> | undefined,
 ): string | undefined {

package/src/generate-speech/index.ts

@@ -4,6 +4,10 @@ import { generateSpeech } from './generate-speech';
 export { generateSpeech } from './generate-speech';
 export type { SpeechResult } from './generate-speech-result';
 export type { GeneratedAudioFile } from './generated-audio-file';
+export type {
+  GenerateSpeechEndEvent,
+  GenerateSpeechStartEvent,
+} from './speech-events';
 
 // deprecated exports
 

package/src/generate-speech/speech-events.ts

@@ -0,0 +1,91 @@
+import type { JSONObject } from '@ai-sdk/provider';
+import type { ProviderOptions } from '@ai-sdk/provider-utils';
+import type { ProviderMetadata } from '../types';
+import type { SpeechModelResponseMetadata } from '../types/speech-model-response-metadata';
+import type { Warning } from '../types/warning';
+
+/**
+ * Event passed to telemetry integrations when speech generation begins.
+ */
+export type GenerateSpeechStartEvent = {
+  /** Unique identifier for this speech generation call. */
+  readonly callId: string;
+
+  /** Identifies the operation type (`ai.generateSpeech`). */
+  readonly operationId: 'ai.generateSpeech';
+
+  /** The provider identifier. */
+  readonly provider: string;
+
+  /** The specific model identifier. */
+  readonly modelId: string;
+
+  /** The text to convert to speech. */
+  readonly text: string;
+
+  /** The requested voice. */
+  readonly voice: string | undefined;
+
+  /** The requested output format. */
+  readonly outputFormat: string | undefined;
+
+  /** Additional speech instructions. */
+  readonly instructions: string | undefined;
+
+  /** The requested speech speed. */
+  readonly speed: number | undefined;
+
+  /** The requested language. */
+  readonly language: string | undefined;
+
+  /** Maximum number of retries for failed requests. */
+  readonly maxRetries: number;
+
+  /** Additional HTTP headers sent with the request. */
+  readonly headers: Record<string, string> | undefined;
+
+  /** Additional provider-specific options. */
+  readonly providerOptions: ProviderOptions;
+};
+
+/**
+ * Event passed to telemetry integrations when speech generation completes.
+ *
+ * Raw audio is intentionally not included. Integrations receive only its size,
+ * media type, and format.
+ */
+export type GenerateSpeechEndEvent = {
+  /** Unique identifier for this speech generation call. */
+  readonly callId: string;
+
+  /** Identifies the operation type (`ai.generateSpeech`). */
+  readonly operationId: 'ai.generateSpeech';
+
+  /** The provider identifier. */
+  readonly provider: string;
+
+  /** The specific model identifier. */
+  readonly modelId: string;
+
+  /** The text that was converted to speech. */
+  readonly text: string;
+
+  /** Metadata about the generated audio. */
+  readonly audio: {
+    readonly byteLength: number;
+    readonly mediaType: string;
+    readonly format: string;
+  };
+
+  /** Usage reported by the provider, when available. */
+  readonly usage: JSONObject | undefined;
+
+  /** Warnings from the speech model. */
+  readonly warnings: Array<Warning>;
+
+  /** Optional provider-specific metadata. */
+  readonly providerMetadata: ProviderMetadata | undefined;
+
+  /** Response metadata from the provider. */
+  readonly response: SpeechModelResponseMetadata;
+};

package/src/telemetry/create-telemetry-dispatcher.ts

@@ -142,7 +142,7 @@ export function createTelemetryDispatcher({
 
   const mergeTelemetryCallback = <KEY extends TelemetryCallbackKey>(
     key: KEY,
-  ): Callback<TelemetryEvent<KEY>> => {
+  ): Callback<TelemetryEvent<KEY>> | undefined => {
     const integrationCallbacks = (
       integrations
         .map(integration => integration[key]?.bind(integration))
@@ -157,6 +157,10 @@ export function createTelemetryDispatcher({
         >,
     );
 
+    if (integrationCallbacks.length === 0) {
+      return undefined;
+    }
+
     const mergedIntegrationCallback = mergeCallbacks(...integrationCallbacks);
 
     return async (event: TelemetryEvent<KEY>) => {
@@ -164,6 +168,9 @@ export function createTelemetryDispatcher({
     };
   };
 
+  const onStepEnd = mergeTelemetryCallback('onStepEnd');
+  const onStepFinish = mergeTelemetryCallback('onStepFinish');
+
   const executeLanguageModelCallWrappers = integrations
     .map(integration => integration.executeLanguageModelCall?.bind(integration))
     .filter(Boolean) as Array<
@@ -205,10 +212,10 @@ export function createTelemetryDispatcher({
     // deprecated `onStepFinish` callback so integrations that still implement
     // only `onStepFinish` keep receiving step-end events during the deprecation
     // window.
-    onStepEnd: mergeCallbacks(
-      mergeTelemetryCallback('onStepEnd'),
-      mergeTelemetryCallback('onStepFinish'),
-    ),
+    onStepEnd:
+      onStepEnd == null && onStepFinish == null
+        ? undefined
+        : mergeCallbacks(onStepEnd, onStepFinish),
     onObjectStepStart: mergeTelemetryCallback('onObjectStepStart'),
     onObjectStepEnd: mergeTelemetryCallback('onObjectStepEnd'),
     onEmbedStart: mergeTelemetryCallback('onEmbedStart'),
@@ -227,6 +234,12 @@ export function createTelemetryDispatcher({
     experimental_onEvaluateEnd: mergeTelemetryCallback(
       'experimental_onEvaluateEnd',
     ),
+    experimental_onStreamTranscriptionStart: mergeTelemetryCallback(
+      'experimental_onStreamTranscriptionStart',
+    ),
+    experimental_onStreamTranscriptionEnd: mergeTelemetryCallback(
+      'experimental_onStreamTranscriptionEnd',
+    ),
     onEnd: mergeTelemetryCallback('onEnd'),
     onAbort: mergeTelemetryCallback('onAbort'),
     onError: mergeTelemetryCallback('onError'),

package/src/telemetry/telemetry.ts

@@ -40,6 +40,16 @@ import type {
   RerankingModelCallEndEvent,
   RerankingModelCallStartEvent,
 } from '../rerank/rerank-events';
+import type {
+  GenerateSpeechEndEvent,
+  GenerateSpeechStartEvent,
+} from '../generate-speech/speech-events';
+import type {
+  StreamTranscriptionEndEvent,
+  StreamTranscriptionStartEvent,
+  TranscriptionEndEvent,
+  TranscriptionStartEvent,
+} from '../transcribe/transcription-events';
 import type { Callback } from '../util/callback';
 import type { TelemetryOptions } from '../telemetry/telemetry-options';
 import type { TelemetryTracingEventType } from './tracing-channel';
@@ -51,17 +61,25 @@ export type InferTelemetryEvent<EVENT> = EVENT &
     'integrations' | 'isEnabled' | 'includeRuntimeContext'
   >;
 
+type BivariantCallback<EVENT> = {
+  bivarianceHack(event: EVENT): PromiseLike<void> | void;
+}['bivarianceHack'];
+
 type OperationStartEvent =
   | GenerateTextStartEvent
   | GenerateObjectStartEvent
   | EmbedStartEvent
-  | RerankStartEvent;
+  | RerankStartEvent
+  | GenerateSpeechStartEvent
+  | TranscriptionStartEvent;
 
 type OperationEndEvent =
   | GenerateTextEndEvent<ToolSet>
   | GenerateObjectEndEvent<unknown>
   | EmbedEndEvent
-  | RerankEndEvent;
+  | RerankEndEvent
+  | GenerateSpeechEndEvent
+  | TranscriptionEndEvent;
 
 export interface TelemetryDispatcher {
   /**
@@ -102,6 +120,8 @@ export interface TelemetryDispatcher {
   experimental_onEvaluationModelCallStart?: Callback<EvaluationModelCallStartEvent>;
   experimental_onEvaluationModelCallEnd?: Callback<EvaluationModelCallEndEvent>;
   experimental_onEvaluateEnd?: Callback<EvaluateEndEvent>;
+  experimental_onStreamTranscriptionStart?: Callback<StreamTranscriptionStartEvent>;
+  experimental_onStreamTranscriptionEnd?: Callback<StreamTranscriptionEndEvent>;
   onEnd?: Callback<OperationEndEvent>;
   onAbort?: Callback<GenerateTextAbortEvent<ToolSet>>;
   onError?: Callback<unknown>;
@@ -117,11 +137,12 @@ export interface Telemetry {
   /**
    * Called when an operation begins. Fired for text generation
    * (generateText/streamText), object generation (generateObject/streamObject),
-   * embedding (embed/embedMany), and reranking operations.
+   * embedding (embed/embedMany), reranking, speech generation, and
+   * non-streaming transcription operations.
    *
    * Use the `operationId` field to distinguish between operation types.
    */
-  onStart?: Callback<InferTelemetryEvent<OperationStartEvent>>;
+  onStart?: BivariantCallback<InferTelemetryEvent<OperationStartEvent>>;
 
   /**
    * Called when an individual step (single LLM invocation) begins.
@@ -248,14 +269,25 @@ export interface Telemetry {
   /** Called when an experimental evaluation operation completes. */
   experimental_onEvaluateEnd?: Callback<InferTelemetryEvent<EvaluateEndEvent>>;
 
+  /** Called when an experimental streaming transcription operation begins. */
+  experimental_onStreamTranscriptionStart?: Callback<
+    InferTelemetryEvent<StreamTranscriptionStartEvent>
+  >;
+
+  /** Called when an experimental streaming transcription operation completes. */
+  experimental_onStreamTranscriptionEnd?: Callback<
+    InferTelemetryEvent<StreamTranscriptionEndEvent>
+  >;
+
   /**
    * Called when an operation completes. Fired for text generation
    * (generateText/streamText), object generation (generateObject/streamObject),
-   * embedding (embed/embedMany), and reranking operations.
+   * embedding (embed/embedMany), reranking, speech generation, and
+   * non-streaming transcription operations.
    *
    * Use the event shape or `operationId` to distinguish between operation types.
    */
-  onEnd?: Callback<InferTelemetryEvent<OperationEndEvent>>;
+  onEnd?: BivariantCallback<InferTelemetryEvent<OperationEndEvent>>;
 
   /**
    * Called when a streaming text generation operation is aborted before it

package/src/telemetry/tracing-channel.ts

@@ -9,6 +9,9 @@ export type TelemetryTracingEventType =
   | 'embed'
   | 'embedMany'
   | 'rerank'
+  | 'generateSpeech'
+  | 'transcribe'
+  | 'streamTranscribe'
   | 'experimental_evaluate';
 
 export type TelemetryTracingChannelMessage<EVENT = unknown> = {

package/src/transcribe/index.ts

@@ -8,6 +8,12 @@ export type {
   TranscriptionStreamPart,
 } from './stream-transcribe-result';
 export type { TranscriptionResult } from './transcribe-result';
+export type {
+  StreamTranscriptionEndEvent as Experimental_StreamTranscriptionEndEvent,
+  StreamTranscriptionStartEvent as Experimental_StreamTranscriptionStartEvent,
+  TranscriptionEndEvent,
+  TranscriptionStartEvent,
+} from './transcription-events';
 
 // deprecated exports