ai 7.0.37 → 7.0.38

package/docs/07-reference/01-ai-sdk-core/11-stream-translate.mdx

@@ -0,0 +1,176 @@
+---
+title: experimental_streamTranslate
+description: API Reference for experimental_streamTranslate.
+---
+
+# `experimental_streamTranslate()`
+
+<Note type="warning">
+  `experimental_streamTranslate` is an experimental feature.
+</Note>
+
+Streams a speech-to-speech translation from live raw audio. Models translate
+live source audio into target-language audio and text.
+
+`experimental_streamTranslate` is built on the speech translation model
+specification (`Experimental_SpeechTranslationModelV4`). Provider
+implementations of the specification ship separately — see your provider's
+documentation for available translation models.
+
+```ts
+import { experimental_streamTranslate as streamTranslate } from 'ai';
+
+const result = streamTranslate({
+  // any provider model instance that implements the experimental
+  // speech translation model specification (Experimental_SpeechTranslationModelV4):
+  model: translationModel,
+  audio: audioStream, // ReadableStream<Uint8Array | string>
+  inputAudioFormat: { type: 'audio/pcm', rate: 24000 },
+  targetLanguage: 'es',
+});
+
+for await (const part of result.fullStream) {
+  if (part.type === 'output-text-delta') {
+    process.stdout.write(part.delta);
+  }
+}
+
+console.log(await result.translationText);
+```
+
+## Import
+
+<Snippet
+  text={`import { experimental_streamTranslate as streamTranslate } from "ai"`}
+  prompt={false}
+/>
+
+## API Signature
+
+### Parameters
+
+<PropertiesTable
+  content={[
+    {
+      name: 'model',
+      type: 'Experimental_SpeechTranslationModelV4',
+      description:
+        'The speech translation model to use. Translation is a streaming-only modality. Pass a provider model instance that implements the experimental speech translation model specification. String model IDs resolve through the global provider (AI Gateway by default) when it supports speech translation models.',
+    },
+    {
+      name: 'audio',
+      type: 'ReadableStream<Uint8Array | string>',
+      description:
+        'Raw audio chunks to translate. `Uint8Array` chunks contain raw audio bytes; `string` chunks contain base64-encoded raw audio bytes.',
+    },
+    {
+      name: 'inputAudioFormat',
+      type: '{ type: string; rate?: number }',
+      description:
+        'The input audio format for the raw audio chunks, e.g. `{ type: "audio/pcm", rate: 24000 }`. Supported types are provider-specific (e.g. `audio/pcm`, `audio/pcmu`, `audio/pcma`).',
+    },
+    {
+      name: 'targetLanguage',
+      type: 'string',
+      description:
+        'The language to translate the audio into, as a BCP-47-style language tag (e.g. `en`, `es`, `fr-CA`). Supported values are provider-specific and validated by the provider.',
+    },
+    {
+      name: 'sourceLanguage',
+      type: 'string',
+      isOptional: true,
+      description:
+        'The language of the source audio, as a BCP-47-style language tag. When absent, providers auto-detect the source language.',
+    },
+    {
+      name: 'outputAudioFormat',
+      type: '{ type: string; rate?: number }',
+      isOptional: true,
+      description:
+        'The desired audio format for translated audio chunks. When absent, the provider default output format is used.',
+    },
+    {
+      name: 'providerOptions',
+      type: 'Record<string, JSONObject>',
+      isOptional: true,
+      description: 'Additional provider-specific options.',
+    },
+    {
+      name: 'abortSignal',
+      type: 'AbortSignal',
+      isOptional: true,
+      description: 'An optional abort signal to cancel the call.',
+    },
+    {
+      name: 'headers',
+      type: 'Record<string, string>',
+      isOptional: true,
+      description:
+        'Additional HTTP/WebSocket headers, if supported by the provider.',
+    },
+    {
+      name: 'includeRawChunks',
+      type: 'boolean',
+      isOptional: true,
+      description:
+        'When true, the provider includes raw provider chunks in the stream as `raw` parts.',
+    },
+  ]}
+/>
+
+### Returns
+
+<PropertiesTable
+  content={[
+    {
+      name: 'fullStream',
+      type: 'AsyncIterableStream<TranslationStreamPart>',
+      description:
+        'A single-consumer live stream of translation parts: `audio`, `output-text-delta`, `output-text-final`, `source-transcript-delta`, `source-transcript-partial`, `source-transcript-final`, `raw`, and `error`. Access it once, before any result promise, when both stream parts and final results are needed. Accessing a result promise first consumes the stream internally and makes `fullStream` unavailable.',
+    },
+    {
+      name: 'sourceText',
+      type: 'Promise<string>',
+      description: 'The final source-language transcript of the input audio.',
+    },
+    {
+      name: 'translationText',
+      type: 'Promise<string>',
+      description:
+        'The final translated text in the target language. May resolve to an empty string for providers that produce only audio output.',
+    },
+    {
+      name: 'durationInSeconds',
+      type: 'Promise<number | undefined>',
+      description: 'The duration of the source audio in seconds, if available.',
+    },
+    {
+      name: 'usage',
+      type: 'Promise<Experimental_SpeechTranslationModelV4Usage | undefined>',
+      description:
+        'Usage information for the translation call, if reported by the provider: `inputAudioSeconds`, `inputAudioTokens`, `outputAudioTokens`, `inputTextTokens`, `outputTextTokens` (all optional numbers).',
+    },
+    {
+      name: 'warnings',
+      type: 'Promise<Warning[]>',
+      description:
+        'Warnings for the call, e.g. unsupported settings. Resolves at stream start, or with an empty array at finish if no stream-start was emitted.',
+    },
+    {
+      name: 'response',
+      type: 'Promise<SpeechTranslationModelResponseMetadata>',
+      description: 'Response metadata (timestamp, model ID, headers).',
+    },
+    {
+      name: 'providerMetadata',
+      type: 'Promise<Record<string, JSONObject>>',
+      description: 'Additional provider-specific metadata.',
+    },
+  ]}
+/>
+
+<Note>
+  The result promises settle as the stream is consumed. If you stop consuming
+  `fullStream` early (e.g. `break` out of the loop), the underlying provider
+  connection is closed and pending result promises reject.
+</Note>

package/docs/07-reference/01-ai-sdk-core/index.mdx

@@ -63,6 +63,11 @@ AI SDK Core contains the following main functions:
       description: 'Stream a transcript from live raw audio.',
       href: '/docs/reference/ai-sdk-core/stream-transcribe',
     },
+    {
+      title: 'experimental_streamTranslate()',
+      description: 'Stream a speech-to-speech translation from live raw audio.',
+      href: '/docs/reference/ai-sdk-core/stream-translate',
+    },
     {
       title: 'generateSpeech()',
       description: 'Generate speech audio from text.',

package/docs/07-reference/05-ai-sdk-errors/ai-no-translation-generated-error.mdx

@@ -0,0 +1,26 @@
+---
+title: AI_NoTranslationGeneratedError
+description: Learn how to fix AI_NoTranslationGeneratedError
+---
+
+# AI_NoTranslationGeneratedError
+
+This error occurs when no translation could be generated from the input audio:
+no `audio` part was emitted and the final output text is empty, or the stream
+ended without a finish event.
+
+## Properties
+
+- `response`: Speech translation model response metadata (required in constructor)
+
+## Checking for this Error
+
+You can check if an error is an instance of `AI_NoTranslationGeneratedError` using:
+
+```typescript
+import { NoTranslationGeneratedError } from 'ai';
+
+if (NoTranslationGeneratedError.isInstance(error)) {
+  // Handle the error
+}
+```

package/docs/07-reference/05-ai-sdk-errors/index.mdx

@@ -24,6 +24,7 @@ collapsed: true
 - [AI_NoContentGeneratedError](/docs/reference/ai-sdk-errors/ai-no-content-generated-error)
 - [AI_NoImageGeneratedError](/docs/reference/ai-sdk-errors/ai-no-image-generated-error)
 - [AI_NoTranscriptGeneratedError](/docs/reference/ai-sdk-errors/ai-no-transcript-generated-error)
+- [AI_NoTranslationGeneratedError](/docs/reference/ai-sdk-errors/ai-no-translation-generated-error)
 - [AI_NoVideoGeneratedError](/docs/reference/ai-sdk-errors/ai-no-video-generated-error)
 - [AI_NoObjectGeneratedError](/docs/reference/ai-sdk-errors/ai-no-object-generated-error)
 - [AI_NoOutputGeneratedError](/docs/reference/ai-sdk-errors/ai-no-output-generated-error)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ai",
-  "version": "7.0.37",
+  "version": "7.0.38",
   "type": "module",
   "description": "AI SDK by Vercel - build apps like ChatGPT, Claude, Gemini, and more with a single interface for any model using the Vercel AI Gateway or go direct to OpenAI, Anthropic, Google, or any other model provider.",
   "license": "Apache-2.0",
@@ -42,9 +42,9 @@
     }
   },
   "dependencies": {
-    "@ai-sdk/gateway": "4.0.28",
-    "@ai-sdk/provider": "4.0.3",
-    "@ai-sdk/provider-utils": "5.0.12"
+    "@ai-sdk/gateway": "4.0.29",
+    "@ai-sdk/provider": "4.0.4",
+    "@ai-sdk/provider-utils": "5.0.13"
   },
   "devDependencies": {
     "@edge-runtime/vm": "^5.0.0",

package/src/error/index.ts

@@ -27,6 +27,7 @@ export { NoObjectGeneratedError } from './no-object-generated-error';
 export { NoOutputGeneratedError } from './no-output-generated-error';
 export { NoSpeechGeneratedError } from './no-speech-generated-error';
 export { NoTranscriptGeneratedError } from './no-transcript-generated-error';
+export { NoTranslationGeneratedError } from './no-translation-generated-error';
 export { NoVideoGeneratedError } from './no-video-generated-error';
 export { NoSuchToolError } from './no-such-tool-error';
 export { ToolCallRepairError } from './tool-call-repair-error';

package/src/error/no-translation-generated-error.ts

@@ -0,0 +1,28 @@
+import { AISDKError } from '@ai-sdk/provider';
+import type { SpeechTranslationModelResponseMetadata } from '../types/speech-translation-model-response-metadata';
+
+const name = 'AI_NoTranslationGeneratedError';
+const marker = `vercel.ai.error.${name}`;
+const symbol = Symbol.for(marker);
+
+/**
+ * Error that is thrown when no translation was generated.
+ */
+export class NoTranslationGeneratedError extends AISDKError {
+  private readonly [symbol] = true; // used in isInstance
+
+  readonly response: SpeechTranslationModelResponseMetadata;
+
+  constructor(options: { response: SpeechTranslationModelResponseMetadata }) {
+    super({
+      name,
+      message: 'No translation generated.',
+    });
+
+    this.response = options.response;
+  }
+
+  static isInstance(error: unknown): error is NoTranslationGeneratedError {
+    return AISDKError.hasMarker(error, marker);
+  }
+}

package/src/generate-text/generate-text.ts

@@ -1164,7 +1164,10 @@ export async function generateText<
               // insert error tool outputs for invalid tool calls:
               // TODO AI SDK 6: invalid inputs should not require output parts
               const invalidToolCalls = stepToolCalls.filter(
-                toolCall => toolCall.invalid && toolCall.dynamic,
+                toolCall =>
+                  toolCall.invalid &&
+                  toolCall.dynamic &&
+                  !toolCall.providerExecuted,
               );
 
               clientToolOutputs = [];

package/src/generate-text/stream-language-model-call.ts

@@ -627,18 +627,20 @@ function createLanguageModelV4StreamPartToLanguageModelStreamPartTransform<
             modelCallContent.push(toolCall);
 
             if (toolCall.invalid) {
-              controller.enqueue({
-                type: 'tool-error',
-                toolCallId: toolCall.toolCallId,
-                toolName: toolCall.toolName,
-                input: toolCall.input,
-                error: getErrorMessage(toolCall.error!),
-                dynamic: true,
-                title: toolCall.title,
-                ...(toolCall.toolMetadata != null
-                  ? { toolMetadata: toolCall.toolMetadata }
-                  : {}),
-              });
+              if (!toolCall.providerExecuted) {
+                controller.enqueue({
+                  type: 'tool-error',
+                  toolCallId: toolCall.toolCallId,
+                  toolName: toolCall.toolName,
+                  input: toolCall.input,
+                  error: getErrorMessage(toolCall.error!),
+                  dynamic: true,
+                  title: toolCall.title,
+                  ...(toolCall.toolMetadata != null
+                    ? { toolMetadata: toolCall.toolMetadata }
+                    : {}),
+                });
+              }
               break;
             }
           } catch (error) {

package/src/generate-text/stream-text-result.ts

@@ -300,8 +300,9 @@ export interface StreamTextResult<
 
   /**
    * A text stream that returns only the generated text deltas. You can use it
-   * as either an AsyncIterable or a ReadableStream. When an error occurs, the
-   * stream will throw the error.
+   * as either an AsyncIterable or a ReadableStream. Error parts are not
+   * surfaced in this stream. Use the `onError` callback or `stream` to observe
+   * them.
    */
   readonly textStream: AsyncIterableStream<string>;
 

package/src/index.ts

@@ -46,6 +46,7 @@ export * from './rerank';
 export * from './telemetry';
 export * from './text-stream';
 export * from './transcribe';
+export * from './translate';
 export * from './types';
 export * from './ui';
 export * from './ui-message-stream';

package/src/model/resolve-model.ts

@@ -1,6 +1,7 @@
 import { gateway } from '@ai-sdk/gateway';
 import type {
   EmbeddingModelV4,
+  Experimental_SpeechTranslationModelV4,
   Experimental_VideoModelV4,
   ImageModelV4,
   LanguageModelV4,
@@ -13,6 +14,7 @@ import { UnsupportedModelVersionError } from '../error';
 import type { EmbeddingModel } from '../types/embedding-model';
 import type { LanguageModel } from '../types/language-model';
 import type { SpeechModel } from '../types/speech-model';
+import type { SpeechTranslationModel } from '../types/speech-translation-model';
 import type { TranscriptionModel } from '../types/transcription-model';
 import { asEmbeddingModelV4 } from './as-embedding-model-v4';
 import { asImageModelV4 } from './as-image-model-v4';
@@ -79,6 +81,39 @@ export function resolveTranscriptionModel(
   return asTranscriptionModelV4(model);
 }
 
+export function resolveSpeechTranslationModel(
+  model: SpeechTranslationModel,
+): Experimental_SpeechTranslationModelV4 {
+  if (typeof model === 'string') {
+    // Use raw global provider because speechTranslationModel is experimental
+    // and not part of the ProviderV4 interface
+    const provider = globalThis.AI_SDK_DEFAULT_PROVIDER ?? gateway;
+    // TODO AI SDK v7
+    // @ts-expect-error - speechTranslationModel support is experimental
+    const speechTranslationModel = provider.speechTranslationModel;
+
+    if (!speechTranslationModel) {
+      throw new Error(
+        'The default provider does not support speech translation models. ' +
+          'Please pass a provider model instance that implements the experimental speech translation model specification.',
+      );
+    }
+
+    return speechTranslationModel(model);
+  }
+
+  if (model.specificationVersion !== 'v4') {
+    const unsupportedModel: any = model;
+    throw new UnsupportedModelVersionError({
+      version: unsupportedModel.specificationVersion,
+      provider: unsupportedModel.provider,
+      modelId: unsupportedModel.modelId,
+    });
+  }
+
+  return model;
+}
+
 export function resolveSpeechModel(
   model: SpeechModel,
 ): SpeechModelV4 | undefined {

package/src/test/mock-speech-translation-model-v4.ts

@@ -0,0 +1,24 @@
+import type { Experimental_SpeechTranslationModelV4 } from '@ai-sdk/provider';
+import { notImplemented } from './not-implemented';
+
+export class MockSpeechTranslationModelV4 implements Experimental_SpeechTranslationModelV4 {
+  readonly specificationVersion = 'v4';
+  readonly provider: Experimental_SpeechTranslationModelV4['provider'];
+  readonly modelId: Experimental_SpeechTranslationModelV4['modelId'];
+
+  doStream: Experimental_SpeechTranslationModelV4['doStream'];
+
+  constructor({
+    provider = 'mock-provider',
+    modelId = 'mock-model-id',
+    doStream = notImplemented,
+  }: {
+    provider?: Experimental_SpeechTranslationModelV4['provider'];
+    modelId?: Experimental_SpeechTranslationModelV4['modelId'];
+    doStream?: Experimental_SpeechTranslationModelV4['doStream'];
+  } = {}) {
+    this.provider = provider;
+    this.modelId = modelId;
+    this.doStream = doStream;
+  }
+}

package/src/transcribe/stream-transcribe.ts

@@ -2,6 +2,7 @@ import {
   UnsupportedFunctionalityError,
   type Experimental_TranscriptionModelV4StreamPart,
   type JSONObject,
+  type SharedV4AudioFormat,
 } from '@ai-sdk/provider';
 import {
   DelayedPromise,
@@ -63,17 +64,7 @@ export function streamTranscribe({
   /**
    * The input audio format for the raw audio chunks.
    */
-  inputAudioFormat: {
-    /**
-     * Audio format type, e.g. `audio/pcm`, `audio/pcmu`, or `audio/pcma`.
-     */
-    type: string;
-
-    /**
-     * Sample rate in Hz. Only applicable for formats that require a rate.
-     */
-    rate?: number;
-  };
+  inputAudioFormat: SharedV4AudioFormat;
 
   /**
    * Additional provider-specific options.

package/src/translate/index.ts

@@ -0,0 +1,5 @@
+export { streamTranslate as experimental_streamTranslate } from './stream-translate';
+export type {
+  StreamTranslationResult as Experimental_StreamTranslationResult,
+  TranslationStreamPart as Experimental_TranslationStreamPart,
+} from './stream-translate-result';

package/src/translate/stream-translate-result.ts

@@ -0,0 +1,146 @@
+import type {
+  Experimental_SpeechTranslationModelV4Usage,
+  JSONObject,
+  SharedV4ProviderMetadata,
+} from '@ai-sdk/provider';
+import type { SpeechTranslationModelResponseMetadata } from '../types/speech-translation-model-response-metadata';
+import type { Warning } from '../types/warning';
+import type { AsyncIterableStream } from '../util/async-iterable-stream';
+
+/**
+ * Stream parts emitted by `experimental_streamTranslate`.
+ *
+ * Speech translation model stream parts are passed through unchanged, except
+ * for `stream-start`, `response-metadata`, and `finish`, which are consumed
+ * internally and surfaced via the result promises.
+ */
+export type TranslationStreamPart =
+  | {
+      /**
+       * Translated audio chunk in the target language.
+       *
+       * `Uint8Array` chunks contain raw audio bytes. `string` chunks contain
+       * base64-encoded raw audio bytes.
+       */
+      type: 'audio';
+      id?: string;
+      audio: Uint8Array | string;
+      providerMetadata?: SharedV4ProviderMetadata;
+    }
+  | {
+      /**
+       * Append-only translated text delta.
+       */
+      type: 'output-text-delta';
+      id?: string;
+      delta: string;
+      providerMetadata?: SharedV4ProviderMetadata;
+    }
+  | {
+      /**
+       * Final translated text for a provider-defined segment or utterance.
+       */
+      type: 'output-text-final';
+      id?: string;
+      text: string;
+      providerMetadata?: SharedV4ProviderMetadata;
+    }
+  | {
+      /**
+       * Append-only source transcript delta.
+       */
+      type: 'source-transcript-delta';
+      id?: string;
+      delta: string;
+      providerMetadata?: SharedV4ProviderMetadata;
+    }
+  | {
+      /**
+       * Non-final source transcript text. The text may be revised by later parts.
+       */
+      type: 'source-transcript-partial';
+      id?: string;
+      text: string;
+      startSecond?: number;
+      endSecond?: number;
+      channelIndex?: number;
+      providerMetadata?: SharedV4ProviderMetadata;
+    }
+  | {
+      /**
+       * Final source transcript text for a provider-defined segment or utterance.
+       */
+      type: 'source-transcript-final';
+      id?: string;
+      text: string;
+      startSecond?: number;
+      endSecond?: number;
+      channelIndex?: number;
+      providerMetadata?: SharedV4ProviderMetadata;
+    }
+  | {
+      /**
+       * Raw provider chunks if enabled via `includeRawChunks`.
+       */
+      type: 'raw';
+      rawValue: unknown;
+    }
+  | {
+      /**
+       * Error parts are streamed, allowing for multiple errors.
+       */
+      type: 'error';
+      error: unknown;
+    };
+
+export interface StreamTranslationResult {
+  /**
+   * The final source-language transcript of the input audio.
+   */
+  readonly sourceText: PromiseLike<string>;
+
+  /**
+   * The final translated text in the target language.
+   *
+   * May resolve to an empty string for providers that produce only audio
+   * output.
+   */
+  readonly translationText: PromiseLike<string>;
+
+  /**
+   * The duration of the source audio in seconds, if available.
+   */
+  readonly durationInSeconds: PromiseLike<number | undefined>;
+
+  /**
+   * Usage information for the translation call, if reported by the provider.
+   */
+  readonly usage: PromiseLike<
+    Experimental_SpeechTranslationModelV4Usage | undefined
+  >;
+
+  /**
+   * Warnings for the call, e.g. unsupported settings.
+   */
+  readonly warnings: PromiseLike<Array<Warning>>;
+
+  /**
+   * Response metadata.
+   */
+  readonly response: PromiseLike<SpeechTranslationModelResponseMetadata>;
+
+  /**
+   * Additional provider-specific metadata.
+   */
+  readonly providerMetadata: PromiseLike<Record<string, JSONObject>>;
+
+  /**
+   * Full stream of translation parts.
+   *
+   * This is a single-consumer live stream and can only be accessed once.
+   * Access it before any result promise when both stream parts and final
+   * results are needed; accessing a result promise first consumes the stream
+   * internally and makes `fullStream` unavailable.
+   */
+  readonly fullStream: AsyncIterableStream<TranslationStreamPart>;
+}