ai 7.0.149 → 7.0.151

package/docs/07-reference/01-ai-sdk-core/12-stream-speech.mdx

@@ -0,0 +1,195 @@
+---
+title: experimental_streamSpeech
+description: API Reference for experimental_streamSpeech.
+---
+
+# `experimental_streamSpeech()`
+
+<Note type="warning">
+  `experimental_streamSpeech` is an experimental feature.
+</Note>
+
+Streams speech audio from text, or from an array or stream of text deltas and
+text parts, using a speech model with streaming support.
+
+```ts
+import { experimental_streamSpeech as streamSpeech } from 'ai';
+import { elevenLabs } from '@ai-sdk/elevenlabs';
+
+const result = streamSpeech({
+  model: elevenLabs.speech('eleven_flash_v2_5'),
+  text: 'Hello, world!',
+});
+
+for await (const chunk of result.audioStream) {
+  // Uint8Array audio bytes
+}
+```
+
+## Import
+
+<Snippet
+  text={`import { experimental_streamSpeech as streamSpeech } from "ai"`}
+  prompt={false}
+/>
+
+## API Signature
+
+### Parameters
+
+<PropertiesTable
+  content={[
+    {
+      name: 'model',
+      type: 'SpeechModel',
+      description:
+        'The speech model to use. The model must implement the experimental streaming speech extension (`Experimental_StreamingSpeechModelV4`).',
+    },
+    {
+      name: 'text',
+      type: 'string | Iterable<string | Experimental_SpeechTextPart> | AsyncIterable<string | Experimental_SpeechTextPart> | ReadableStream<string | Experimental_SpeechTextPart>',
+      description:
+        "The text to convert to speech: a complete string, or an array, iterable, or stream of text deltas (e.g. the `textStream` of `streamText`) and/or text parts `{ type: 'text', text, voice?, providerOptions? }`. A part's `voice` selects its speaker on providers that support several voices per stream; single-voice providers error the stream when a part's voice differs from the call's voice. The end of the stream ends the input.",
+    },
+    {
+      name: 'voice',
+      type: 'string',
+      isOptional: true,
+      description: 'The voice to use for speech generation.',
+    },
+    {
+      name: 'outputFormat',
+      type: '{ type: string; rate?: number }',
+      isOptional: true,
+      description:
+        "The format of the output audio chunks, e.g. `{ type: 'audio/pcm', rate: 24000 }` or `{ type: 'audio/mpeg' }`. Supported formats are provider-specific.",
+    },
+    {
+      name: 'instructions',
+      type: 'string',
+      isOptional: true,
+      description: 'Instructions for the speech generation.',
+    },
+    {
+      name: 'speed',
+      type: 'number',
+      isOptional: true,
+      description: 'The speed of the speech generation.',
+    },
+    {
+      name: 'language',
+      type: 'string',
+      isOptional: true,
+      description:
+        'The language for speech generation (ISO 639-1 code), or "auto". Provider support varies.',
+    },
+    {
+      name: 'providerOptions',
+      type: 'Record<string, JSONObject>',
+      isOptional: true,
+      description: 'Additional provider-specific options.',
+    },
+    {
+      name: 'abortSignal',
+      type: 'AbortSignal',
+      isOptional: true,
+      description: 'An optional abort signal to cancel the call.',
+    },
+    {
+      name: 'headers',
+      type: 'Record<string, string>',
+      isOptional: true,
+      description:
+        'Additional HTTP/WebSocket headers, if supported by the provider.',
+    },
+    {
+      name: 'includeRawChunks',
+      type: 'boolean',
+      isOptional: true,
+      description:
+        'When true, the provider includes raw provider chunks in `fullStream` as `raw` parts.',
+    },
+    {
+      name: 'onError',
+      type: '(event: { error: unknown }) => void | PromiseLike<void>',
+      isOptional: true,
+      description:
+        'Called for each `error` part the provider reports. Defaults to `console.error`.',
+    },
+  ]}
+/>
+
+### Returns
+
+<PropertiesTable
+  content={[
+    {
+      name: 'audioStream',
+      type: 'AsyncIterableStream<Uint8Array>',
+      description:
+        'Audio bytes in generation order. Shares one live stream with `fullStream`: only one of them can be accessed, once, before any result promise. `error` parts are skipped (they are reported via `onError`); the stream errors only when the speech stream itself fails.',
+    },
+    {
+      name: 'fullStream',
+      type: 'AsyncIterableStream<Experimental_SpeechStreamPart>',
+      description:
+        'A single-consumer live stream of `audio` (with `audio: Uint8Array` and `audioFormat`), `raw`, and `error` parts. Error parts are not fatal; the stream continues after them.',
+    },
+    {
+      name: 'usage',
+      type: 'Promise<{ inputCharacters?: number; inputTokens?: number; outputTokens?: number; raw?: JSONObject }>',
+      description: 'Usage information for the call, if known.',
+    },
+    {
+      name: 'warnings',
+      type: 'Promise<Warning[]>',
+      description:
+        'Warnings for the call, e.g. unsupported settings. Resolves when the provider emits the stream start.',
+    },
+    {
+      name: 'responses',
+      type: 'Promise<Array<SpeechModelResponseMetadata>>',
+      description: 'Response metadata (timestamp, model ID, headers, body).',
+    },
+    {
+      name: 'providerMetadata',
+      type: 'Promise<Record<string, JSONObject>>',
+      description: 'Additional provider-specific metadata.',
+    },
+  ]}
+/>
+
+<Note>
+  The result promises settle as the stream is consumed. If you cancel the stream
+  early (for example by breaking out of a `for await` loop) or abort the call,
+  pending result promises reject immediately and the provider is signalled to
+  close its connection. A stream that is not read stalls under backpressure and
+  keeps the provider call open.
+</Note>
+
+## Errors
+
+- `UnsupportedFunctionalityError` is thrown synchronously when the model does
+  not support streaming speech.
+- `InvalidArgumentError` is thrown synchronously when `text` is not a string,
+  an iterable or async iterable, or a readable stream.
+- Provider `error` parts are not fatal. They appear in `fullStream`, are
+  skipped by `audioStream`, and are passed to `onError`.
+- If the stream ends without audio, the result rejects with
+  `NoSpeechGeneratedError`, whose `cause` is the last provider error part.
+- If the stream stops after audio but before it finished, the result rejects
+  with the last provider error part, or with a `StreamProviderError`.
+- If the provider call fails (for example an authentication error), the result
+  rejects with that error. Cancelling or aborting rejects with the
+  cancellation or abort reason.
+
+## Wire format (experimental)
+
+Streaming speech over WebSocket is serialized with the experimental
+speech-stream envelope defined in `@ai-sdk/provider-utils`
+(`experimental_parseSpeechStreamClientFrame`,
+`experimental_serializeSpeechStreamPart`, `experimental_parseSpeechStreamPart`):
+the client sends one `speech-stream.start` TEXT frame, input text as
+`speech-stream.text-delta` TEXT frames (each carrying its text part's `voice`
+and `providerOptions`), and a `speech-stream.text-done` TEXT frame; each server TEXT frame is one JSON-serialized speech stream part with
+base64 audio. AI Gateway implements the server side of this envelope.

package/docs/07-reference/01-ai-sdk-core/index.mdx

@@ -98,6 +98,11 @@ AI SDK Core contains the following main functions:
       description: 'Generate speech audio from text.',
       href: '/docs/reference/ai-sdk-core/generate-speech',
     },
+    {
+      title: 'experimental_streamSpeech()',
+      description: 'Stream speech audio from text or a text stream.',
+      href: '/docs/reference/ai-sdk-core/stream-speech',
+    },
     {
       title: 'uploadFile()',
       description: 'Upload a file to a provider and get a provider reference.',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ai",
-  "version": "7.0.149",
+  "version": "7.0.151",
   "type": "module",
   "description": "AI SDK by Vercel - build apps like ChatGPT, Claude, Gemini, and more with a single interface for any model using the Vercel AI Gateway or go direct to OpenAI, Anthropic, Google, or any other model provider.",
   "license": "Apache-2.0",
@@ -42,20 +42,20 @@
     }
   },
   "dependencies": {
-    "@ai-sdk/gateway": "4.0.122",
-    "@ai-sdk/provider": "4.0.32",
-    "@ai-sdk/provider-utils": "5.0.66"
+    "@ai-sdk/gateway": "4.0.124",
+    "@ai-sdk/provider": "4.0.33",
+    "@ai-sdk/provider-utils": "5.0.67"
   },
   "devDependencies": {
-    "@ai-sdk/amazon-bedrock": "5.0.122",
-    "@ai-sdk/deepseek": "3.0.71",
-    "@ai-sdk/google": "4.0.103",
-    "@ai-sdk/groq": "4.0.68",
-    "@ai-sdk/huggingface": "2.0.76",
-    "@ai-sdk/moonshotai": "3.0.75",
-    "@ai-sdk/openai": "4.0.100",
+    "@ai-sdk/amazon-bedrock": "5.0.123",
+    "@ai-sdk/deepseek": "3.0.72",
+    "@ai-sdk/google": "4.0.104",
+    "@ai-sdk/groq": "4.0.69",
+    "@ai-sdk/huggingface": "2.0.77",
+    "@ai-sdk/moonshotai": "3.0.76",
+    "@ai-sdk/openai": "4.0.101",
     "@ai-sdk/test-server": "2.0.4",
-    "@ai-sdk/xai": "5.0.28",
+    "@ai-sdk/xai": "5.0.29",
     "@edge-runtime/vm": "^5.0.0",
     "@smithy/eventstream-codec": "^4.3.3",
     "@smithy/util-utf8": "^4.3.3",

package/src/error/no-speech-generated-error.ts

@@ -13,10 +13,14 @@ export class NoSpeechGeneratedError extends AISDKError {
 
   readonly responses: Array<SpeechModelResponseMetadata>;
 
-  constructor(options: { responses: Array<SpeechModelResponseMetadata> }) {
+  constructor(options: {
+    responses: Array<SpeechModelResponseMetadata>;
+    cause?: unknown;
+  }) {
     super({
       name,
       message: 'No speech audio generated.',
+      ...(options.cause !== undefined && { cause: options.cause }),
     });
 
     this.responses = options.responses;

package/src/generate-speech/index.ts

@@ -3,6 +3,13 @@ import { generateSpeech } from './generate-speech';
 import { logWarnings } from '../logger/log-warnings';
 
 export { generateSpeech } from './generate-speech';
+export { streamSpeech as experimental_streamSpeech } from './stream-speech';
+export type {
+  SpeechStreamPart as Experimental_SpeechStreamPart,
+  SpeechStreamUsage as Experimental_SpeechStreamUsage,
+  SpeechTextPart as Experimental_SpeechTextPart,
+  StreamSpeechResult as Experimental_StreamSpeechResult,
+} from './stream-speech-result';
 export type { SpeechResult } from './generate-speech-result';
 export type { GeneratedAudioFile } from './generated-audio-file';
 export type {

package/src/generate-speech/is-streaming-speech-model.ts

@@ -0,0 +1,17 @@
+import type {
+  Experimental_StreamingSpeechModelV4,
+  SpeechModelV4,
+} from '@ai-sdk/provider';
+
+/**
+ * Whether a resolved speech model implements the experimental streaming
+ * speech extension (`Experimental_StreamingSpeechModelV4`).
+ */
+export function isStreamingSpeechModel(
+  model: SpeechModelV4,
+): model is Experimental_StreamingSpeechModelV4 {
+  return (
+    typeof (model as Partial<Experimental_StreamingSpeechModelV4>).doStream ===
+    'function'
+  );
+}

package/src/generate-speech/stream-speech-result.ts

@@ -0,0 +1,95 @@
+import type {
+  Experimental_SpeechModelV4StreamUsage,
+  JSONObject,
+  SharedV4AudioFormat,
+  SharedV4ProviderMetadata,
+} from '@ai-sdk/provider';
+import type { ProviderOptions } from '@ai-sdk/provider-utils';
+import type { SpeechModelResponseMetadata } from '../types/speech-model-response-metadata';
+import type { Warning } from '../types/warning';
+import type { AsyncIterableStream } from '../util/async-iterable-stream';
+
+/**
+ * A piece of text to synthesize. Pass a stream of these (or mix them with
+ * plain strings) to give parts their own voice, e.g. for dialogue.
+ */
+export type SpeechTextPart = {
+  type: 'text';
+  text: string;
+
+  /**
+   * The voice that speaks this part (provider-specific, like the call's
+   * `voice`). Single-voice providers reject a voice that differs from the
+   * call's voice.
+   */
+  voice?: string;
+
+  /**
+   * Additional provider-specific options for this part.
+   */
+  providerOptions?: ProviderOptions;
+};
+
+export type SpeechStreamPart =
+  | {
+      /**
+       * Audio chunk in generation order. Chunks are not necessarily
+       * standalone files; concatenate them to reconstruct the audio. A chunk
+       * is empty only when it carries `providerMetadata` (e.g. a turn marker).
+       */
+      type: 'audio';
+      id?: string;
+      audio: Uint8Array;
+      audioFormat: SharedV4AudioFormat;
+      providerMetadata?: SharedV4ProviderMetadata;
+    }
+  | {
+      type: 'raw';
+      rawValue: unknown;
+    }
+  | {
+      type: 'error';
+      error: unknown;
+    };
+
+export type SpeechStreamUsage = Experimental_SpeechModelV4StreamUsage;
+
+export interface StreamSpeechResult {
+  /**
+   * Usage information reported for the call, if available.
+   */
+  readonly usage: PromiseLike<SpeechStreamUsage>;
+
+  /**
+   * Warnings for the call, e.g. unsupported settings.
+   */
+  readonly warnings: PromiseLike<Array<Warning>>;
+
+  /**
+   * Response metadata.
+   */
+  readonly responses: PromiseLike<Array<SpeechModelResponseMetadata>>;
+
+  /**
+   * Additional provider-specific metadata.
+   */
+  readonly providerMetadata: PromiseLike<Record<string, JSONObject>>;
+
+  /**
+   * Stream of audio bytes in generation order.
+   *
+   * `audioStream` and `fullStream` share one live stream: only one of them
+   * can be accessed, and only once. Access it before any result promise;
+   * accessing a result promise first consumes the stream internally.
+   */
+  readonly audioStream: AsyncIterableStream<Uint8Array>;
+
+  /**
+   * Full stream of speech parts (audio, raw, and error parts).
+   *
+   * `audioStream` and `fullStream` share one live stream: only one of them
+   * can be accessed, and only once. Access it before any result promise;
+   * accessing a result promise first consumes the stream internally.
+   */
+  readonly fullStream: AsyncIterableStream<SpeechStreamPart>;
+}